diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
new file mode 100644
index 0000000..1c880bd
--- /dev/null
+++ b/.github/CODEOWNERS
@@ -0,0 +1 @@
+* @MicroMilo
diff --git a/.github/ISSUE_TEMPLATE/correction.yml b/.github/ISSUE_TEMPLATE/correction.yml
new file mode 100644
index 0000000..d62a182
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/correction.yml
@@ -0,0 +1,53 @@
+name: Correct a catalog record
+description: Report incorrect metadata, evidence, classification, or a broken primary link.
+title: "[Correction]: "
+labels: [metadata-correction]
+body:
+ - type: input
+ id: record
+ attributes:
+ label: Catalog record
+ description: Paper title, system name, or catalog ID.
+ validations:
+ required: true
+ - type: dropdown
+ id: correction_type
+ attributes:
+ label: Correction type
+ options:
+ - Product, model, or version
+ - Venue or publication status
+ - Method or task description
+ - Quantitative result
+ - Comparison fairness or caveat
+ - Authors or identifiers
+ - Artifact or broken link
+ - Other metadata
+ validations:
+ required: true
+ - type: textarea
+ id: current
+ attributes:
+ label: Current catalog statement
+ validations:
+ required: true
+ - type: textarea
+ id: corrected
+ attributes:
+ label: Proposed correction
+ validations:
+ required: true
+ - type: input
+ id: source
+ attributes:
+ label: Primary-source URL
+ description: Link to the paper, proceedings, DOI, OpenReview, arXiv, or official artifact.
+ validations:
+ required: true
+ - type: checkboxes
+ id: checks
+ attributes:
+ label: Submission checks
+ options:
+ - label: The correction is supported by the linked primary source.
+ required: true
diff --git a/.github/ISSUE_TEMPLATE/paper.yml b/.github/ISSUE_TEMPLATE/paper.yml
index 6a9a7cf..705ad5b 100644
--- a/.github/ISSUE_TEMPLATE/paper.yml
+++ b/.github/ISSUE_TEMPLATE/paper.yml
@@ -16,6 +16,13 @@ body:
description: Official proceedings, DOI, OpenReview, or arXiv URL.
validations:
required: true
+ - type: input
+ id: system
+ attributes:
+ label: Proposed system or benchmark name
+ description: For example, QLCoder, ZeroRepo, or Terminal-Bench 2.0.
+ validations:
+ required: true
- type: dropdown
id: products
attributes:
@@ -42,7 +49,14 @@ body:
id: evidence
attributes:
label: Baseline and result evidence
- description: Quote table or section numbers and paraphrase the result. Include model and budget details when available.
+ description: Give table, figure, section, or appendix locations and paraphrase the result. Include model and budget details when available.
+ validations:
+ required: true
+ - type: textarea
+ id: configuration
+ attributes:
+ label: Product configuration
+ description: Product/CLI version, model snapshot, reasoning mode, tool permissions, time/turn/token budget, and number of runs. Use "not reported" when needed.
validations:
required: true
- type: input
diff --git a/.github/workflows/discover.yml b/.github/workflows/discover.yml
new file mode 100644
index 0000000..5fa3da2
--- /dev/null
+++ b/.github/workflows/discover.yml
@@ -0,0 +1,36 @@
+name: Discover paper candidates
+
+on:
+ schedule:
+ - cron: "17 4 * * 1"
+ workflow_dispatch:
+
+permissions:
+ contents: read
+ issues: write
+
+jobs:
+ discover:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+ - uses: actions/setup-python@v5
+ with:
+ python-version: "3.12"
+ cache: pip
+ - name: Install catalog dependencies
+ run: python -m pip install -r requirements-dev.txt
+ - name: Discover recent candidates
+ id: discover
+ run: python scripts/discover_candidates.py --output candidate-report.md --github-output
+ - name: Create or update candidate issue
+ if: steps.discover.outputs.count != '0'
+ env:
+ GH_TOKEN: ${{ github.token }}
+ run: |
+ issue_number="$(gh issue list --state open --label paper-candidate --search 'Weekly paper candidates in:title' --json number --jq '.[0].number')"
+ if [ -n "$issue_number" ]; then
+ gh issue edit "$issue_number" --body-file candidate-report.md
+ else
+ gh issue create --title "Weekly paper candidates" --label paper-candidate --body-file candidate-report.md
+ fi
diff --git a/.gitignore b/.gitignore
index bdaace1..023ac76 100644
--- a/.gitignore
+++ b/.gitignore
@@ -4,3 +4,4 @@ __pycache__/
.ruff_cache/
*.pyc
.DS_Store
+candidate-report.md
diff --git a/AGENTS.md b/AGENTS.md
new file mode 100644
index 0000000..ee1b564
--- /dev/null
+++ b/AGENTS.md
@@ -0,0 +1,25 @@
+# Repository instructions
+
+## Source of truth
+
+- Edit `data/papers.yaml`; do not hand-edit generated catalog tables, `data/papers.json`, `papers/*.md`, or `views/*.md`.
+- Update `data/schema.json` and `docs/taxonomy.md` together when adding fields, products, statuses, or method tags.
+- A product mention means the complete agent or harness, not merely a model from the same vendor.
+
+## Evidence rules
+
+- Prefer official proceedings, DOI, OpenReview, arXiv, and author-maintained artifacts.
+- Record paper claims as claims; do not silently convert them into universal conclusions.
+- Use `unknown` when model or budget parity is not established.
+- Keep direct, related, evaluation-only, and historical entries separate.
+- Never commit paper PDFs.
+
+## Commands
+
+```bash
+python -m pip install -r requirements-dev.txt
+make build
+make check
+```
+
+Run `make build` after catalog edits and include all generated changes in the same commit.
diff --git a/CHANGELOG.md b/CHANGELOG.md
new file mode 100644
index 0000000..91e8919
--- /dev/null
+++ b/CHANGELOG.md
@@ -0,0 +1,23 @@
+# Changelog
+
+All notable catalog and schema changes are recorded here.
+
+## [Unreleased]
+
+### Planned
+
+- Deeper audits of product versions, model snapshots, and execution budgets
+- Additional official artifacts and evidence anchors
+
+## [0.1.0] — 2026-08-21
+
+### Added
+
+- Initial Claude Code and Codex CLI scope
+- Nineteen reviewed seed papers
+- YAML catalog, JSON Schema, and generated JSON export
+- Per-paper evidence dossiers and research views
+- English and Simplified Chinese READMEs
+- Automated generation, validation, tests, linting, and GitHub Actions
+- Weekly arXiv candidate discovery with deduplication and bounded issue reports
+- Contribution, citation, security, and community guidance
diff --git a/CITATION.cff b/CITATION.cff
new file mode 100644
index 0000000..e8dedff
--- /dev/null
+++ b/CITATION.cff
@@ -0,0 +1,19 @@
+cff-version: 1.2.0
+message: "If this catalog supports your research, please cite the repository and the original papers you use."
+title: "Awesome Claude Code & Codex Papers"
+type: dataset
+authors:
+ - name: MicroMilo
+repository-code: "https://github.com/MicroMilo/awesome-claude-code-codex-papers"
+url: "https://github.com/MicroMilo/awesome-claude-code-codex-papers"
+abstract: "An evidence-first catalog of research papers that evaluate, analyze, or improve Claude Code and Codex CLI as production coding-agent products."
+keywords:
+ - coding agents
+ - Claude Code
+ - Codex CLI
+ - agentic software engineering
+ - benchmarks
+ - harness engineering
+license: MIT
+version: 0.1.0
+date-released: "2026-08-21"
diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md
new file mode 100644
index 0000000..31816da
--- /dev/null
+++ b/CODE_OF_CONDUCT.md
@@ -0,0 +1,21 @@
+# Code of Conduct
+
+## Our standard
+
+We want research disagreements to be precise, constructive, and welcoming. Contributors should:
+
+- critique claims and methods rather than people;
+- distinguish verified facts, paper claims, and personal interpretation;
+- disclose relevant affiliations or conflicts when they materially affect a contribution;
+- avoid harassment, discrimination, personal attacks, and deliberate disruption;
+- respect paper authors, tool maintainers, and other contributors even when correcting their work.
+
+## Research-integrity expectations
+
+Do not inflate results, hide comparison caveats, fabricate citations, or present model-only experiments as product-agent evidence. Good-faith corrections are encouraged and will be recorded transparently.
+
+## Enforcement
+
+Report conduct concerns privately through the repository Security tab. Maintainers may edit or remove contributions, lock discussions, or restrict participation when needed to protect the project and its community.
+
+This policy applies to repository issues, pull requests, discussions, and other project-managed spaces.
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 7f9b9fc..cd4029d 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -30,11 +30,22 @@ same_budget: unknown
1. Add one entry to `data/papers.yaml`.
2. Use a stable identifier such as `short-title-year`.
-3. Choose the narrowest correct evidence class.
-4. Run `make build` to regenerate README tables.
-5. Run `make check` before opening a pull request.
+3. Record authors, identifiers, system name, artifact status, and publication date from primary sources.
+4. Choose the narrowest correct evidence class and comparison scope.
+5. Record where the supporting result appears in the paper.
+6. Run `make build` to regenerate READMEs, JSON, views, and paper dossiers.
+7. Run `make check` before opening a pull request.
-Do not manually edit content between `CATALOG:*:START` and `CATALOG:*:END` markers in `README.md`; those sections are generated.
+Do not manually edit content between `CATALOG:*:START` and `CATALOG:*:END` markers in either README. Files under `papers/` and `views/`, plus `data/papers.json`, are also generated.
+
+## Required source checks
+
+- Confirm the paper title and authors against an official paper page or PDF.
+- Distinguish arXiv publication dates from final venue publication.
+- Confirm that an artifact repository is author-maintained before marking it `official`.
+- Record the exact CLI version and model when the paper or artifact reports them.
+- Identify whether the result is product-level, component-level, a configuration ablation, benchmark-only, or historical-model evidence.
+- Keep missing details as `not-reported`, `not-found`, or `unknown`; absence is useful data.
## Evidence classes
@@ -58,6 +69,8 @@ Future products must be added to both `data/schema.json` and `docs/taxonomy.md`
- [ ] The paper satisfies the inclusion criteria.
- [ ] All quantitative claims match the cited primary source.
- [ ] Model, version, budget, and tool details are recorded or marked unknown.
+- [ ] Author, identifier, publication, and artifact fields come from primary sources.
+- [ ] The evidence location and comparison scope are recorded.
- [ ] Comparison caveats are explicit.
- [ ] `make check` passes.
- [ ] No copyrighted PDF has been committed.
diff --git a/Makefile b/Makefile
index da9f295..a1f362e 100644
--- a/Makefile
+++ b/Makefile
@@ -14,8 +14,11 @@ test:
lint:
python -m ruff check scripts tests
+ python -m ruff format --check scripts tests
check: validate
python scripts/build_readme.py --check
+ python scripts/check_links.py
python -m pytest
python -m ruff check scripts tests
+ python -m ruff format --check scripts tests
diff --git a/README.md b/README.md
index c1ed3b7..2590737 100644
--- a/README.md
+++ b/README.md
@@ -1,137 +1,187 @@
-# Awesome Claude Code & Codex Papers
+
+ 
+
-> Research papers that evaluate, analyze, or improve **Claude Code** and **Codex CLI**.
+Awesome Claude Code & Codex Papers
-This is an evidence-first catalog of research on production coding agents. It records not only which papers mention an agent, but also the exact baseline role, task, method, reported result, experimental controls, and reproducibility artifacts.
+
+ Research that measures, analyzes, or beats real production coding agents.
+
-The initial scope is intentionally narrow: **Claude Code** and **Codex CLI**. The schema is product-agnostic so future releases can add Kimi Code, Zed, Cursor, Gemini CLI, and other industrial coding agents without changing the data model.
+
+ English · 简体中文
+
-## Why this catalog exists
+
+ 
+ 
+ 
+ 
+
-Broad agent-paper lists answer “what has been published?” This catalog focuses on a more operational question:
+
+
+ 
+ 
+ 
+ 
+ 
+
+
-> When researchers evaluate or outperform a production coding agent, what did they add, what exactly improved, and was the comparison fair?
+This repository is an evidence-first catalog of papers that study **Claude Code** and **Codex CLI** as complete products—not merely Claude or GPT-family models. It records the baseline configuration, task, intervention, reported result, comparison controls, caveats, and official artifacts.
-Each entry is reviewed for:
+> [!IMPORTANT]
+> This is not a leaderboard. Product versions, backbone models, budgets, tools, and task domains often differ. Every direct result links to a paper dossier that makes those differences explicit.
-- the product and model used as the baseline;
-- the task, benchmark, and resource budget;
-- the method added around or instead of the baseline;
-- the reported metric and result;
-- same-model and same-budget comparability;
-- paper, code, data, and reproduction links.
+## Start here
-## Scope
+| View | Best for |
+|---|---|
+| [All paper dossiers](papers/README.md) | Inspecting authors, products, models, versions, evidence locations, and caveats |
+| [By product](views/by-product.md) | Finding Claude Code or Codex CLI papers |
+| [By method](views/by-method.md) | Studying retrieval, repository graphs, verification, multi-agent coordination, and harness evolution |
+| [Comparison fairness](views/fair-comparisons.md) | Separating controlled comparisons from model or budget confounds |
+| [By venue](views/by-venue.md) | Browsing main conferences, workshops, benchmark tracks, and preprints |
+| [Machine-readable catalog](data/papers.json) | Building tools, visualizations, or downstream surveys |
-Included:
+## The question this catalog answers
-- papers that directly run Claude Code or Codex CLI;
-- papers that improve, wrap, configure, or compare their agent harnesses;
-- benchmarks and empirical studies that report product-level results;
-- older OpenAI Codex model papers, clearly separated from Codex CLI.
+Broad agent-paper lists answer **“What has been published?”** This catalog asks:
-Not included in the primary catalog:
+> **When researchers evaluate or outperform Claude Code or Codex CLI, what did they add, what improved, and was the comparison fair?**
-- papers that only use a Claude or GPT model without the corresponding product agent;
-- blog posts or product comparisons without a research artifact;
-- claims that cannot be traced to a paper or official artifact.
+Each reviewed entry captures:
-## Evidence classes
+- product, CLI version, model, and role in the experiment;
+- task, benchmark, and evaluation scale;
+- the method added around or instead of the product baseline;
+- reported quality, efficiency, or diagnostic evidence;
+- same-model and same-budget controls;
+- evidence location, caveats, and artifact availability.
+
+## Scope
+
+The initial scope is deliberately narrow: **Claude Code** and **Codex CLI**. The schema is already product-agnostic, so later releases can add Kimi Code, Zed, Cursor, Gemini CLI, and other industrial coding agents without redesigning the catalog.
+
+Included:
+
+- direct product-level comparisons;
+- harness, configuration, retrieval, planning, and verification improvements;
+- benchmarks and empirical studies with product-level results;
+- historical OpenAI Codex model papers, isolated from modern Codex CLI results.
-- **Direct comparison** — the proposed method is numerically compared with Claude Code or Codex CLI.
-- **Related method** — the product is discussed or partially used, but the key numeric comparison is not a clean product-level head-to-head.
-- **Evaluation only** — the paper evaluates the product without proposing a method intended to beat it.
-- **Historical model** — the paper studies the pre-CLI OpenAI Codex model.
+Excluded from the primary catalog:
+
+- papers that use a vendor model but never run the corresponding product agent;
+- blog-only or marketing comparisons without a research artifact;
+- claims that cannot be traced to a primary paper or official artifact.
## Direct comparisons
+These papers numerically compare a proposed method with Claude Code or Codex CLI. “Direct” does not imply same-model or same-budget parity—check the controls column and linked dossier.
+
-| Paper | Venue | Product baseline | Task / benchmark | Method | Reported result | Controls |
+| System / paper | Venue | Product baseline | Task | What changed | Reported evidence | Controls |
|---|---|---|---|---|---|---|
-| [Agentic Harness Engineering: Observability-Driven Automatic Evolution of Coding-Agent Harnesses](https://arxiv.org/abs/2604.25850) | arXiv 2026
preprint | Codex CLI | Automatic evolution of coding-agent harnesses
Terminal-Bench 2 and SWE-bench Verified | Closed-loop harness evolution using component, trajectory, and decision observability with evidence-backed edits and rollback
`harness-evolution` `observability` `memory` `verifier-loop` | Terminal-Bench 2 pass@1 improves from 69.7 to 77.0, above a human-designed Codex-CLI harness at 71.9; transfer uses 12% fewer tokens on SWE-bench Verified. | model: unknown
budget: unknown
evidence: medium |
-| [Comparing AI Agents to Cybersecurity Professionals in Real-World Penetration Testing](https://arxiv.org/abs/2512.09882)
[artifact](https://github.com/Stanford-Trinity/ARTEMIS) | ICLR 2026
main | Claude Code (Claude Sonnet 4)
Codex CLI (GPT-5) | Long-horizon penetration testing in a live enterprise network
Approximately 8,000 hosts across 12 subnets | Supervisor with dynamic expert agents, recursive task decomposition, parallel exploration, context management, triage, and reporting
`multi-agent` `dynamic-specialization` `parallelism` `memory` `verifier-loop` | ARTEMIS found 9 valid vulnerabilities with an 82% valid-submission rate, ranked second overall, and outperformed 9 of 10 human participants. | model: unknown
budget: different
evidence: medium |
-| [FormAct: Agentic Source Editing for Rich-Format Document Generation](https://openreview.net/forum?id=n7Ta0YEcgw) | ICML 2026
main | Codex CLI | Rich-format document generation and editing
RichDocBench and RichDocFuzz | HTML source editor, template retrieval, rendered-page review agent, iterative refinement, and edit-triggered context compression
`retrieval` `context-reduction` `verifier-loop` `visual-review` | Render correctness 4.81 versus 4.39 for multi-pass Codex; human rank-1 rate 0.760 versus 0.140. | model: unknown
budget: unknown
evidence: high |
-| [On the Impact of AGENTS.md Files on the Efficiency of AI Coding Agents](https://conf.researchr.org/details/icse-2026/jaws-2026-papers/31/On-the-Impact-of-AGENTS-md-Files-on-the-Efficiency-of-AI-Coding-Agents) | ICSE JAWs 2026
workshop | Claude Code
Codex CLI | Real pull-request tasks with and without repository instructions
10 repositories and 124 pull requests | Persistent repository-level build, test, style, and directory guidance in AGENTS.md
`repository-instructions` `structured-state` `context-reduction` | Median runtime falls 28.64% and output tokens fall 16.58% with comparable completion behavior. | model: same
budget: same
evidence: high |
-| [Prefactory: Automated Discovery and Application of Library-Adoption Refactorings](https://arxiv.org/abs/2607.17211) | arXiv 2026
preprint | Codex CLI | Discovery and application of library-adoption refactorings
PrefactoryBench | LLM-synthesized lexical and structural search heuristics, deterministic candidate ranking, targeted refactoring, and differential tests
`deterministic-search` `static-analysis` `test-feedback` `verifier-loop` | File-level detection 75 versus Codex 35; function-level detection 56 versus 32; 40 of 56 candidates yield test-validated refactorings. | model: unknown
budget: unknown
evidence: medium |
-| [QLCoder: A Query Synthesizer For Static Analysis of Security Vulnerabilities](https://proceedings.iclr.cc/paper_files/paper/2026/hash/2adf01ab15adde8820622f7f24bd516b-Abstract-Conference.html) | ICLR 2026
main | Claude Code | CodeQL query synthesis from CVE metadata
176 CVEs across 111 Java projects | CVE-grounded retrieval, AST guidance, CodeQL LSP tools, and iterative execution feedback
`retrieval` `verifier-loop` `static-analysis` | Correct-query rate 53.4% versus 10% for Claude Code-only; F1 0.70 versus 0.048 for IRIS and 0.073 for CodeQL suites. | model: unknown
budget: unknown
evidence: high |
-| [RepoOMP: Repository-Aware Hotspot OpenMP Parallelization via Dependency-Aware Context Reduction](https://arxiv.org/abs/2608.05855)
[artifact](https://github.com/Qlalq/RepoOMP_Simplified) | arXiv 2026
preprint | Claude Code | Repository-aware OpenMP hotspot parallelization
951 hotspots including 330 accepted real-world hotspots | Multi-granularity dependency graph, rule-or-LLM hotspot routing, reduced transformation context, and compile/workload/speedup validation
`repository-graph` `context-reduction` `deterministic-search` `performance-feedback` `verifier-loop` | Across nine detailed kernels, speedup improves 18% to 28% and token cost falls 47% to 68% relative to unstructured Claude Code. | model: unknown
budget: unknown
evidence: medium |
-| [RPG: A Repository Planning Graph for Unified and Scalable Codebase Generation](https://proceedings.iclr.cc/paper_files/paper/2026/file/9482f45fdd89aba9130bb04c44f788a9-Paper-Conference.pdf)
[artifact](https://github.com/microsoft/RPG-ZeroRepo) | ICLR 2026
main | Claude Code (Claude 4 Sonnet)
Codex CLI (o3-pro) | Repository-level codebase generation
RepoCraft | Persistent graph over features, functions, files, interfaces, and data flow with graph-guided generation and validation
`structured-state` `repository-graph` `dependency-aware-planning` `test-feedback` | ZeroRepo coverage 81.5% versus Claude Code 54.2% and Codex 28.4%; pass rate 69.7% versus Claude Code 33.9%. | model: different
budget: unknown
evidence: high |
-| [RustPrint: Documentation-Guided Agentic Codebase Migration from C to Rust](https://arxiv.org/abs/2605.14634) | arXiv 2026
preprint | Claude Code | Whole-codebase migration from C to Rust
Eight real C repositories from 11K to 84K lines of code | Architecture documentation blueprint, module and data-flow planning, compile feedback, documentation mismatch repair, and source-test translation
`structured-state` `dependency-aware-planning` `verifier-loop` `test-feedback` | In the reported Kimi-K2-Instruct setting, feature preservation is 93.26% versus 52.52% for agentic Claude Code; cross-evaluation test pass is 95.17% versus 79.85%. | model: different
budget: unknown
evidence: medium |
-| [Unlocking Model Potentials Through Adaptive Multi-Agent Scaffolding for Efficient Issue Resolution](https://arxiv.org/abs/2606.25514) | arXiv 2026
preprint | Claude Code | Repository issue resolution
SWE-bench Verified and SWE-bench Pro | Event-based agent communication, issue-quality routing, parallel patching and validation, and exploratory fallback
`multi-agent` `dynamic-specialization` `parallelism` `test-feedback` | SWE-bench Verified improves 3.6 to 8.4 percentage points and SWE-bench Pro 6.3 to 18.5 points; average cost is $1.18 lower than multi-agent Claude Code. | model: unknown
budget: unknown
evidence: medium |
-| [WebDesignIter: Co-Evolving Design Knowledge for Repository-Level Front-End Code Generation](https://arxiv.org/abs/2607.10621) | arXiv 2026
preprint | Claude Code
Codex CLI | Repository-level front-end code generation
Web-Bench | Persistent architecture and design knowledge graph, design-informed planning, targeted patches, and sandbox repair
`structured-state` `repository-graph` `retrieval` `test-feedback` | Average Pass@2 improves by 9.55 percentage points over existing baselines; removing design knowledge reduces Pass@1 by 11.4 points. | model: unknown
budget: unknown
evidence: medium |
-| [When Parallelism Pays Off: Cohesion-Aware Task Partitioning for Multi-Agent Coding](https://arxiv.org/abs/2606.00953) | arXiv 2026
preprint | Claude Code (Agent Teams) | Parallel multi-agent coding
28 DevEval and CodeProjectEval tasks | Static dependency graph partitioning, structural hub isolation, community detection, and dependency-aware scheduling
`repository-graph` `dependency-aware-planning` `multi-agent` `parallelism` | Pass rate improves 14.0%, wall-clock speedup reaches 2.10x, and API cost falls 35% relative to Claude Code Agent Teams and simpler schedules. | model: unknown
budget: unknown
evidence: medium |
-| [MLZero: A Multi-Agent System for End-to-end Machine Learning Automation](https://papers.neurips.cc/paper_files/paper/2025/hash/63ed15a46a143ff57484b38cd6b85d91-Abstract-Conference.html)
[artifact](https://github.com/autogluon/autogluon-assistant) | NeurIPS 2025
main | Codex CLI (GPT-4.1 and o4-mini) | End-to-end machine learning automation
MLE-Bench Lite and Multimodal AutoML Agent Benchmark | Hierarchical agents with multimodal perception, semantic and episodic memory, and a planning-coding-evaluation loop
`multi-agent` `memory` `verifier-loop` `test-feedback` | MLZero led MLE-Bench Lite with six gold medals and achieved 0.92 success on its multimodal benchmark; one appendix setting reports 6.5% errors versus 26.9% for Codex CLI. | model: unknown
budget: unknown
evidence: medium |
+| **[Agentic Harness Engineering](papers/agentic-harness-engineering-2026.md)**
[paper](https://arxiv.org/abs/2604.25850) · [artifact](https://github.com/china-qijizhifeng/agentic-harness-engineering) | arXiv 2026
preprint | Codex CLI | Automatic evolution of coding-agent harnesses | Closed-loop harness evolution using component, trajectory, and decision observability with evidence-backed edits and rollback
`harness-evolution` `observability` `memory` `verifier-loop` | Terminal-Bench 2 pass@1 improves from 69.7 to 77.0, above a human-designed Codex-CLI harness at 71.9; transfer uses 12% fewer tokens on SWE-bench Verified. | model: unknown
budget: unknown
evidence: medium |
+| **[AGENTS.md](papers/agents-md-impact-2026.md)**
[paper](https://conf.researchr.org/details/icse-2026/jaws-2026-papers/31/On-the-Impact-of-AGENTS-md-Files-on-the-Efficiency-of-AI-Coding-Agents) | ICSE JAWs 2026
workshop | Claude Code
Codex CLI | Real pull-request tasks with and without repository instructions | Persistent repository-level build, test, style, and directory guidance in AGENTS.md
`repository-instructions` `structured-state` `context-reduction` | Median runtime falls 28.64% and output tokens fall 16.58% with comparable completion behavior. | model: same
budget: same
evidence: high |
+| **[ARTEMIS](papers/artemis-2026.md)**
[paper](https://arxiv.org/abs/2512.09882) · [artifact](https://github.com/Stanford-Trinity/ARTEMIS) | ICLR 2026
main | Claude Code (Claude Sonnet 4)
Codex CLI (GPT-5) | Long-horizon penetration testing in a live enterprise network | Supervisor with dynamic expert agents, recursive task decomposition, parallel exploration, context management, triage, and reporting
`multi-agent` `dynamic-specialization` `parallelism` `memory` `verifier-loop` | ARTEMIS found 9 valid vulnerabilities with an 82% valid-submission rate, ranked second overall, and outperformed 9 of 10 human participants. | model: unknown
budget: different
evidence: medium |
+| **[Co-Coder](papers/co-coder-2026.md)**
[paper](https://arxiv.org/abs/2606.00953) · [artifact](https://github.com/Flitternie/CoCoder) | arXiv 2026
preprint | Claude Code (Agent Teams) | Parallel multi-agent coding | Static dependency graph partitioning, structural hub isolation, community detection, and dependency-aware scheduling
`repository-graph` `dependency-aware-planning` `multi-agent` `parallelism` | Pass rate improves 14.0%, wall-clock speedup reaches 2.10x, and API cost falls 35% relative to Claude Code Agent Teams and simpler schedules. | model: unknown
budget: unknown
evidence: medium |
+| **[FormAct](papers/formact-2026.md)**
[paper](https://openreview.net/forum?id=n7Ta0YEcgw) | ICML 2026
main | Codex CLI | Rich-format document generation and editing | HTML source editor, template retrieval, rendered-page review agent, iterative refinement, and edit-triggered context compression
`retrieval` `context-reduction` `verifier-loop` `visual-review` | Render correctness 4.81 versus 4.39 for multi-pass Codex; human rank-1 rate 0.760 versus 0.140. | model: unknown
budget: unknown
evidence: high |
+| **[icat-agent](papers/icat-agent-2026.md)**
[paper](https://arxiv.org/abs/2606.25514) | arXiv 2026
preprint | Claude Code | Repository issue resolution | Event-based agent communication, issue-quality routing, parallel patching and validation, and exploratory fallback
`multi-agent` `dynamic-specialization` `parallelism` `test-feedback` | SWE-bench Verified improves 3.6 to 8.4 percentage points and SWE-bench Pro 6.3 to 18.5 points; average cost is $1.18 lower than multi-agent Claude Code. | model: unknown
budget: unknown
evidence: medium |
+| **[Prefactory](papers/prefactory-2026.md)**
[paper](https://arxiv.org/abs/2607.17211) | arXiv 2026
preprint | Codex CLI | Discovery and application of library-adoption refactorings | LLM-synthesized lexical and structural search heuristics, deterministic candidate ranking, targeted refactoring, and differential tests
`deterministic-search` `static-analysis` `test-feedback` `verifier-loop` | File-level detection 75 versus Codex 35; function-level detection 56 versus 32; 40 of 56 candidates yield test-validated refactorings. | model: unknown
budget: unknown
evidence: medium |
+| **[QLCoder](papers/qlcoder-2026.md)**
[paper](https://proceedings.iclr.cc/paper_files/paper/2026/hash/2adf01ab15adde8820622f7f24bd516b-Abstract-Conference.html) · [artifact](https://github.com/neuralprogram/qlcoder) | ICLR 2026
main | Claude Code (Claude Sonnet 4, 1.0.120) | CodeQL query synthesis from CVE metadata | CVE-grounded retrieval, AST guidance, CodeQL LSP tools, and iterative execution feedback
`retrieval` `verifier-loop` `static-analysis` | Correct-query rate 53.4% versus 10% for Claude Code-only; F1 0.70 versus 0.048 for IRIS and 0.073 for CodeQL suites. | model: unknown
budget: unknown
evidence: high |
+| **[RepoOMP](papers/repoomp-2026.md)**
[paper](https://arxiv.org/abs/2608.05855) · [artifact](https://github.com/Qlalq/RepoOMP_Simplified) | arXiv 2026
preprint | Claude Code | Repository-aware OpenMP hotspot parallelization | Multi-granularity dependency graph, rule-or-LLM hotspot routing, reduced transformation context, and compile/workload/speedup validation
`repository-graph` `context-reduction` `deterministic-search` `performance-feedback` `verifier-loop` | Across nine detailed kernels, speedup improves 18% to 28% and token cost falls 47% to 68% relative to unstructured Claude Code. | model: unknown
budget: unknown
evidence: medium |
+| **[RustPrint](papers/rustprint-2026.md)**
[paper](https://arxiv.org/abs/2605.14634) | arXiv 2026
preprint | Claude Code | Whole-codebase migration from C to Rust | Architecture documentation blueprint, module and data-flow planning, compile feedback, documentation mismatch repair, and source-test translation
`structured-state` `dependency-aware-planning` `verifier-loop` `test-feedback` | In the reported Kimi-K2-Instruct setting, feature preservation is 93.26% versus 52.52% for agentic Claude Code; cross-evaluation test pass is 95.17% versus 79.85%. | model: different
budget: unknown
evidence: medium |
+| **[WebDesignIter](papers/webdesigniter-2026.md)**
[paper](https://arxiv.org/abs/2607.10621) · [artifact](https://github.com/SYSUSELab/WebDesignIter) | arXiv 2026
preprint | Claude Code
Codex CLI | Repository-level front-end code generation | Persistent architecture and design knowledge graph, design-informed planning, targeted patches, and sandbox repair
`structured-state` `repository-graph` `retrieval` `test-feedback` | Average Pass@2 improves by 9.55 percentage points over existing baselines; removing design knowledge reduces Pass@1 by 11.4 points. | model: unknown
budget: unknown
evidence: medium |
+| **[ZeroRepo](papers/rpg-zerorepo-2026.md)**
[paper](https://proceedings.iclr.cc/paper_files/paper/2026/file/9482f45fdd89aba9130bb04c44f788a9-Paper-Conference.pdf) · [artifact](https://github.com/microsoft/RPG-ZeroRepo) | ICLR 2026
main | Claude Code (Claude 4 Sonnet)
Codex CLI (o3-pro) | Repository-level codebase generation | Persistent graph over features, functions, files, interfaces, and data flow with graph-guided generation and validation
`structured-state` `repository-graph` `dependency-aware-planning` `test-feedback` | ZeroRepo coverage 81.5% versus Claude Code 54.2% and Codex 28.4%; pass rate 69.7% versus Claude Code 33.9%. | model: different
budget: unknown
evidence: high |
+| **[MLZero](papers/mlzero-2025.md)**
[paper](https://papers.neurips.cc/paper_files/paper/2025/hash/63ed15a46a143ff57484b38cd6b85d91-Abstract-Conference.html) · [artifact](https://github.com/autogluon/autogluon-assistant) | NeurIPS 2025
main | Codex CLI (GPT-4.1 and o4-mini) | End-to-end machine learning automation | Hierarchical agents with multimodal perception, semantic and episodic memory, and a planning-coding-evaluation loop
`multi-agent` `memory` `verifier-loop` `test-feedback` | MLZero led MLE-Bench Lite with six gold medals and achieved 0.92 success on its multimodal benchmark; one appendix setting reports 6.5% errors versus 26.9% for Codex CLI. | model: unknown
budget: unknown
evidence: medium |
+## What repeatedly works
+
+Across the current catalog, successful methods repeatedly add structure around the model:
+
+1. **Persistent repository state** — plans, architecture, dependencies, and design knowledge survive beyond one context window.
+2. **Executable feedback** — tests, compilers, static analyzers, renderers, or performance measurements decide what is correct.
+3. **Targeted retrieval** — specialized search and context reduction replace indiscriminate long-context loading.
+4. **Dependency-aware orchestration** — task graphs determine what can run in parallel and what must remain serial.
+5. **Harness observability** — trajectories, components, and decisions become inspectable and reversible.
+6. **Low-cost repository guidance** — focused project instructions reduce wasted actions and tokens.
+
+See the generated [method view](views/by-method.md) and [taxonomy](docs/taxonomy.md).
+
## Related methods
+These papers contain useful product-level evidence, but their central numeric result is a component comparison or another agent's result rather than a clean Claude Code/Codex CLI head-to-head.
+
-| Paper | Venue | Product baseline | Task / benchmark | Method | Reported result | Controls |
+| System / paper | Venue | Product baseline | Task | What changed | Reported evidence | Controls |
|---|---|---|---|---|---|---|
-| [CodeGrep: An RL-Trained Retrieval Agent for LLM Coding Agents](https://arxiv.org/abs/2608.05886) | arXiv 2026
preprint | Claude Code (Claude-Code-like agent) | Repository retrieval for issue resolution
SWE-bench Verified | GRPO-trained 14B retrieval agent that issues multi-turn parallel grep, glob, and read operations for a frozen downstream coding agent
`retrieval` `context-reduction` `parallelism` | Resolve rate 27.0% versus 25.8% without retrieval; resolved tasks use 15% fewer rounds and 19% fewer tokens. | model: same
budget: unknown
evidence: contextual |
-| [SCATE: Learning to Supervise Coding Agents for Cost-Effective Test Generation](https://arxiv.org/abs/2607.08983) | arXiv 2026
preprint | Claude Code | Automated test generation
Repository-level test-generation benchmark | Contextual-bandit supervisor that selects testing actions from current coverage and testability state
`test-feedback` `verifier-loop` | Line coverage improves 32.3% and branch coverage 30.9% over agent-only Gemini CLI; adaptation is also evaluated with Claude Code. | model: unknown
budget: unknown
evidence: contextual |
+| **[CodeGrep](papers/codegrep-2026.md)**
[paper](https://arxiv.org/abs/2608.05886) | arXiv 2026
preprint | Claude Code (Claude-Code-like agent) | Repository retrieval for issue resolution | GRPO-trained 14B retrieval agent that issues multi-turn parallel grep, glob, and read operations for a frozen downstream coding agent
`retrieval` `context-reduction` `parallelism` | Resolve rate 27.0% versus 25.8% without retrieval; resolved tasks use 15% fewer rounds and 19% fewer tokens. | model: same
budget: unknown
evidence: contextual |
+| **[SCATE](papers/scate-2026.md)**
[paper](https://arxiv.org/abs/2607.08983) | arXiv 2026
preprint | Claude Code | Automated test generation | Contextual-bandit supervisor that selects testing actions from current coverage and testability state
`test-feedback` `verifier-loop` | Line coverage improves 32.3% and branch coverage 30.9% over agent-only Gemini CLI; adaptation is also evaluated with Claude Code. | model: unknown
budget: unknown
evidence: contextual |
## Evaluation-only papers
+These papers benchmark or diagnose the products without proposing a method intended to outperform them.
+
-| Paper | Venue | Product baseline | Task / benchmark | Method | Reported result | Controls |
+| System / paper | Venue | Product baseline | Task | What changed | Reported evidence | Controls |
|---|---|---|---|---|---|---|
-| [Engineering Pitfalls in AI Coding Tools: An Empirical Study of Bugs in Claude Code, Codex, and Gemini CLI](https://conf.researchr.org/details/fse-2026/fse-2026-industry-papers/12/Engineering-Pitfalls-in-AI-Coding-Tools-An-Empirical-Study-of-Bugs-in-Claude-Code-C) | FSE Industry Track 2026
conference | Claude Code
Codex CLI | Empirical analysis of product bug reports
More than 3,800 reported bugs | Evaluation-only manual taxonomy of functionality, integration, invocation, and command-execution failures
`observability` | More than 67% of studied bugs concern functionality; API, integration, and configuration account for 36.9% of root causes. | model: unknown
budget: unknown
evidence: high |
-| [Terminal-Bench 2.0: Advancing Agentic Terminal Intelligence through Better Evaluation and Verification](https://proceedings.iclr.cc/paper_files/paper/2026/file/444a3737adaee10d86ad2ef5f74468e6-Paper-Conference.pdf) | ICLR 2026
main | Claude Code (Claude Opus 4.5)
Codex CLI (GPT-5.2) | General terminal-agent evaluation
Terminal-Bench 2.0 | Evaluation-only benchmark with stronger task verification and a neutral terminal scaffold
`verifier-loop` | GPT-5.2 with Codex CLI scores 62.9; Claude Opus 4.5 scores 57.8 with Terminus 2 and 52.1 with Claude Code. | model: unknown
budget: unknown
evidence: high |
-| [BountyBench: Dollar Impact of AI Agent Attackers and Defenders on Real-World Cybersecurity Systems](https://proceedings.neurips.cc/paper_files/paper/2025/hash/faed4276b52ef762879db4142655c699-Abstract-Datasets_and_Benchmarks_Track.html) | NeurIPS 2025
dataset benchmark | Claude Code
Codex CLI (o3-high and o4-mini) | Vulnerability detection, exploitation, and patching
BountyBench | Evaluation-only benchmark of offensive and defensive cybersecurity agents
`verifier-loop` | Codex o3-high detects 12.5% and patches 90%; Claude Code patches 87.5% and exploits 57.5% in the reported settings. | model: unknown
budget: unknown
evidence: high |
+| **[Bug taxonomy](papers/engineering-pitfalls-2026.md)**
[paper](https://conf.researchr.org/details/fse-2026/fse-2026-industry-papers/12/Engineering-Pitfalls-in-AI-Coding-Tools-An-Empirical-Study-of-Bugs-in-Claude-Code-C) | FSE Industry Track 2026
conference | Claude Code
Codex CLI | Empirical analysis of product bug reports | Evaluation-only manual taxonomy of functionality, integration, invocation, and command-execution failures
`observability` | More than 67% of studied bugs concern functionality; API, integration, and configuration account for 36.9% of root causes. | model: unknown
budget: unknown
evidence: high |
+| **[Terminal-Bench 2.0](papers/terminal-bench-2-2026.md)**
[paper](https://proceedings.iclr.cc/paper_files/paper/2026/file/444a3737adaee10d86ad2ef5f74468e6-Paper-Conference.pdf) · [artifact](https://github.com/harbor-framework/terminal-bench-2) | ICLR 2026
main | Claude Code (Claude Opus 4.5)
Codex CLI (GPT-5.2) | General terminal-agent evaluation | Evaluation-only benchmark with stronger task verification and a neutral terminal scaffold
`verifier-loop` | GPT-5.2 with Codex CLI scores 62.9; Claude Opus 4.5 scores 57.8 with Terminus 2 and 52.1 with Claude Code. | model: unknown
budget: unknown
evidence: high |
+| **[BountyBench](papers/bountybench-2025.md)**
[paper](https://proceedings.neurips.cc/paper_files/paper/2025/hash/faed4276b52ef762879db4142655c699-Abstract-Datasets_and_Benchmarks_Track.html) · [artifact](https://github.com/bountybench/bountybench) | NeurIPS 2025
dataset benchmark | Claude Code
Codex CLI (o3-high and o4-mini) | Vulnerability detection, exploitation, and patching | Evaluation-only benchmark of offensive and defensive cybersecurity agents
`verifier-loop` | Codex o3-high detects 12.5% and patches 90%; Claude Code patches 87.5% and exploits 57.5% in the reported settings. | model: unknown
budget: unknown
evidence: high |
## Historical OpenAI Codex model papers
+These entries study the pre-CLI OpenAI Codex model and are never mixed with modern Codex CLI results.
+
-| Paper | Venue | Product baseline | Task / benchmark | Method | Reported result | Controls |
+| System / paper | Venue | Product baseline | Task | What changed | Reported evidence | Controls |
|---|---|---|---|---|---|---|
-| [Interactive Code Generation via Test-Driven User-Intent Formalization](https://arxiv.org/abs/2208.05950)
[artifact](https://github.com/microsoft/TiCoder) | AST at ICSE 2022
workshop | OpenAI Codex model (OpenAI Codex) | Interactive function-level code generation
MBPP and HumanEval | Generate tests to formalize user intent, collect lightweight feedback, and rank or prune candidate programs
`test-feedback` `verifier-loop` | With one to five simulated user queries, Codex pass@1 gains 22.49 to 37.71 points on MBPP and 24.79 to 53.98 points on HumanEval. | model: same
budget: different
evidence: high |
+| **[TiCoder](papers/ticoder-2022.md)**
[paper](https://arxiv.org/abs/2208.05950) · [artifact](https://github.com/microsoft/TiCoder) | AST at ICSE 2022
workshop | OpenAI Codex model (OpenAI Codex) | Interactive function-level code generation | Generate tests to formalize user intent, collect lightweight feedback, and rank or prune candidate programs
`test-feedback` `verifier-loop` | With one to five simulated user queries, Codex pass@1 gains 22.49 to 37.71 points on MBPP and 24.79 to 53.98 points on HumanEval. | model: same
budget: different
evidence: high |
-## Method taxonomy
-
-The catalog currently tracks these recurring ways of improving coding agents:
+## Evidence classes
-- persistent structured state and repository graphs;
-- retrieval and context reduction;
-- executable verification and feedback loops;
-- dependency-aware planning and parallelism;
-- multi-agent coordination and specialization;
-- harness observability and self-improvement;
-- repository-level instructions and configuration;
-- deterministic search or analysis before LLM invocation.
+- **Direct comparison** — numeric comparison with the production product.
+- **Related method** — product-relevant evidence without a clean product-level head-to-head.
+- **Evaluation only** — benchmark, dataset, measurement, or failure analysis.
+- **Historical model** — pre-CLI OpenAI Codex research.
-See [docs/taxonomy.md](docs/taxonomy.md) for definitions.
+Evidence strength and comparison controls are separate fields. A paper can report a strong result while still using a different backbone or budget.
-## Data and validation
+## Reproducible data pipeline
-`data/papers.yaml` is the source of truth. The JSON Schema and project checks validate required metadata, unique identifiers, supported products, evidence classes, and comparison fields.
+`data/papers.yaml` is the source of truth. One command validates the schema, regenerates both READMEs, exports JSON, builds paper dossiers and research views, checks internal links, runs tests, and lints the scripts:
```bash
python -m pip install -r requirements-dev.txt
+make build
make check
```
-To regenerate the tables after editing the catalog:
-
-```bash
-make build
-```
+Generated sections and pages are checked in CI. Hand edits to generated catalog tables fail the build.
## Contributing
-Paper suggestions and corrections are welcome. Read [CONTRIBUTING.md](CONTRIBUTING.md), then open the paper-submission issue template or submit a pull request that updates `data/papers.yaml`.
+The fastest way to help is to [suggest a paper](https://github.com/MicroMilo/awesome-claude-code-codex-papers/issues/new?template=paper.yml) or correct an existing evidence record.
+
+Before submitting a PR, read [CONTRIBUTING.md](CONTRIBUTING.md). Quantitative claims must point to a primary source, and missing model or budget information must stay explicitly `unknown` rather than being guessed.
+
+## Roadmap
-Please link to official paper and artifact pages. Do not upload or redistribute paper PDFs in this repository.
+See [ROADMAP.md](ROADMAP.md). Near-term priorities are deeper baseline-version audits, weekly candidate discovery, richer artifact coverage, and carefully scoped expansion to additional industrial coding agents.
## Related collections
-- [PurCL/ASE](https://github.com/PurCL/ASE) — a broad Agentic Software Engineering literature database.
-- [EuniAI/awesome-code-agents](https://github.com/EuniAI/awesome-code-agents) — a research map of code agents.
-- [Awesome Agent Harnesses](https://github.com/NeuraLiying/Awesome-Agent-Harnesses) — papers and engineering resources about agent harnesses.
+- [PurCL/ASE](https://github.com/PurCL/ASE) — broad Agentic Software Engineering literature database.
+- [EuniAI/awesome-code-agents](https://github.com/EuniAI/awesome-code-agents) — research map of code agents.
+- [Awesome Agent Harnesses](https://github.com/NeuraLiying/Awesome-Agent-Harnesses) — harness research and engineering resources.
-## License
+## Citation and license
-Code and catalog metadata are available under the [MIT License](LICENSE). Copyright remains with the original paper authors and publishers.
+If this catalog supports academic work, use [CITATION.cff](CITATION.cff). Code and catalog metadata are available under the [MIT License](LICENSE); paper copyright remains with the original authors and publishers.
diff --git a/README.zh-CN.md b/README.zh-CN.md
new file mode 100644
index 0000000..8c22ebc
--- /dev/null
+++ b/README.zh-CN.md
@@ -0,0 +1,164 @@
+
+
+
+
+Claude Code 与 Codex 研究论文精选
+
+
+ 专门收录评测、分析或超越真实工业 coding agent 的研究。
+
+
+
+ English · 简体中文
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+这是一个以证据为核心的论文库，研究对象是完整的 **Claude Code** 与 **Codex CLI** 产品，而不只是 Claude 或 GPT 系列模型。每篇论文都记录 baseline 配置、任务、新增方法、论文结果、实验控制、风险说明和官方 artifact。
+
+> [!IMPORTANT]
+> 这不是排行榜。产品版本、底层模型、预算、工具权限和任务领域经常不同。每项直接结果都链接到独立证据页，明确展示这些差异。
+
+## 从这里开始
+
+| 入口 | 适合查看什么 |
+|---|---|
+| [全部论文证据页](papers/README.md) | 作者、产品、模型、版本、证据位置和风险说明 |
+| [按产品浏览](views/by-product.md) | Claude Code 或 Codex CLI 相关论文 |
+| [按方法浏览](views/by-method.md) | 检索、仓库图、验证闭环、多 agent 和 harness 演化 |
+| [实验公平性](views/fair-comparisons.md) | 区分严格控制、模型不同和预算不明的实验 |
+| [按会议浏览](views/by-venue.md) | 主会、workshop、benchmark track 和预印本 |
+| [机器可读数据](data/papers.json) | 二次开发、可视化和下游综述 |
+
+## 这个仓库回答的问题
+
+普通论文列表回答“有哪些论文”，这个仓库重点回答：
+
+> **研究者用 Claude Code 或 Codex CLI 做 baseline 时，增加了什么、提升了什么，以及比较是否公平？**
+
+每个条目都会记录：
+
+- 产品、CLI 版本、底层模型及其在实验中的角色；
+- 任务、benchmark 和实验规模；
+- 围绕产品 baseline 增加或替换的方法；
+- 质量、效率或故障分析结果；
+- 是否保持相同模型和相同预算；
+- 证据位置、限制条件和 artifact 状态。
+
+## 收录范围
+
+第一阶段只做 **Claude Code** 和 **Codex CLI**。数据结构已经与产品解耦，未来可以自然加入 Kimi Code、Zed、Cursor、Gemini CLI 等工业 coding agent。
+
+收录：
+
+- 产品级直接对比；
+- harness、配置、检索、规划和验证方法；
+- 包含产品级结果的 benchmark 与实证研究；
+- 单独隔离的旧 OpenAI Codex 模型论文。
+
+主列表不收录：
+
+- 只使用厂商模型、没有运行对应产品 agent 的论文；
+- 没有研究材料的博客或营销对比；
+- 无法追溯到原论文或官方 artifact 的说法。
+
+## 直接对比
+
+这些论文把提出的方法与 Claude Code 或 Codex CLI 做了数值比较。“直接”不代表模型和预算相同，请同时查看可比性字段和论文证据页。
+
+
+| 系统 / 论文 | 会议 | 产品 baseline | 任务 | 新增方法 | 论文结果 | 可比性 |
+|---|---|---|---|---|---|---|
+| **[Agentic Harness Engineering](papers/agentic-harness-engineering-2026.md)**
[paper](https://arxiv.org/abs/2604.25850) · [artifact](https://github.com/china-qijizhifeng/agentic-harness-engineering) | arXiv 2026
preprint | Codex CLI | Automatic evolution of coding-agent harnesses | Closed-loop harness evolution using component, trajectory, and decision observability with evidence-backed edits and rollback
`harness-evolution` `observability` `memory` `verifier-loop` | Terminal-Bench 2 pass@1 improves from 69.7 to 77.0, above a human-designed Codex-CLI harness at 71.9; transfer uses 12% fewer tokens on SWE-bench Verified. | model: unknown
budget: unknown
evidence: medium |
+| **[AGENTS.md](papers/agents-md-impact-2026.md)**
[paper](https://conf.researchr.org/details/icse-2026/jaws-2026-papers/31/On-the-Impact-of-AGENTS-md-Files-on-the-Efficiency-of-AI-Coding-Agents) | ICSE JAWs 2026
workshop | Claude Code
Codex CLI | Real pull-request tasks with and without repository instructions | Persistent repository-level build, test, style, and directory guidance in AGENTS.md
`repository-instructions` `structured-state` `context-reduction` | Median runtime falls 28.64% and output tokens fall 16.58% with comparable completion behavior. | model: same
budget: same
evidence: high |
+| **[ARTEMIS](papers/artemis-2026.md)**
[paper](https://arxiv.org/abs/2512.09882) · [artifact](https://github.com/Stanford-Trinity/ARTEMIS) | ICLR 2026
main | Claude Code (Claude Sonnet 4)
Codex CLI (GPT-5) | Long-horizon penetration testing in a live enterprise network | Supervisor with dynamic expert agents, recursive task decomposition, parallel exploration, context management, triage, and reporting
`multi-agent` `dynamic-specialization` `parallelism` `memory` `verifier-loop` | ARTEMIS found 9 valid vulnerabilities with an 82% valid-submission rate, ranked second overall, and outperformed 9 of 10 human participants. | model: unknown
budget: different
evidence: medium |
+| **[Co-Coder](papers/co-coder-2026.md)**
[paper](https://arxiv.org/abs/2606.00953) · [artifact](https://github.com/Flitternie/CoCoder) | arXiv 2026
preprint | Claude Code (Agent Teams) | Parallel multi-agent coding | Static dependency graph partitioning, structural hub isolation, community detection, and dependency-aware scheduling
`repository-graph` `dependency-aware-planning` `multi-agent` `parallelism` | Pass rate improves 14.0%, wall-clock speedup reaches 2.10x, and API cost falls 35% relative to Claude Code Agent Teams and simpler schedules. | model: unknown
budget: unknown
evidence: medium |
+| **[FormAct](papers/formact-2026.md)**
[paper](https://openreview.net/forum?id=n7Ta0YEcgw) | ICML 2026
main | Codex CLI | Rich-format document generation and editing | HTML source editor, template retrieval, rendered-page review agent, iterative refinement, and edit-triggered context compression
`retrieval` `context-reduction` `verifier-loop` `visual-review` | Render correctness 4.81 versus 4.39 for multi-pass Codex; human rank-1 rate 0.760 versus 0.140. | model: unknown
budget: unknown
evidence: high |
+| **[icat-agent](papers/icat-agent-2026.md)**
[paper](https://arxiv.org/abs/2606.25514) | arXiv 2026
preprint | Claude Code | Repository issue resolution | Event-based agent communication, issue-quality routing, parallel patching and validation, and exploratory fallback
`multi-agent` `dynamic-specialization` `parallelism` `test-feedback` | SWE-bench Verified improves 3.6 to 8.4 percentage points and SWE-bench Pro 6.3 to 18.5 points; average cost is $1.18 lower than multi-agent Claude Code. | model: unknown
budget: unknown
evidence: medium |
+| **[Prefactory](papers/prefactory-2026.md)**
[paper](https://arxiv.org/abs/2607.17211) | arXiv 2026
preprint | Codex CLI | Discovery and application of library-adoption refactorings | LLM-synthesized lexical and structural search heuristics, deterministic candidate ranking, targeted refactoring, and differential tests
`deterministic-search` `static-analysis` `test-feedback` `verifier-loop` | File-level detection 75 versus Codex 35; function-level detection 56 versus 32; 40 of 56 candidates yield test-validated refactorings. | model: unknown
budget: unknown
evidence: medium |
+| **[QLCoder](papers/qlcoder-2026.md)**
[paper](https://proceedings.iclr.cc/paper_files/paper/2026/hash/2adf01ab15adde8820622f7f24bd516b-Abstract-Conference.html) · [artifact](https://github.com/neuralprogram/qlcoder) | ICLR 2026
main | Claude Code (Claude Sonnet 4, 1.0.120) | CodeQL query synthesis from CVE metadata | CVE-grounded retrieval, AST guidance, CodeQL LSP tools, and iterative execution feedback
`retrieval` `verifier-loop` `static-analysis` | Correct-query rate 53.4% versus 10% for Claude Code-only; F1 0.70 versus 0.048 for IRIS and 0.073 for CodeQL suites. | model: unknown
budget: unknown
evidence: high |
+| **[RepoOMP](papers/repoomp-2026.md)**
[paper](https://arxiv.org/abs/2608.05855) · [artifact](https://github.com/Qlalq/RepoOMP_Simplified) | arXiv 2026
preprint | Claude Code | Repository-aware OpenMP hotspot parallelization | Multi-granularity dependency graph, rule-or-LLM hotspot routing, reduced transformation context, and compile/workload/speedup validation
`repository-graph` `context-reduction` `deterministic-search` `performance-feedback` `verifier-loop` | Across nine detailed kernels, speedup improves 18% to 28% and token cost falls 47% to 68% relative to unstructured Claude Code. | model: unknown
budget: unknown
evidence: medium |
+| **[RustPrint](papers/rustprint-2026.md)**
[paper](https://arxiv.org/abs/2605.14634) | arXiv 2026
preprint | Claude Code | Whole-codebase migration from C to Rust | Architecture documentation blueprint, module and data-flow planning, compile feedback, documentation mismatch repair, and source-test translation
`structured-state` `dependency-aware-planning` `verifier-loop` `test-feedback` | In the reported Kimi-K2-Instruct setting, feature preservation is 93.26% versus 52.52% for agentic Claude Code; cross-evaluation test pass is 95.17% versus 79.85%. | model: different
budget: unknown
evidence: medium |
+| **[WebDesignIter](papers/webdesigniter-2026.md)**
[paper](https://arxiv.org/abs/2607.10621) · [artifact](https://github.com/SYSUSELab/WebDesignIter) | arXiv 2026
preprint | Claude Code
Codex CLI | Repository-level front-end code generation | Persistent architecture and design knowledge graph, design-informed planning, targeted patches, and sandbox repair
`structured-state` `repository-graph` `retrieval` `test-feedback` | Average Pass@2 improves by 9.55 percentage points over existing baselines; removing design knowledge reduces Pass@1 by 11.4 points. | model: unknown
budget: unknown
evidence: medium |
+| **[ZeroRepo](papers/rpg-zerorepo-2026.md)**
[paper](https://proceedings.iclr.cc/paper_files/paper/2026/file/9482f45fdd89aba9130bb04c44f788a9-Paper-Conference.pdf) · [artifact](https://github.com/microsoft/RPG-ZeroRepo) | ICLR 2026
main | Claude Code (Claude 4 Sonnet)
Codex CLI (o3-pro) | Repository-level codebase generation | Persistent graph over features, functions, files, interfaces, and data flow with graph-guided generation and validation
`structured-state` `repository-graph` `dependency-aware-planning` `test-feedback` | ZeroRepo coverage 81.5% versus Claude Code 54.2% and Codex 28.4%; pass rate 69.7% versus Claude Code 33.9%. | model: different
budget: unknown
evidence: high |
+| **[MLZero](papers/mlzero-2025.md)**
[paper](https://papers.neurips.cc/paper_files/paper/2025/hash/63ed15a46a143ff57484b38cd6b85d91-Abstract-Conference.html) · [artifact](https://github.com/autogluon/autogluon-assistant) | NeurIPS 2025
main | Codex CLI (GPT-4.1 and o4-mini) | End-to-end machine learning automation | Hierarchical agents with multimodal perception, semantic and episodic memory, and a planning-coding-evaluation loop
`multi-agent` `memory` `verifier-loop` `test-feedback` | MLZero led MLE-Bench Lite with six gold medals and achieved 0.92 success on its multimodal benchmark; one appendix setting reports 6.5% errors versus 26.9% for Codex CLI. | model: unknown
budget: unknown
evidence: medium |
+
+
+## 反复出现的有效方法
+
+1. **持久化仓库状态**：把计划、架构、依赖和设计知识放到上下文窗口之外。
+2. **可执行反馈**：让测试、编译器、静态分析器、渲染器或性能测量判断正确性。
+3. **定向检索**：使用专门检索和上下文压缩，减少无关信息。
+4. **依赖感知调度**：根据任务图判断哪些步骤可以并行、哪些必须串行。
+5. **Harness 可观测性**：让轨迹、组件和决策可检查、可归因、可回滚。
+6. **低成本仓库指令**：用精炼的项目级说明减少无效操作和 token。
+
+详见[按方法浏览](views/by-method.md)和[方法分类](docs/taxonomy.md)。
+
+## 相关方法
+
+这些论文包含有用的产品证据，但主要数值可能来自组件实验或其他 agent，不构成严格的 Claude Code/Codex CLI 正面对比。
+
+
+| 系统 / 论文 | 会议 | 产品 baseline | 任务 | 新增方法 | 论文结果 | 可比性 |
+|---|---|---|---|---|---|---|
+| **[CodeGrep](papers/codegrep-2026.md)**
[paper](https://arxiv.org/abs/2608.05886) | arXiv 2026
preprint | Claude Code (Claude-Code-like agent) | Repository retrieval for issue resolution | GRPO-trained 14B retrieval agent that issues multi-turn parallel grep, glob, and read operations for a frozen downstream coding agent
`retrieval` `context-reduction` `parallelism` | Resolve rate 27.0% versus 25.8% without retrieval; resolved tasks use 15% fewer rounds and 19% fewer tokens. | model: same
budget: unknown
evidence: contextual |
+| **[SCATE](papers/scate-2026.md)**
[paper](https://arxiv.org/abs/2607.08983) | arXiv 2026
preprint | Claude Code | Automated test generation | Contextual-bandit supervisor that selects testing actions from current coverage and testability state
`test-feedback` `verifier-loop` | Line coverage improves 32.3% and branch coverage 30.9% over agent-only Gemini CLI; adaptation is also evaluated with Claude Code. | model: unknown
budget: unknown
evidence: contextual |
+
+
+## 仅评测论文
+
+这些论文负责 benchmark、测量或故障分析，没有提出专门超越产品的新方法。
+
+
+| 系统 / 论文 | 会议 | 产品 baseline | 任务 | 新增方法 | 论文结果 | 可比性 |
+|---|---|---|---|---|---|---|
+| **[Bug taxonomy](papers/engineering-pitfalls-2026.md)**
[paper](https://conf.researchr.org/details/fse-2026/fse-2026-industry-papers/12/Engineering-Pitfalls-in-AI-Coding-Tools-An-Empirical-Study-of-Bugs-in-Claude-Code-C) | FSE Industry Track 2026
conference | Claude Code
Codex CLI | Empirical analysis of product bug reports | Evaluation-only manual taxonomy of functionality, integration, invocation, and command-execution failures
`observability` | More than 67% of studied bugs concern functionality; API, integration, and configuration account for 36.9% of root causes. | model: unknown
budget: unknown
evidence: high |
+| **[Terminal-Bench 2.0](papers/terminal-bench-2-2026.md)**
[paper](https://proceedings.iclr.cc/paper_files/paper/2026/file/444a3737adaee10d86ad2ef5f74468e6-Paper-Conference.pdf) · [artifact](https://github.com/harbor-framework/terminal-bench-2) | ICLR 2026
main | Claude Code (Claude Opus 4.5)
Codex CLI (GPT-5.2) | General terminal-agent evaluation | Evaluation-only benchmark with stronger task verification and a neutral terminal scaffold
`verifier-loop` | GPT-5.2 with Codex CLI scores 62.9; Claude Opus 4.5 scores 57.8 with Terminus 2 and 52.1 with Claude Code. | model: unknown
budget: unknown
evidence: high |
+| **[BountyBench](papers/bountybench-2025.md)**
[paper](https://proceedings.neurips.cc/paper_files/paper/2025/hash/faed4276b52ef762879db4142655c699-Abstract-Datasets_and_Benchmarks_Track.html) · [artifact](https://github.com/bountybench/bountybench) | NeurIPS 2025
dataset benchmark | Claude Code
Codex CLI (o3-high and o4-mini) | Vulnerability detection, exploitation, and patching | Evaluation-only benchmark of offensive and defensive cybersecurity agents
`verifier-loop` | Codex o3-high detects 12.5% and patches 90%; Claude Code patches 87.5% and exploits 57.5% in the reported settings. | model: unknown
budget: unknown
evidence: high |
+
+
+## 旧 OpenAI Codex 模型论文
+
+这些论文研究的是 Codex CLI 出现前的 OpenAI Codex 模型，不与现代 Codex CLI 结果混合。
+
+
+| 系统 / 论文 | 会议 | 产品 baseline | 任务 | 新增方法 | 论文结果 | 可比性 |
+|---|---|---|---|---|---|---|
+| **[TiCoder](papers/ticoder-2022.md)**
[paper](https://arxiv.org/abs/2208.05950) · [artifact](https://github.com/microsoft/TiCoder) | AST at ICSE 2022
workshop | OpenAI Codex model (OpenAI Codex) | Interactive function-level code generation | Generate tests to formalize user intent, collect lightweight feedback, and rank or prune candidate programs
`test-feedback` `verifier-loop` | With one to five simulated user queries, Codex pass@1 gains 22.49 to 37.71 points on MBPP and 24.79 to 53.98 points on HumanEval. | model: same
budget: different
evidence: high |
+
+
+## 数据与自动校验
+
+`data/papers.yaml` 是唯一数据源。构建流程会校验 Schema、生成中英文 README、导出 JSON、生成每篇论文的证据页和研究视图，并检查内部链接、执行测试与 lint：
+
+```bash
+python -m pip install -r requirements-dev.txt
+make build
+make check
+```
+
+## 参与贡献
+
+可以直接[推荐论文](https://github.com/MicroMilo/awesome-claude-code-codex-papers/issues/new?template=paper.yml)，也可以修正现有证据记录。提交前请阅读 [CONTRIBUTING.md](CONTRIBUTING.md)。定量结果必须指向一手来源；论文没写清模型或预算时，应保留为 `unknown`，不能自行猜测。
+
+## 路线图、引用与许可
+
+- 后续计划见 [ROADMAP.md](ROADMAP.md)。
+- 学术引用信息见 [CITATION.cff](CITATION.cff)。
+- 代码和目录元数据使用 [MIT License](LICENSE)，论文版权属于原作者与出版方。
diff --git a/ROADMAP.md b/ROADMAP.md
new file mode 100644
index 0000000..9910b83
--- /dev/null
+++ b/ROADMAP.md
@@ -0,0 +1,51 @@
+# Roadmap
+
+The catalog optimizes for evidence quality before paper count.
+
+## v0.1 — Claude Code and Codex foundation
+
+- [x] Strict product-versus-model distinction
+- [x] Machine-readable YAML and JSON catalog
+- [x] Direct, related, evaluation-only, and historical evidence classes
+- [x] Same-model and same-budget controls
+- [x] Per-paper evidence dossiers
+- [x] Product, method, venue, and fairness views
+- [x] English and Simplified Chinese entry points
+- [x] Schema, tests, linting, and CI
+- [x] Structured paper-suggestion workflow
+
+## v0.2 — Evidence depth
+
+- [ ] Audit every paper for exact CLI version, model snapshot, reasoning mode, and execution date
+- [ ] Record wall-clock, turn, token, concurrency, and monetary budgets when reported
+- [ ] Add table, figure, or section anchors for every quantitative claim
+- [ ] Link official evaluation traces and datasets when available
+- [ ] Add reproducibility grades separate from evidence strength
+- [ ] Add a non-blocking link-health monitor
+
+## v0.3 — Discovery and maintenance
+
+- [x] Weekly arXiv candidate discovery for exact product mentions
+- [ ] Venue-proceedings ingestion for ICSE, FSE, ASE, ISSTA, ICLR, ICML, NeurIPS, AAAI, and security venues
+- [ ] Duplicate detection across arXiv, OpenReview, DOI, and final proceedings versions
+- [ ] Release notes for every catalog update
+
+## v0.4 — Additional industrial products
+
+Expansion requires enough product-level research to justify a maintained category. Candidate products include:
+
+- Kimi Code
+- Zed
+- Cursor
+- Gemini CLI
+- GitHub Copilot CLI
+- OpenCode
+
+Model-only papers remain out of scope unless they study a product harness directly.
+
+## Long-term
+
+- Interactive static site generated from the same catalog
+- Comparison matrices with matched-model and matched-budget filters
+- Reproduction packs for a small set of high-impact claims
+- Community-maintained venue watchlist and artifact-status audits
diff --git a/SECURITY.md b/SECURITY.md
new file mode 100644
index 0000000..adba06f
--- /dev/null
+++ b/SECURITY.md
@@ -0,0 +1,20 @@
+# Security Policy
+
+This repository contains public research metadata and maintenance scripts; it does not execute the coding agents described by the papers.
+
+## Reporting a vulnerability
+
+Please use GitHub's private vulnerability reporting flow in the repository Security tab. Do not open a public issue for a vulnerability that could affect contributors or downstream users.
+
+Include:
+
+- the affected file and revision;
+- a minimal reproduction;
+- the expected security impact;
+- any suggested mitigation.
+
+Catalog corrections, broken links, and disputed research claims are not security vulnerabilities. Use the issue templates for those reports.
+
+## Supported version
+
+Security fixes apply to the current `main` branch and the latest release.
diff --git a/assets/hero.png b/assets/hero.png
new file mode 100644
index 0000000..1f72519
Binary files /dev/null and b/assets/hero.png differ
diff --git a/data/papers.json b/data/papers.json
new file mode 100644
index 0000000..534b870
--- /dev/null
+++ b/data/papers.json
@@ -0,0 +1,1164 @@
+{
+ "catalog_version": 1,
+ "reviewed_at": "2026-08-21",
+ "papers": [
+ {
+ "id": "qlcoder-2026",
+ "title": "QLCoder: A Query Synthesizer For Static Analysis of Security Vulnerabilities",
+ "authors": [
+ "Claire Wang",
+ "Ziyang Li",
+ "Saikat Dutta",
+ "Mayur Naik"
+ ],
+ "year": 2026,
+ "venue": "ICLR",
+ "publication_status": "main",
+ "classification": "direct",
+ "system": "QLCoder",
+ "paper_url": "https://proceedings.iclr.cc/paper_files/paper/2026/hash/2adf01ab15adde8820622f7f24bd516b-Abstract-Conference.html",
+ "artifact_url": "https://github.com/neuralprogram/qlcoder",
+ "artifact_status": "official",
+ "arxiv_id": "2511.08462",
+ "published_at": "2025-11-11",
+ "products": [
+ {
+ "product": "claude-code",
+ "role": "baseline",
+ "model": "Claude Sonnet 4",
+ "version": "1.0.120"
+ }
+ ],
+ "task": {
+ "summary": "CodeQL query synthesis from CVE metadata",
+ "benchmark": "176 CVEs across 111 Java projects"
+ },
+ "method": {
+ "summary": "CVE-grounded retrieval, AST guidance, CodeQL LSP tools, and iterative execution feedback",
+ "tags": [
+ "retrieval",
+ "verifier-loop",
+ "static-analysis"
+ ]
+ },
+ "evidence": {
+ "result": "Correct-query rate 53.4% versus 10% for Claude Code-only; F1 0.70 versus 0.048 for IRIS and 0.073 for CodeQL suites.",
+ "same_model": "unknown",
+ "same_budget": "unknown",
+ "strength": "high",
+ "claim_type": "quality",
+ "comparison_scope": "product-level",
+ "source_location": "Official abstract, main evaluation, and artifact paper environment",
+ "caveats": "Security-query synthesis is narrower than general software engineering."
+ }
+ },
+ {
+ "id": "rpg-zerorepo-2026",
+ "title": "RPG: A Repository Planning Graph for Unified and Scalable Codebase Generation",
+ "authors": [
+ "Jane Luo",
+ "Xin Zhang",
+ "Steven Liu",
+ "Jie Wu",
+ "Jianfeng Liu",
+ "Yiming Huang",
+ "Yangyu Huang",
+ "Chengyu Yin",
+ "Ying Xin",
+ "Yuefeng Zhan",
+ "Hao Sun",
+ "Qi Chen",
+ "Scarlett Li",
+ "Mao Yang"
+ ],
+ "year": 2026,
+ "venue": "ICLR",
+ "publication_status": "main",
+ "classification": "direct",
+ "system": "ZeroRepo",
+ "paper_url": "https://proceedings.iclr.cc/paper_files/paper/2026/file/9482f45fdd89aba9130bb04c44f788a9-Paper-Conference.pdf",
+ "artifact_url": "https://github.com/microsoft/RPG-ZeroRepo",
+ "artifact_status": "official",
+ "arxiv_id": "2509.16198",
+ "published_at": "2025-09-19",
+ "products": [
+ {
+ "product": "claude-code",
+ "role": "baseline",
+ "model": "Claude 4 Sonnet",
+ "version": "not-reported"
+ },
+ {
+ "product": "codex-cli",
+ "role": "baseline",
+ "model": "o3-pro",
+ "version": "not-reported"
+ }
+ ],
+ "task": {
+ "summary": "Repository-level codebase generation",
+ "benchmark": "RepoCraft"
+ },
+ "method": {
+ "summary": "Persistent graph over features, functions, files, interfaces, and data flow with graph-guided generation and validation",
+ "tags": [
+ "structured-state",
+ "repository-graph",
+ "dependency-aware-planning",
+ "test-feedback"
+ ]
+ },
+ "evidence": {
+ "result": "ZeroRepo coverage 81.5% versus Claude Code 54.2% and Codex 28.4%; pass rate 69.7% versus Claude Code 33.9%.",
+ "same_model": "no",
+ "same_budget": "unknown",
+ "strength": "high",
+ "claim_type": "quality",
+ "comparison_scope": "product-level",
+ "source_location": "Section 4.3 and RepoCraft main-results table",
+ "caveats": "ZeroRepo uses o3-mini while the Claude Code baseline uses Claude 4 Sonnet, so model and harness effects are mixed."
+ }
+ },
+ {
+ "id": "artemis-2026",
+ "title": "Comparing AI Agents to Cybersecurity Professionals in Real-World Penetration Testing",
+ "authors": [
+ "Justin W. Lin",
+ "Eliot Krzysztof Jones",
+ "Donovan Julian Jasper",
+ "Ethan Jun-shen Ho",
+ "Anna Wu",
+ "Arnold Tianyi Yang",
+ "Neil Perry",
+ "Andy Zou",
+ "Matt Fredrikson",
+ "J. Zico Kolter",
+ "Percy Liang",
+ "Dan Boneh",
+ "Daniel E. Ho"
+ ],
+ "year": 2026,
+ "venue": "ICLR",
+ "publication_status": "main",
+ "classification": "direct",
+ "system": "ARTEMIS",
+ "paper_url": "https://arxiv.org/abs/2512.09882",
+ "artifact_url": "https://github.com/Stanford-Trinity/ARTEMIS",
+ "artifact_status": "official",
+ "arxiv_id": "2512.09882",
+ "published_at": "2025-12-10",
+ "products": [
+ {
+ "product": "claude-code",
+ "role": "baseline",
+ "model": "Claude Sonnet 4",
+ "version": "not-reported"
+ },
+ {
+ "product": "codex-cli",
+ "role": "baseline",
+ "model": "GPT-5",
+ "version": "not-reported"
+ }
+ ],
+ "task": {
+ "summary": "Long-horizon penetration testing in a live enterprise network",
+ "benchmark": "Approximately 8,000 hosts across 12 subnets"
+ },
+ "method": {
+ "summary": "Supervisor with dynamic expert agents, recursive task decomposition, parallel exploration, context management, triage, and reporting",
+ "tags": [
+ "multi-agent",
+ "dynamic-specialization",
+ "parallelism",
+ "memory",
+ "verifier-loop"
+ ]
+ },
+ "evidence": {
+ "result": "ARTEMIS found 9 valid vulnerabilities with an 82% valid-submission rate, ranked second overall, and outperformed 9 of 10 human participants.",
+ "same_model": "unknown",
+ "same_budget": "no",
+ "strength": "medium",
+ "claim_type": "mixed",
+ "comparison_scope": "product-level",
+ "source_location": "Abstract and main penetration-testing results",
+ "caveats": "Runtime limits, model combinations, safety refusals, and product policies differ across systems."
+ }
+ },
+ {
+ "id": "mlzero-2025",
+ "title": "MLZero: A Multi-Agent System for End-to-end Machine Learning Automation",
+ "authors": [
+ "Haoyang Fang",
+ "Boran Han",
+ "Nick Erickson",
+ "Xiyuan Zhang",
+ "Su Zhou",
+ "Anirudh Dagar",
+ "Jiani Zhang",
+ "Ali Caner Turkmen",
+ "Cuixiong Hu",
+ "Huzefa Rangwala",
+ "Ying Nian Wu",
+ "Bernie Wang",
+ "George Karypis"
+ ],
+ "year": 2025,
+ "venue": "NeurIPS",
+ "publication_status": "main",
+ "classification": "direct",
+ "system": "MLZero",
+ "paper_url": "https://papers.neurips.cc/paper_files/paper/2025/hash/63ed15a46a143ff57484b38cd6b85d91-Abstract-Conference.html",
+ "artifact_url": "https://github.com/autogluon/autogluon-assistant",
+ "artifact_status": "official",
+ "arxiv_id": "2505.13941",
+ "published_at": "2025-05-20",
+ "products": [
+ {
+ "product": "codex-cli",
+ "role": "baseline",
+ "model": "GPT-4.1 and o4-mini",
+ "version": "not-reported"
+ }
+ ],
+ "task": {
+ "summary": "End-to-end machine learning automation",
+ "benchmark": "MLE-Bench Lite and Multimodal AutoML Agent Benchmark"
+ },
+ "method": {
+ "summary": "Hierarchical agents with multimodal perception, semantic and episodic memory, and a planning-coding-evaluation loop",
+ "tags": [
+ "multi-agent",
+ "memory",
+ "verifier-loop",
+ "test-feedback"
+ ]
+ },
+ "evidence": {
+ "result": "MLZero led MLE-Bench Lite with six gold medals and achieved 0.92 success on its multimodal benchmark; one appendix setting reports 6.5% errors versus 26.9% for Codex CLI.",
+ "same_model": "unknown",
+ "same_budget": "unknown",
+ "strength": "medium",
+ "claim_type": "quality",
+ "comparison_scope": "product-level",
+ "source_location": "Abstract, main benchmark results, and appendix error analysis",
+ "caveats": "Results concern machine-learning automation and should not be generalized to all software-engineering tasks."
+ }
+ },
+ {
+ "id": "formact-2026",
+ "title": "FormAct: Agentic Source Editing for Rich-Format Document Generation",
+ "authors": [
+ "Eugene J. Yu",
+ "Xingxing Zhang",
+ "Yuan Xia",
+ "Tao Ge",
+ "Xun Wang",
+ "FNU Kartik",
+ "Vishwas Suryanarayanan",
+ "Cheng Yang",
+ "Amanda Jiang",
+ "Jiayu Ding",
+ "Xiangyu Wong",
+ "Tengchao Lv",
+ "Lei Cui",
+ "Si-Qing Chen",
+ "Furu Wei",
+ "Sujian Li"
+ ],
+ "year": 2026,
+ "venue": "ICML",
+ "publication_status": "main",
+ "classification": "direct",
+ "system": "FormAct",
+ "paper_url": "https://openreview.net/forum?id=n7Ta0YEcgw",
+ "artifact_status": "not-found",
+ "products": [
+ {
+ "product": "codex-cli",
+ "role": "baseline",
+ "model": "not-reported",
+ "version": "not-reported"
+ }
+ ],
+ "task": {
+ "summary": "Rich-format document generation and editing",
+ "benchmark": "RichDocBench and RichDocFuzz"
+ },
+ "method": {
+ "summary": "HTML source editor, template retrieval, rendered-page review agent, iterative refinement, and edit-triggered context compression",
+ "tags": [
+ "retrieval",
+ "context-reduction",
+ "verifier-loop",
+ "visual-review"
+ ]
+ },
+ "evidence": {
+ "result": "Render correctness 4.81 versus 4.39 for multi-pass Codex; human rank-1 rate 0.760 versus 0.140.",
+ "same_model": "unknown",
+ "same_budget": "unknown",
+ "strength": "high",
+ "claim_type": "mixed",
+ "comparison_scope": "product-level",
+ "source_location": "Main comparison table and review-refinement ablation",
+ "caveats": "Codex has slightly higher content alignment, and the task is document formatting rather than source-code repair."
+ }
+ },
+ {
+ "id": "agentic-harness-engineering-2026",
+ "title": "Agentic Harness Engineering: Observability-Driven Automatic Evolution of Coding-Agent Harnesses",
+ "authors": [
+ "Jiahang Lin",
+ "Shichun Liu",
+ "Chengjun Pan",
+ "Lizhi Lin",
+ "Shihan Dou",
+ "Zhiheng Xi",
+ "Xuanjing Huang",
+ "Hang Yan",
+ "Zhenhua Han",
+ "Tao Gui",
+ "Yu-Gang Jiang"
+ ],
+ "year": 2026,
+ "venue": "arXiv",
+ "publication_status": "preprint",
+ "classification": "direct",
+ "system": "Agentic Harness Engineering",
+ "paper_url": "https://arxiv.org/abs/2604.25850",
+ "artifact_url": "https://github.com/china-qijizhifeng/agentic-harness-engineering",
+ "artifact_status": "official",
+ "arxiv_id": "2604.25850",
+ "published_at": "2026-04-28",
+ "products": [
+ {
+ "product": "codex-cli",
+ "role": "baseline",
+ "model": "not-reported",
+ "version": "not-reported"
+ }
+ ],
+ "task": {
+ "summary": "Automatic evolution of coding-agent harnesses",
+ "benchmark": "Terminal-Bench 2 and SWE-bench Verified"
+ },
+ "method": {
+ "summary": "Closed-loop harness evolution using component, trajectory, and decision observability with evidence-backed edits and rollback",
+ "tags": [
+ "harness-evolution",
+ "observability",
+ "memory",
+ "verifier-loop"
+ ]
+ },
+ "evidence": {
+ "result": "Terminal-Bench 2 pass@1 improves from 69.7 to 77.0, above a human-designed Codex-CLI harness at 71.9; transfer uses 12% fewer tokens on SWE-bench Verified.",
+ "same_model": "unknown",
+ "same_budget": "unknown",
+ "strength": "medium",
+ "claim_type": "mixed",
+ "comparison_scope": "product-level",
+ "source_location": "Abstract and Terminal-Bench 2 transfer and ablation results",
+ "caveats": "Preprint; model, harness, and evaluation-budget effects require careful separation."
+ }
+ },
+ {
+ "id": "webdesigniter-2026",
+ "title": "WebDesignIter: Co-Evolving Design Knowledge for Repository-Level Front-End Code Generation",
+ "authors": [
+ "Zheng Pei",
+ "Mingwei Liu",
+ "Zhenxi Chen",
+ "Zihao Wang",
+ "Yanlin Wang"
+ ],
+ "year": 2026,
+ "venue": "arXiv",
+ "publication_status": "preprint",
+ "classification": "direct",
+ "system": "WebDesignIter",
+ "paper_url": "https://arxiv.org/abs/2607.10621",
+ "artifact_url": "https://github.com/SYSUSELab/WebDesignIter",
+ "artifact_status": "official",
+ "arxiv_id": "2607.10621",
+ "published_at": "2026-07-12",
+ "products": [
+ {
+ "product": "claude-code",
+ "role": "baseline",
+ "model": "not-reported",
+ "version": "not-reported"
+ },
+ {
+ "product": "codex-cli",
+ "role": "baseline",
+ "model": "not-reported",
+ "version": "not-reported"
+ }
+ ],
+ "task": {
+ "summary": "Repository-level front-end code generation",
+ "benchmark": "Web-Bench"
+ },
+ "method": {
+ "summary": "Persistent architecture and design knowledge graph, design-informed planning, targeted patches, and sandbox repair",
+ "tags": [
+ "structured-state",
+ "repository-graph",
+ "retrieval",
+ "test-feedback"
+ ]
+ },
+ "evidence": {
+ "result": "Average Pass@2 improves by 9.55 percentage points over existing baselines; removing design knowledge reduces Pass@1 by 11.4 points.",
+ "same_model": "unknown",
+ "same_budget": "unknown",
+ "strength": "medium",
+ "claim_type": "quality",
+ "comparison_scope": "product-level",
+ "source_location": "Abstract, main Web-Bench comparison, and design-knowledge ablation",
+ "caveats": "Front-end-specific benchmark and preprint status."
+ }
+ },
+ {
+ "id": "prefactory-2026",
+ "title": "Prefactory: Automated Discovery and Application of Library-Adoption Refactorings",
+ "authors": [
+ "Islem Bouzenia",
+ "Michael Pradel"
+ ],
+ "year": 2026,
+ "venue": "arXiv",
+ "publication_status": "preprint",
+ "classification": "direct",
+ "system": "Prefactory",
+ "paper_url": "https://arxiv.org/abs/2607.17211",
+ "artifact_status": "not-found",
+ "arxiv_id": "2607.17211",
+ "published_at": "2026-07-19",
+ "products": [
+ {
+ "product": "codex-cli",
+ "role": "baseline",
+ "model": "not-reported",
+ "version": "not-reported"
+ }
+ ],
+ "task": {
+ "summary": "Discovery and application of library-adoption refactorings",
+ "benchmark": "PrefactoryBench"
+ },
+ "method": {
+ "summary": "LLM-synthesized lexical and structural search heuristics, deterministic candidate ranking, targeted refactoring, and differential tests",
+ "tags": [
+ "deterministic-search",
+ "static-analysis",
+ "test-feedback",
+ "verifier-loop"
+ ]
+ },
+ "evidence": {
+ "result": "File-level detection 75 versus Codex 35; function-level detection 56 versus 32; 40 of 56 candidates yield test-validated refactorings.",
+ "same_model": "unknown",
+ "same_budget": "unknown",
+ "strength": "medium",
+ "claim_type": "quality",
+ "comparison_scope": "product-level",
+ "source_location": "Abstract and PrefactoryBench detection and validation results",
+ "caveats": "The strongest comparison concerns candidate detection, not universal end-to-end task success."
+ }
+ },
+ {
+ "id": "rustprint-2026",
+ "title": "RustPrint: Documentation-Guided Agentic Codebase Migration from C to Rust",
+ "authors": [
+ "Minh Le-Anh",
+ "Anh Nguyen Hoang",
+ "Bach Le",
+ "Nghi D. Q. Bui"
+ ],
+ "year": 2026,
+ "venue": "arXiv",
+ "publication_status": "preprint",
+ "classification": "direct",
+ "system": "RustPrint",
+ "paper_url": "https://arxiv.org/abs/2605.14634",
+ "artifact_status": "not-found",
+ "arxiv_id": "2605.14634",
+ "published_at": "2026-05-14",
+ "products": [
+ {
+ "product": "claude-code",
+ "role": "baseline",
+ "model": "not-reported",
+ "version": "not-reported"
+ }
+ ],
+ "task": {
+ "summary": "Whole-codebase migration from C to Rust",
+ "benchmark": "Eight real C repositories from 11K to 84K lines of code"
+ },
+ "method": {
+ "summary": "Architecture documentation blueprint, module and data-flow planning, compile feedback, documentation mismatch repair, and source-test translation",
+ "tags": [
+ "structured-state",
+ "dependency-aware-planning",
+ "verifier-loop",
+ "test-feedback"
+ ]
+ },
+ "evidence": {
+ "result": "In the reported Kimi-K2-Instruct setting, feature preservation is 93.26% versus 52.52% for agentic Claude Code; cross-evaluation test pass is 95.17% versus 79.85%.",
+ "same_model": "no",
+ "same_budget": "unknown",
+ "strength": "medium",
+ "claim_type": "quality",
+ "comparison_scope": "product-level",
+ "source_location": "Abstract and feature-preservation and cross-evaluation results",
+ "caveats": "The comparison mixes model and scaffold choices and is specific to C-to-Rust migration."
+ }
+ },
+ {
+ "id": "repoomp-2026",
+ "title": "RepoOMP: Repository-Aware Hotspot OpenMP Parallelization via Dependency-Aware Context Reduction",
+ "authors": [
+ "Yongjie Qian",
+ "Ke Gao",
+ "Zhibin Zhang",
+ "Shaohui Peng",
+ "Ling Li"
+ ],
+ "year": 2026,
+ "venue": "arXiv",
+ "publication_status": "preprint",
+ "classification": "direct",
+ "system": "RepoOMP",
+ "paper_url": "https://arxiv.org/abs/2608.05855",
+ "artifact_url": "https://github.com/Qlalq/RepoOMP_Simplified",
+ "artifact_status": "official",
+ "arxiv_id": "2608.05855",
+ "published_at": "2026-08-06",
+ "products": [
+ {
+ "product": "claude-code",
+ "role": "baseline",
+ "model": "not-reported",
+ "version": "not-reported"
+ }
+ ],
+ "task": {
+ "summary": "Repository-aware OpenMP hotspot parallelization",
+ "benchmark": "951 hotspots including 330 accepted real-world hotspots"
+ },
+ "method": {
+ "summary": "Multi-granularity dependency graph, rule-or-LLM hotspot routing, reduced transformation context, and compile/workload/speedup validation",
+ "tags": [
+ "repository-graph",
+ "context-reduction",
+ "deterministic-search",
+ "performance-feedback",
+ "verifier-loop"
+ ]
+ },
+ "evidence": {
+ "result": "Across nine detailed kernels, speedup improves 18% to 28% and token cost falls 47% to 68% relative to unstructured Claude Code.",
+ "same_model": "unknown",
+ "same_budget": "unknown",
+ "strength": "medium",
+ "claim_type": "mixed",
+ "comparison_scope": "product-level",
+ "source_location": "Abstract and nine-kernel cross-backbone comparison",
+ "caveats": "Domain-specific high-performance-computing task and preprint status."
+ }
+ },
+ {
+ "id": "icat-agent-2026",
+ "title": "Unlocking Model Potentials Through Adaptive Multi-Agent Scaffolding for Efficient Issue Resolution",
+ "authors": [
+ "Yang Chen",
+ "Aliya Ahmad",
+ "Yiheng Zhou",
+ "Reyhaneh Jabbarvand"
+ ],
+ "year": 2026,
+ "venue": "arXiv",
+ "publication_status": "preprint",
+ "classification": "direct",
+ "system": "icat-agent",
+ "paper_url": "https://arxiv.org/abs/2606.25514",
+ "artifact_status": "not-found",
+ "arxiv_id": "2606.25514",
+ "published_at": "2026-06-24",
+ "products": [
+ {
+ "product": "claude-code",
+ "role": "baseline",
+ "model": "not-reported",
+ "version": "not-reported"
+ }
+ ],
+ "task": {
+ "summary": "Repository issue resolution",
+ "benchmark": "SWE-bench Verified and SWE-bench Pro"
+ },
+ "method": {
+ "summary": "Event-based agent communication, issue-quality routing, parallel patching and validation, and exploratory fallback",
+ "tags": [
+ "multi-agent",
+ "dynamic-specialization",
+ "parallelism",
+ "test-feedback"
+ ]
+ },
+ "evidence": {
+ "result": "SWE-bench Verified improves 3.6 to 8.4 percentage points and SWE-bench Pro 6.3 to 18.5 points; average cost is $1.18 lower than multi-agent Claude Code.",
+ "same_model": "unknown",
+ "same_budget": "unknown",
+ "strength": "medium",
+ "claim_type": "mixed",
+ "comparison_scope": "product-level",
+ "source_location": "Abstract and SWE-bench Verified and Pro comparisons",
+ "caveats": "Exact model parity varies by reported configuration; preprint."
+ }
+ },
+ {
+ "id": "co-coder-2026",
+ "title": "When Parallelism Pays Off: Cohesion-Aware Task Partitioning for Multi-Agent Coding",
+ "authors": [
+ "Xu Yang",
+ "Lunyiu Nie",
+ "Ethan Chandra",
+ "Stanislav Gannutin",
+ "Fangru Lin",
+ "Swarat Chaudhuri"
+ ],
+ "year": 2026,
+ "venue": "arXiv",
+ "publication_status": "preprint",
+ "classification": "direct",
+ "system": "Co-Coder",
+ "paper_url": "https://arxiv.org/abs/2606.00953",
+ "artifact_url": "https://github.com/Flitternie/CoCoder",
+ "artifact_status": "official",
+ "arxiv_id": "2606.00953",
+ "published_at": "2026-05-31",
+ "products": [
+ {
+ "product": "claude-code",
+ "role": "baseline",
+ "model": "not-reported",
+ "version": "Agent Teams"
+ }
+ ],
+ "task": {
+ "summary": "Parallel multi-agent coding",
+ "benchmark": "28 DevEval and CodeProjectEval tasks"
+ },
+ "method": {
+ "summary": "Static dependency graph partitioning, structural hub isolation, community detection, and dependency-aware scheduling",
+ "tags": [
+ "repository-graph",
+ "dependency-aware-planning",
+ "multi-agent",
+ "parallelism"
+ ]
+ },
+ "evidence": {
+ "result": "Pass rate improves 14.0%, wall-clock speedup reaches 2.10x, and API cost falls 35% relative to Claude Code Agent Teams and simpler schedules.",
+ "same_model": "unknown",
+ "same_budget": "unknown",
+ "strength": "medium",
+ "claim_type": "mixed",
+ "comparison_scope": "product-level",
+ "source_location": "Abstract and DevEval and CodeProjectEval comparisons",
+ "caveats": "Small 28-task evaluation and preprint status."
+ }
+ },
+ {
+ "id": "agents-md-impact-2026",
+ "title": "On the Impact of AGENTS.md Files on the Efficiency of AI Coding Agents",
+ "authors": [
+ "Jai Lal Lulla",
+ "Seyedmoein Mohsenimofidi",
+ "Matthias Galster",
+ "Jie M. Zhang",
+ "Sebastian Baltes",
+ "Christoph Treude"
+ ],
+ "year": 2026,
+ "venue": "ICSE JAWs",
+ "publication_status": "workshop",
+ "classification": "direct",
+ "system": "AGENTS.md",
+ "paper_url": "https://conf.researchr.org/details/icse-2026/jaws-2026-papers/31/On-the-Impact-of-AGENTS-md-Files-on-the-Efficiency-of-AI-Coding-Agents",
+ "artifact_status": "not-found",
+ "arxiv_id": "2601.20404",
+ "published_at": "2026-01-28",
+ "products": [
+ {
+ "product": "claude-code",
+ "role": "baseline",
+ "model": "not-reported",
+ "version": "not-reported"
+ },
+ {
+ "product": "codex-cli",
+ "role": "baseline",
+ "model": "not-reported",
+ "version": "not-reported"
+ }
+ ],
+ "task": {
+ "summary": "Real pull-request tasks with and without repository instructions",
+ "benchmark": "10 repositories and 124 pull requests"
+ },
+ "method": {
+ "summary": "Persistent repository-level build, test, style, and directory guidance in AGENTS.md",
+ "tags": [
+ "repository-instructions",
+ "structured-state",
+ "context-reduction"
+ ]
+ },
+ "evidence": {
+ "result": "Median runtime falls 28.64% and output tokens fall 16.58% with comparable completion behavior.",
+ "same_model": "yes",
+ "same_budget": "yes",
+ "strength": "high",
+ "claim_type": "efficiency",
+ "comparison_scope": "configuration-ablation",
+ "source_location": "Abstract and with-versus-without AGENTS.md results",
+ "caveats": "Workshop study and configuration intervention rather than a standalone agent."
+ }
+ },
+ {
+ "id": "codegrep-2026",
+ "title": "CodeGrep: An RL-Trained Retrieval Agent for LLM Coding Agents",
+ "authors": [
+ "Wuya Chen",
+ "Yihao Yang",
+ "Yang Cao",
+ "Yue Lin"
+ ],
+ "year": 2026,
+ "venue": "arXiv",
+ "publication_status": "preprint",
+ "classification": "related",
+ "system": "CodeGrep",
+ "paper_url": "https://arxiv.org/abs/2608.05886",
+ "artifact_status": "not-found",
+ "arxiv_id": "2608.05886",
+ "published_at": "2026-08-06",
+ "products": [
+ {
+ "product": "claude-code",
+ "role": "evaluated",
+ "model": "not-reported",
+ "version": "Claude-Code-like agent"
+ }
+ ],
+ "task": {
+ "summary": "Repository retrieval for issue resolution",
+ "benchmark": "SWE-bench Verified"
+ },
+ "method": {
+ "summary": "GRPO-trained 14B retrieval agent that issues multi-turn parallel grep, glob, and read operations for a frozen downstream coding agent",
+ "tags": [
+ "retrieval",
+ "context-reduction",
+ "parallelism"
+ ]
+ },
+ "evidence": {
+ "result": "Resolve rate 27.0% versus 25.8% without retrieval; resolved tasks use 15% fewer rounds and 19% fewer tokens.",
+ "same_model": "yes",
+ "same_budget": "unknown",
+ "strength": "contextual",
+ "claim_type": "mixed",
+ "comparison_scope": "component-level",
+ "source_location": "Abstract and frozen-agent retrieval comparison",
+ "caveats": "The clean numeric comparison is against a frozen no-retrieval downstream agent, not necessarily the released Claude Code CLI."
+ }
+ },
+ {
+ "id": "scate-2026",
+ "title": "SCATE: Learning to Supervise Coding Agents for Cost-Effective Test Generation",
+ "authors": [
+ "Sijia Gu",
+ "Noor Nashid",
+ "Ali Mesbah"
+ ],
+ "year": 2026,
+ "venue": "arXiv",
+ "publication_status": "preprint",
+ "classification": "related",
+ "system": "SCATE",
+ "paper_url": "https://arxiv.org/abs/2607.08983",
+ "artifact_status": "not-found",
+ "arxiv_id": "2607.08983",
+ "published_at": "2026-07-09",
+ "products": [
+ {
+ "product": "claude-code",
+ "role": "evaluated",
+ "model": "not-reported",
+ "version": "not-reported"
+ }
+ ],
+ "task": {
+ "summary": "Automated test generation",
+ "benchmark": "Repository-level test-generation benchmark"
+ },
+ "method": {
+ "summary": "Contextual-bandit supervisor that selects testing actions from current coverage and testability state",
+ "tags": [
+ "test-feedback",
+ "verifier-loop"
+ ]
+ },
+ "evidence": {
+ "result": "Line coverage improves 32.3% and branch coverage 30.9% over agent-only Gemini CLI; adaptation is also evaluated with Claude Code.",
+ "same_model": "unknown",
+ "same_budget": "unknown",
+ "strength": "contextual",
+ "claim_type": "quality",
+ "comparison_scope": "component-level",
+ "source_location": "Abstract and cross-agent adaptation experiment",
+ "caveats": "The abstract does not provide an equally complete numeric head-to-head against Claude Code."
+ }
+ },
+ {
+ "id": "terminal-bench-2-2026",
+ "title": "Terminal-Bench 2.0: Advancing Agentic Terminal Intelligence through Better Evaluation and Verification",
+ "authors": [
+ "Mike A. Merrill",
+ "Alexander G. Shaw",
+ "Nicholas Carlini",
+ "Boxuan Li",
+ "Harsh Raj",
+ "Ivan Bercovich",
+ "Lin Shi",
+ "Jeong Yeon Shin",
+ "Thomas Walshe",
+ "E. Kelly Buchanan",
+ "Junhong Shen",
+ "Guanghao Ye",
+ "Haowei Lin",
+ "Jason Poulos",
+ "Maoyu Wang",
+ "Marianna Nezhurina",
+ "Jenia Jitsev",
+ "Di Lu",
+ "Orfeas Menis Mastromichalakis",
+ "Zhiwei Xu",
+ "Zizhao Chen",
+ "Yue Liu",
+ "Robert Zhang",
+ "Leon Liangyu Chen",
+ "Anurag Kashyap",
+ "Jan-Lucas Uslu",
+ "Jeffrey Li",
+ "Jianbo Wu",
+ "Minghao Yan",
+ "Song Bian",
+ "Vedang Sharma",
+ "Ke Sun",
+ "Steven Dillmann",
+ "Akshay Anand",
+ "Andrew Lanpouthakoun",
+ "Bardia Koopah",
+ "Changran Hu",
+ "Etash Guha",
+ "Gabriel H. S. Dreiman",
+ "Jiacheng Zhu",
+ "Karl Krauth",
+ "Li Zhong",
+ "Niklas Muennighoff",
+ "Robert Amanfu",
+ "Shangyin Tan",
+ "Shreyas Pimpalgaonkar",
+ "Tushar Aggarwal",
+ "Xiangning Lin",
+ "Xin Lan",
+ "Xuandong Zhao",
+ "Yiqing Liang",
+ "Yuanli Wang",
+ "Zilong Wang",
+ "Changzhi Zhou",
+ "David Heineman",
+ "Hange Liu",
+ "Harsh Trivedi",
+ "John Yang",
+ "Junhong Lin",
+ "Manish Shetty",
+ "Michael Yang",
+ "Nabil Omi",
+ "Negin Raoof",
+ "Shanda Li",
+ "Terry Yue Zhuo",
+ "Wuwei Lin",
+ "Yiwei Dai",
+ "Yuxin Wang",
+ "Wenhao Chai",
+ "Shang Zhou",
+ "Dariush Wahdany",
+ "Ziyu She",
+ "Jiaming Hu",
+ "Zhikang Dong",
+ "Yuxuan Zhu",
+ "Sasha Cui",
+ "Ahson Saiyed",
+ "Arinbjörn Kolbeinsson",
+ "Jesse Hu",
+ "Christopher Michael Rytting",
+ "Ryan Marten",
+ "Yixin Wang",
+ "Alex Dimakis",
+ "Andy Konwinski",
+ "Ludwig Schmidt"
+ ],
+ "year": 2026,
+ "venue": "ICLR",
+ "publication_status": "main",
+ "classification": "evaluation",
+ "system": "Terminal-Bench 2.0",
+ "paper_url": "https://proceedings.iclr.cc/paper_files/paper/2026/file/444a3737adaee10d86ad2ef5f74468e6-Paper-Conference.pdf",
+ "artifact_url": "https://github.com/harbor-framework/terminal-bench-2",
+ "artifact_status": "official",
+ "arxiv_id": "2601.11868",
+ "published_at": "2026-01-17",
+ "products": [
+ {
+ "product": "claude-code",
+ "role": "evaluated",
+ "model": "Claude Opus 4.5",
+ "version": "not-reported"
+ },
+ {
+ "product": "codex-cli",
+ "role": "evaluated",
+ "model": "GPT-5.2",
+ "version": "not-reported"
+ }
+ ],
+ "task": {
+ "summary": "General terminal-agent evaluation",
+ "benchmark": "Terminal-Bench 2.0"
+ },
+ "method": {
+ "summary": "Evaluation-only benchmark with stronger task verification and a neutral terminal scaffold",
+ "tags": [
+ "verifier-loop"
+ ]
+ },
+ "evidence": {
+ "result": "GPT-5.2 with Codex CLI scores 62.9; Claude Opus 4.5 scores 57.8 with Terminus 2 and 52.1 with Claude Code.",
+ "same_model": "unknown",
+ "same_budget": "unknown",
+ "strength": "high",
+ "claim_type": "mixed",
+ "comparison_scope": "benchmark-only",
+ "source_location": "Main agent and model results table",
+ "caveats": "Benchmark result; differences can reflect both model and harness, and no new method is proposed to beat the products."
+ }
+ },
+ {
+ "id": "bountybench-2025",
+ "title": "BountyBench: Dollar Impact of AI Agent Attackers and Defenders on Real-World Cybersecurity Systems",
+ "authors": [
+ "Andy K. Zhang",
+ "Joey Ji",
+ "Celeste Menders",
+ "Riya Dulepet",
+ "Thomas Qin",
+ "Ron Y. Wang",
+ "Junrong Wu",
+ "Kyleen Liao",
+ "Jiliang Li",
+ "Jinghan Hu",
+ "Sara Hong",
+ "Nardos Demilew",
+ "Shivatmica Murgai",
+ "Jason Tran",
+ "Nishka Kacheria",
+ "Ethan Ho",
+ "Denis Liu",
+ "Lauren McLane",
+ "Olivia Bruvik",
+ "Dai-Rong Han",
+ "Seungwoo Kim",
+ "Akhil Vyas",
+ "Cuiyuanxiu Chen",
+ "Ryan Li",
+ "Weiran Xu",
+ "Jonathan Z. Ye",
+ "Prerit Choudhary",
+ "Siddharth M. Bhatia",
+ "Vikram Sivashankar",
+ "Yuxuan Bao",
+ "Dawn Song",
+ "Dan Boneh",
+ "Daniel E. Ho",
+ "Percy Liang"
+ ],
+ "year": 2025,
+ "venue": "NeurIPS",
+ "publication_status": "dataset-benchmark",
+ "classification": "evaluation",
+ "system": "BountyBench",
+ "paper_url": "https://proceedings.neurips.cc/paper_files/paper/2025/hash/faed4276b52ef762879db4142655c699-Abstract-Datasets_and_Benchmarks_Track.html",
+ "artifact_url": "https://github.com/bountybench/bountybench",
+ "artifact_status": "official",
+ "arxiv_id": "2505.15216",
+ "published_at": "2025-05-21",
+ "products": [
+ {
+ "product": "claude-code",
+ "role": "evaluated",
+ "model": "not-reported",
+ "version": "not-reported"
+ },
+ {
+ "product": "codex-cli",
+ "role": "evaluated",
+ "model": "o3-high and o4-mini",
+ "version": "not-reported"
+ }
+ ],
+ "task": {
+ "summary": "Vulnerability detection, exploitation, and patching",
+ "benchmark": "BountyBench"
+ },
+ "method": {
+ "summary": "Evaluation-only benchmark of offensive and defensive cybersecurity agents",
+ "tags": [
+ "verifier-loop"
+ ]
+ },
+ "evidence": {
+ "result": "Codex o3-high detects 12.5% and patches 90%; Claude Code patches 87.5% and exploits 57.5% in the reported settings.",
+ "same_model": "unknown",
+ "same_budget": "unknown",
+ "strength": "high",
+ "claim_type": "quality",
+ "comparison_scope": "benchmark-only",
+ "source_location": "Official abstract and agent task-results tables",
+ "caveats": "Benchmark-only evidence across different tasks and model configurations."
+ }
+ },
+ {
+ "id": "engineering-pitfalls-2026",
+ "title": "Engineering Pitfalls in AI Coding Tools: An Empirical Study of Bugs in Claude Code, Codex, and Gemini CLI",
+ "authors": [
+ "Ruixin Zhang",
+ "Wuyang Dai",
+ "Hung Viet Pham",
+ "Gias Uddin",
+ "Jinqiu Yang",
+ "Song Wang"
+ ],
+ "year": 2026,
+ "venue": "FSE Industry Track",
+ "publication_status": "conference",
+ "classification": "evaluation",
+ "system": "Bug taxonomy",
+ "paper_url": "https://conf.researchr.org/details/fse-2026/fse-2026-industry-papers/12/Engineering-Pitfalls-in-AI-Coding-Tools-An-Empirical-Study-of-Bugs-in-Claude-Code-C",
+ "artifact_status": "not-found",
+ "arxiv_id": "2603.20847",
+ "doi": "10.1145/3803437.3805213",
+ "published_at": "2026-03-21",
+ "products": [
+ {
+ "product": "claude-code",
+ "role": "evaluated",
+ "model": "not-applicable",
+ "version": "multiple"
+ },
+ {
+ "product": "codex-cli",
+ "role": "evaluated",
+ "model": "not-applicable",
+ "version": "multiple"
+ }
+ ],
+ "task": {
+ "summary": "Empirical analysis of product bug reports",
+ "benchmark": "More than 3,800 reported bugs"
+ },
+ "method": {
+ "summary": "Evaluation-only manual taxonomy of functionality, integration, invocation, and command-execution failures",
+ "tags": [
+ "observability"
+ ]
+ },
+ "evidence": {
+ "result": "More than 67% of studied bugs concern functionality; API, integration, and configuration account for 36.9% of root causes.",
+ "same_model": "unknown",
+ "same_budget": "unknown",
+ "strength": "high",
+ "claim_type": "diagnostic",
+ "comparison_scope": "benchmark-only",
+ "source_location": "Abstract and root-cause, symptom, and workflow-stage findings",
+ "caveats": "Failure-taxonomy study; it does not propose an agent that outperforms the products."
+ }
+ },
+ {
+ "id": "ticoder-2022",
+ "title": "Interactive Code Generation via Test-Driven User-Intent Formalization",
+ "authors": [
+ "Shuvendu K. Lahiri",
+ "Sarah Fakhoury",
+ "Aaditya Naik",
+ "Georgios Sakkas",
+ "Saikat Chakraborty",
+ "Madanlal Musuvathi",
+ "Piali Choudhury",
+ "Curtis von Veh",
+ "Jeevana Priya Inala",
+ "Chenglong Wang",
+ "Jianfeng Gao"
+ ],
+ "year": 2022,
+ "venue": "AST at ICSE",
+ "publication_status": "workshop",
+ "classification": "historical",
+ "system": "TiCoder",
+ "paper_url": "https://arxiv.org/abs/2208.05950",
+ "artifact_url": "https://github.com/microsoft/TiCoder",
+ "artifact_status": "official",
+ "arxiv_id": "2208.05950",
+ "published_at": "2022-08-11",
+ "products": [
+ {
+ "product": "openai-codex-model",
+ "role": "historical-baseline",
+ "model": "OpenAI Codex",
+ "version": "not-reported"
+ }
+ ],
+ "task": {
+ "summary": "Interactive function-level code generation",
+ "benchmark": "MBPP and HumanEval"
+ },
+ "method": {
+ "summary": "Generate tests to formalize user intent, collect lightweight feedback, and rank or prune candidate programs",
+ "tags": [
+ "test-feedback",
+ "verifier-loop"
+ ]
+ },
+ "evidence": {
+ "result": "With one to five simulated user queries, Codex pass@1 gains 22.49 to 37.71 points on MBPP and 24.79 to 53.98 points on HumanEval.",
+ "same_model": "yes",
+ "same_budget": "no",
+ "strength": "high",
+ "claim_type": "quality",
+ "comparison_scope": "historical-model",
+ "source_location": "MBPP and HumanEval evaluation section",
+ "caveats": "This is the historical OpenAI Codex model, not the modern Codex CLI product."
+ }
+ }
+ ]
+}
diff --git a/data/papers.yaml b/data/papers.yaml
index 46f6149..0ee48c9 100644
--- a/data/papers.yaml
+++ b/data/papers.yaml
@@ -3,16 +3,22 @@ reviewed_at: "2026-08-21"
papers:
- id: qlcoder-2026
title: "QLCoder: A Query Synthesizer For Static Analysis of Security Vulnerabilities"
+ authors: [Claire Wang, Ziyang Li, Saikat Dutta, Mayur Naik]
year: 2026
venue: ICLR
publication_status: main
classification: direct
+ system: QLCoder
paper_url: https://proceedings.iclr.cc/paper_files/paper/2026/hash/2adf01ab15adde8820622f7f24bd516b-Abstract-Conference.html
+ artifact_url: https://github.com/neuralprogram/qlcoder
+ artifact_status: official
+ arxiv_id: "2511.08462"
+ published_at: "2025-11-11"
products:
- product: claude-code
role: baseline
- model: not-reported
- version: not-reported
+ model: Claude Sonnet 4
+ version: 1.0.120
task:
summary: CodeQL query synthesis from CVE metadata
benchmark: 176 CVEs across 111 Java projects
@@ -24,16 +30,24 @@ papers:
same_model: unknown
same_budget: unknown
strength: high
+ claim_type: quality
+ comparison_scope: product-level
+ source_location: Official abstract, main evaluation, and artifact paper environment
caveats: Security-query synthesis is narrower than general software engineering.
- id: rpg-zerorepo-2026
title: "RPG: A Repository Planning Graph for Unified and Scalable Codebase Generation"
+ authors: [Jane Luo, Xin Zhang, Steven Liu, Jie Wu, Jianfeng Liu, Yiming Huang, Yangyu Huang, Chengyu Yin, Ying Xin, Yuefeng Zhan, Hao Sun, Qi Chen, Scarlett Li, Mao Yang]
year: 2026
venue: ICLR
publication_status: main
classification: direct
+ system: ZeroRepo
paper_url: https://proceedings.iclr.cc/paper_files/paper/2026/file/9482f45fdd89aba9130bb04c44f788a9-Paper-Conference.pdf
artifact_url: https://github.com/microsoft/RPG-ZeroRepo
+ artifact_status: official
+ arxiv_id: "2509.16198"
+ published_at: "2025-09-19"
products:
- product: claude-code
role: baseline
@@ -54,16 +68,24 @@ papers:
same_model: "no"
same_budget: unknown
strength: high
+ claim_type: quality
+ comparison_scope: product-level
+ source_location: Section 4.3 and RepoCraft main-results table
caveats: ZeroRepo uses o3-mini while the Claude Code baseline uses Claude 4 Sonnet, so model and harness effects are mixed.
- id: artemis-2026
title: "Comparing AI Agents to Cybersecurity Professionals in Real-World Penetration Testing"
+ authors: [Justin W. Lin, Eliot Krzysztof Jones, Donovan Julian Jasper, Ethan Jun-shen Ho, Anna Wu, Arnold Tianyi Yang, Neil Perry, Andy Zou, Matt Fredrikson, J. Zico Kolter, Percy Liang, Dan Boneh, Daniel E. Ho]
year: 2026
venue: ICLR
publication_status: main
classification: direct
+ system: ARTEMIS
paper_url: https://arxiv.org/abs/2512.09882
artifact_url: https://github.com/Stanford-Trinity/ARTEMIS
+ artifact_status: official
+ arxiv_id: "2512.09882"
+ published_at: "2025-12-10"
products:
- product: claude-code
role: baseline
@@ -84,16 +106,24 @@ papers:
same_model: unknown
same_budget: "no"
strength: medium
+ claim_type: mixed
+ comparison_scope: product-level
+ source_location: Abstract and main penetration-testing results
caveats: Runtime limits, model combinations, safety refusals, and product policies differ across systems.
- id: mlzero-2025
title: "MLZero: A Multi-Agent System for End-to-end Machine Learning Automation"
+ authors: [Haoyang Fang, Boran Han, Nick Erickson, Xiyuan Zhang, Su Zhou, Anirudh Dagar, Jiani Zhang, Ali Caner Turkmen, Cuixiong Hu, Huzefa Rangwala, Ying Nian Wu, Bernie Wang, George Karypis]
year: 2025
venue: NeurIPS
publication_status: main
classification: direct
+ system: MLZero
paper_url: https://papers.neurips.cc/paper_files/paper/2025/hash/63ed15a46a143ff57484b38cd6b85d91-Abstract-Conference.html
artifact_url: https://github.com/autogluon/autogluon-assistant
+ artifact_status: official
+ arxiv_id: "2505.13941"
+ published_at: "2025-05-20"
products:
- product: codex-cli
role: baseline
@@ -110,15 +140,21 @@ papers:
same_model: unknown
same_budget: unknown
strength: medium
+ claim_type: quality
+ comparison_scope: product-level
+ source_location: Abstract, main benchmark results, and appendix error analysis
caveats: Results concern machine-learning automation and should not be generalized to all software-engineering tasks.
- id: formact-2026
title: "FormAct: Agentic Source Editing for Rich-Format Document Generation"
+ authors: [Eugene J. Yu, Xingxing Zhang, Yuan Xia, Tao Ge, Xun Wang, FNU Kartik, Vishwas Suryanarayanan, Cheng Yang, Amanda Jiang, Jiayu Ding, Xiangyu Wong, Tengchao Lv, Lei Cui, Si-Qing Chen, Furu Wei, Sujian Li]
year: 2026
venue: ICML
publication_status: main
classification: direct
+ system: FormAct
paper_url: https://openreview.net/forum?id=n7Ta0YEcgw
+ artifact_status: not-found
products:
- product: codex-cli
role: baseline
@@ -135,15 +171,24 @@ papers:
same_model: unknown
same_budget: unknown
strength: high
+ claim_type: mixed
+ comparison_scope: product-level
+ source_location: Main comparison table and review-refinement ablation
caveats: Codex has slightly higher content alignment, and the task is document formatting rather than source-code repair.
- id: agentic-harness-engineering-2026
title: "Agentic Harness Engineering: Observability-Driven Automatic Evolution of Coding-Agent Harnesses"
+ authors: [Jiahang Lin, Shichun Liu, Chengjun Pan, Lizhi Lin, Shihan Dou, Zhiheng Xi, Xuanjing Huang, Hang Yan, Zhenhua Han, Tao Gui, Yu-Gang Jiang]
year: 2026
venue: arXiv
publication_status: preprint
classification: direct
+ system: Agentic Harness Engineering
paper_url: https://arxiv.org/abs/2604.25850
+ artifact_url: https://github.com/china-qijizhifeng/agentic-harness-engineering
+ artifact_status: official
+ arxiv_id: "2604.25850"
+ published_at: "2026-04-28"
products:
- product: codex-cli
role: baseline
@@ -160,15 +205,24 @@ papers:
same_model: unknown
same_budget: unknown
strength: medium
+ claim_type: mixed
+ comparison_scope: product-level
+ source_location: Abstract and Terminal-Bench 2 transfer and ablation results
caveats: Preprint; model, harness, and evaluation-budget effects require careful separation.
- id: webdesigniter-2026
title: "WebDesignIter: Co-Evolving Design Knowledge for Repository-Level Front-End Code Generation"
+ authors: [Zheng Pei, Mingwei Liu, Zhenxi Chen, Zihao Wang, Yanlin Wang]
year: 2026
venue: arXiv
publication_status: preprint
classification: direct
+ system: WebDesignIter
paper_url: https://arxiv.org/abs/2607.10621
+ artifact_url: https://github.com/SYSUSELab/WebDesignIter
+ artifact_status: official
+ arxiv_id: "2607.10621"
+ published_at: "2026-07-12"
products:
- product: claude-code
role: baseline
@@ -189,15 +243,23 @@ papers:
same_model: unknown
same_budget: unknown
strength: medium
+ claim_type: quality
+ comparison_scope: product-level
+ source_location: Abstract, main Web-Bench comparison, and design-knowledge ablation
caveats: Front-end-specific benchmark and preprint status.
- id: prefactory-2026
title: "Prefactory: Automated Discovery and Application of Library-Adoption Refactorings"
+ authors: [Islem Bouzenia, Michael Pradel]
year: 2026
venue: arXiv
publication_status: preprint
classification: direct
+ system: Prefactory
paper_url: https://arxiv.org/abs/2607.17211
+ artifact_status: not-found
+ arxiv_id: "2607.17211"
+ published_at: "2026-07-19"
products:
- product: codex-cli
role: baseline
@@ -214,15 +276,23 @@ papers:
same_model: unknown
same_budget: unknown
strength: medium
+ claim_type: quality
+ comparison_scope: product-level
+ source_location: Abstract and PrefactoryBench detection and validation results
caveats: The strongest comparison concerns candidate detection, not universal end-to-end task success.
- id: rustprint-2026
title: "RustPrint: Documentation-Guided Agentic Codebase Migration from C to Rust"
+ authors: [Minh Le-Anh, Anh Nguyen Hoang, Bach Le, Nghi D. Q. Bui]
year: 2026
venue: arXiv
publication_status: preprint
classification: direct
+ system: RustPrint
paper_url: https://arxiv.org/abs/2605.14634
+ artifact_status: not-found
+ arxiv_id: "2605.14634"
+ published_at: "2026-05-14"
products:
- product: claude-code
role: baseline
@@ -239,16 +309,24 @@ papers:
same_model: "no"
same_budget: unknown
strength: medium
+ claim_type: quality
+ comparison_scope: product-level
+ source_location: Abstract and feature-preservation and cross-evaluation results
caveats: The comparison mixes model and scaffold choices and is specific to C-to-Rust migration.
- id: repoomp-2026
title: "RepoOMP: Repository-Aware Hotspot OpenMP Parallelization via Dependency-Aware Context Reduction"
+ authors: [Yongjie Qian, Ke Gao, Zhibin Zhang, Shaohui Peng, Ling Li]
year: 2026
venue: arXiv
publication_status: preprint
classification: direct
+ system: RepoOMP
paper_url: https://arxiv.org/abs/2608.05855
artifact_url: https://github.com/Qlalq/RepoOMP_Simplified
+ artifact_status: official
+ arxiv_id: "2608.05855"
+ published_at: "2026-08-06"
products:
- product: claude-code
role: baseline
@@ -265,15 +343,23 @@ papers:
same_model: unknown
same_budget: unknown
strength: medium
+ claim_type: mixed
+ comparison_scope: product-level
+ source_location: Abstract and nine-kernel cross-backbone comparison
caveats: Domain-specific high-performance-computing task and preprint status.
- id: icat-agent-2026
title: "Unlocking Model Potentials Through Adaptive Multi-Agent Scaffolding for Efficient Issue Resolution"
+ authors: [Yang Chen, Aliya Ahmad, Yiheng Zhou, Reyhaneh Jabbarvand]
year: 2026
venue: arXiv
publication_status: preprint
classification: direct
+ system: icat-agent
paper_url: https://arxiv.org/abs/2606.25514
+ artifact_status: not-found
+ arxiv_id: "2606.25514"
+ published_at: "2026-06-24"
products:
- product: claude-code
role: baseline
@@ -290,15 +376,24 @@ papers:
same_model: unknown
same_budget: unknown
strength: medium
+ claim_type: mixed
+ comparison_scope: product-level
+ source_location: Abstract and SWE-bench Verified and Pro comparisons
caveats: Exact model parity varies by reported configuration; preprint.
- id: co-coder-2026
title: "When Parallelism Pays Off: Cohesion-Aware Task Partitioning for Multi-Agent Coding"
+ authors: [Xu Yang, Lunyiu Nie, Ethan Chandra, Stanislav Gannutin, Fangru Lin, Swarat Chaudhuri]
year: 2026
venue: arXiv
publication_status: preprint
classification: direct
+ system: Co-Coder
paper_url: https://arxiv.org/abs/2606.00953
+ artifact_url: https://github.com/Flitternie/CoCoder
+ artifact_status: official
+ arxiv_id: "2606.00953"
+ published_at: "2026-05-31"
products:
- product: claude-code
role: baseline
@@ -315,15 +410,23 @@ papers:
same_model: unknown
same_budget: unknown
strength: medium
+ claim_type: mixed
+ comparison_scope: product-level
+ source_location: Abstract and DevEval and CodeProjectEval comparisons
caveats: Small 28-task evaluation and preprint status.
- id: agents-md-impact-2026
title: "On the Impact of AGENTS.md Files on the Efficiency of AI Coding Agents"
+ authors: [Jai Lal Lulla, Seyedmoein Mohsenimofidi, Matthias Galster, Jie M. Zhang, Sebastian Baltes, Christoph Treude]
year: 2026
venue: ICSE JAWs
publication_status: workshop
classification: direct
+ system: AGENTS.md
paper_url: https://conf.researchr.org/details/icse-2026/jaws-2026-papers/31/On-the-Impact-of-AGENTS-md-Files-on-the-Efficiency-of-AI-Coding-Agents
+ artifact_status: not-found
+ arxiv_id: "2601.20404"
+ published_at: "2026-01-28"
products:
- product: claude-code
role: baseline
@@ -344,15 +447,23 @@ papers:
same_model: "yes"
same_budget: "yes"
strength: high
+ claim_type: efficiency
+ comparison_scope: configuration-ablation
+ source_location: Abstract and with-versus-without AGENTS.md results
caveats: Workshop study and configuration intervention rather than a standalone agent.
- id: codegrep-2026
title: "CodeGrep: An RL-Trained Retrieval Agent for LLM Coding Agents"
+ authors: [Wuya Chen, Yihao Yang, Yang Cao, Yue Lin]
year: 2026
venue: arXiv
publication_status: preprint
classification: related
+ system: CodeGrep
paper_url: https://arxiv.org/abs/2608.05886
+ artifact_status: not-found
+ arxiv_id: "2608.05886"
+ published_at: "2026-08-06"
products:
- product: claude-code
role: evaluated
@@ -369,15 +480,23 @@ papers:
same_model: "yes"
same_budget: unknown
strength: contextual
+ claim_type: mixed
+ comparison_scope: component-level
+ source_location: Abstract and frozen-agent retrieval comparison
caveats: The clean numeric comparison is against a frozen no-retrieval downstream agent, not necessarily the released Claude Code CLI.
- id: scate-2026
title: "SCATE: Learning to Supervise Coding Agents for Cost-Effective Test Generation"
+ authors: [Sijia Gu, Noor Nashid, Ali Mesbah]
year: 2026
venue: arXiv
publication_status: preprint
classification: related
+ system: SCATE
paper_url: https://arxiv.org/abs/2607.08983
+ artifact_status: not-found
+ arxiv_id: "2607.08983"
+ published_at: "2026-07-09"
products:
- product: claude-code
role: evaluated
@@ -394,15 +513,24 @@ papers:
same_model: unknown
same_budget: unknown
strength: contextual
+ claim_type: quality
+ comparison_scope: component-level
+ source_location: Abstract and cross-agent adaptation experiment
caveats: The abstract does not provide an equally complete numeric head-to-head against Claude Code.
- id: terminal-bench-2-2026
title: "Terminal-Bench 2.0: Advancing Agentic Terminal Intelligence through Better Evaluation and Verification"
+ authors: [Mike A. Merrill, Alexander G. Shaw, Nicholas Carlini, Boxuan Li, Harsh Raj, Ivan Bercovich, Lin Shi, Jeong Yeon Shin, Thomas Walshe, E. Kelly Buchanan, Junhong Shen, Guanghao Ye, Haowei Lin, Jason Poulos, Maoyu Wang, Marianna Nezhurina, Jenia Jitsev, Di Lu, Orfeas Menis Mastromichalakis, Zhiwei Xu, Zizhao Chen, Yue Liu, Robert Zhang, Leon Liangyu Chen, Anurag Kashyap, Jan-Lucas Uslu, Jeffrey Li, Jianbo Wu, Minghao Yan, Song Bian, Vedang Sharma, Ke Sun, Steven Dillmann, Akshay Anand, Andrew Lanpouthakoun, Bardia Koopah, Changran Hu, Etash Guha, Gabriel H. S. Dreiman, Jiacheng Zhu, Karl Krauth, Li Zhong, Niklas Muennighoff, Robert Amanfu, Shangyin Tan, Shreyas Pimpalgaonkar, Tushar Aggarwal, Xiangning Lin, Xin Lan, Xuandong Zhao, Yiqing Liang, Yuanli Wang, Zilong Wang, Changzhi Zhou, David Heineman, Hange Liu, Harsh Trivedi, John Yang, Junhong Lin, Manish Shetty, Michael Yang, Nabil Omi, Negin Raoof, Shanda Li, Terry Yue Zhuo, Wuwei Lin, Yiwei Dai, Yuxin Wang, Wenhao Chai, Shang Zhou, Dariush Wahdany, Ziyu She, Jiaming Hu, Zhikang Dong, Yuxuan Zhu, Sasha Cui, Ahson Saiyed, Arinbjörn Kolbeinsson, Jesse Hu, Christopher Michael Rytting, Ryan Marten, Yixin Wang, Alex Dimakis, Andy Konwinski, Ludwig Schmidt]
year: 2026
venue: ICLR
publication_status: main
classification: evaluation
+ system: Terminal-Bench 2.0
paper_url: https://proceedings.iclr.cc/paper_files/paper/2026/file/444a3737adaee10d86ad2ef5f74468e6-Paper-Conference.pdf
+ artifact_url: https://github.com/harbor-framework/terminal-bench-2
+ artifact_status: official
+ arxiv_id: "2601.11868"
+ published_at: "2026-01-17"
products:
- product: claude-code
role: evaluated
@@ -423,15 +551,24 @@ papers:
same_model: unknown
same_budget: unknown
strength: high
+ claim_type: mixed
+ comparison_scope: benchmark-only
+ source_location: Main agent and model results table
caveats: Benchmark result; differences can reflect both model and harness, and no new method is proposed to beat the products.
- id: bountybench-2025
title: "BountyBench: Dollar Impact of AI Agent Attackers and Defenders on Real-World Cybersecurity Systems"
+ authors: [Andy K. Zhang, Joey Ji, Celeste Menders, Riya Dulepet, Thomas Qin, Ron Y. Wang, Junrong Wu, Kyleen Liao, Jiliang Li, Jinghan Hu, Sara Hong, Nardos Demilew, Shivatmica Murgai, Jason Tran, Nishka Kacheria, Ethan Ho, Denis Liu, Lauren McLane, Olivia Bruvik, Dai-Rong Han, Seungwoo Kim, Akhil Vyas, Cuiyuanxiu Chen, Ryan Li, Weiran Xu, Jonathan Z. Ye, Prerit Choudhary, Siddharth M. Bhatia, Vikram Sivashankar, Yuxuan Bao, Dawn Song, Dan Boneh, Daniel E. Ho, Percy Liang]
year: 2025
venue: NeurIPS
publication_status: dataset-benchmark
classification: evaluation
+ system: BountyBench
paper_url: https://proceedings.neurips.cc/paper_files/paper/2025/hash/faed4276b52ef762879db4142655c699-Abstract-Datasets_and_Benchmarks_Track.html
+ artifact_url: https://github.com/bountybench/bountybench
+ artifact_status: official
+ arxiv_id: "2505.15216"
+ published_at: "2025-05-21"
products:
- product: claude-code
role: evaluated
@@ -452,15 +589,24 @@ papers:
same_model: unknown
same_budget: unknown
strength: high
+ claim_type: quality
+ comparison_scope: benchmark-only
+ source_location: Official abstract and agent task-results tables
caveats: Benchmark-only evidence across different tasks and model configurations.
- id: engineering-pitfalls-2026
title: "Engineering Pitfalls in AI Coding Tools: An Empirical Study of Bugs in Claude Code, Codex, and Gemini CLI"
+ authors: [Ruixin Zhang, Wuyang Dai, Hung Viet Pham, Gias Uddin, Jinqiu Yang, Song Wang]
year: 2026
venue: FSE Industry Track
publication_status: conference
classification: evaluation
+ system: Bug taxonomy
paper_url: https://conf.researchr.org/details/fse-2026/fse-2026-industry-papers/12/Engineering-Pitfalls-in-AI-Coding-Tools-An-Empirical-Study-of-Bugs-in-Claude-Code-C
+ artifact_status: not-found
+ arxiv_id: "2603.20847"
+ doi: 10.1145/3803437.3805213
+ published_at: "2026-03-21"
products:
- product: claude-code
role: evaluated
@@ -481,16 +627,24 @@ papers:
same_model: unknown
same_budget: unknown
strength: high
+ claim_type: diagnostic
+ comparison_scope: benchmark-only
+ source_location: Abstract and root-cause, symptom, and workflow-stage findings
caveats: Failure-taxonomy study; it does not propose an agent that outperforms the products.
- id: ticoder-2022
title: "Interactive Code Generation via Test-Driven User-Intent Formalization"
+ authors: [Shuvendu K. Lahiri, Sarah Fakhoury, Aaditya Naik, Georgios Sakkas, Saikat Chakraborty, Madanlal Musuvathi, Piali Choudhury, Curtis von Veh, Jeevana Priya Inala, Chenglong Wang, Jianfeng Gao]
year: 2022
venue: AST at ICSE
publication_status: workshop
classification: historical
+ system: TiCoder
paper_url: https://arxiv.org/abs/2208.05950
artifact_url: https://github.com/microsoft/TiCoder
+ artifact_status: official
+ arxiv_id: "2208.05950"
+ published_at: "2022-08-11"
products:
- product: openai-codex-model
role: historical-baseline
@@ -507,4 +661,7 @@ papers:
same_model: "yes"
same_budget: "no"
strength: high
+ claim_type: quality
+ comparison_scope: historical-model
+ source_location: MBPP and HumanEval evaluation section
caveats: This is the historical OpenAI Codex model, not the modern Codex CLI product.
diff --git a/data/schema.json b/data/schema.json
index 1e1cda7..c9c9ff9 100644
--- a/data/schema.json
+++ b/data/schema.json
@@ -49,11 +49,14 @@
"required": [
"id",
"title",
+ "authors",
"year",
"venue",
"publication_status",
"classification",
+ "system",
"paper_url",
+ "artifact_status",
"products",
"task",
"method",
@@ -68,6 +71,15 @@
"type": "string",
"minLength": 5
},
+ "authors": {
+ "type": "array",
+ "minItems": 1,
+ "uniqueItems": true,
+ "items": {
+ "type": "string",
+ "minLength": 2
+ }
+ },
"year": {
"type": "integer",
"minimum": 2020,
@@ -91,6 +103,10 @@
"classification": {
"enum": ["direct", "related", "evaluation", "historical"]
},
+ "system": {
+ "type": "string",
+ "minLength": 2
+ },
"paper_url": {
"type": "string",
"format": "uri",
@@ -101,6 +117,21 @@
"format": "uri",
"pattern": "^https://"
},
+ "artifact_status": {
+ "enum": ["official", "community", "not-found"]
+ },
+ "arxiv_id": {
+ "type": "string",
+ "pattern": "^[0-9]{4}\\.[0-9]{4,5}$"
+ },
+ "doi": {
+ "type": "string",
+ "pattern": "^10\\.[0-9]{4,9}/[-._;()/:A-Za-z0-9]+$"
+ },
+ "published_at": {
+ "type": "string",
+ "format": "date"
+ },
"products": {
"type": "array",
"minItems": 1,
@@ -163,7 +194,16 @@
"evidence": {
"type": "object",
"additionalProperties": false,
- "required": ["result", "same_model", "same_budget", "strength", "caveats"],
+ "required": [
+ "result",
+ "same_model",
+ "same_budget",
+ "strength",
+ "claim_type",
+ "comparison_scope",
+ "source_location",
+ "caveats"
+ ],
"properties": {
"result": {
"type": "string",
@@ -178,6 +218,22 @@
"strength": {
"enum": ["high", "medium", "contextual"]
},
+ "claim_type": {
+ "enum": ["quality", "efficiency", "mixed", "diagnostic"]
+ },
+ "comparison_scope": {
+ "enum": [
+ "product-level",
+ "component-level",
+ "configuration-ablation",
+ "benchmark-only",
+ "historical-model"
+ ]
+ },
+ "source_location": {
+ "type": "string",
+ "minLength": 3
+ },
"caveats": {
"type": "string",
"minLength": 3
diff --git a/docs/taxonomy.md b/docs/taxonomy.md
index 0db3402..bd8b440 100644
--- a/docs/taxonomy.md
+++ b/docs/taxonomy.md
@@ -56,3 +56,32 @@ The paper studies the pre-CLI OpenAI Codex model. These entries are retained for
- `unknown`: the source does not establish parity clearly enough.
These fields describe experimental controls, not paper quality.
+
+## Claim types
+
+- `quality`: correctness, pass rate, coverage, ranking, or another outcome-quality measure.
+- `efficiency`: latency, token use, monetary cost, throughput, or another resource measure.
+- `mixed`: the central evidence includes both quality and efficiency.
+- `diagnostic`: the paper measures behavior or failures rather than claiming an improvement.
+
+## Comparison scope
+
+- `product-level`: the proposed system and production product are compared end to end.
+- `component-level`: the main evidence isolates retrieval, planning, testing, or another component.
+- `configuration-ablation`: the product is compared under changed instructions or configuration.
+- `benchmark-only`: the product is measured without an improvement intervention.
+- `historical-model`: the result concerns the pre-CLI OpenAI Codex model.
+
+## Evidence strength
+
+- `high`: the central claim is clearly reported in a primary paper table, figure, or section.
+- `medium`: useful product-level evidence is reported, but controls or reporting are incomplete.
+- `contextual`: the product appears in a partial, secondary, or non-central comparison.
+
+Strength records how directly the catalog claim is supported. It is not a paper-quality score.
+
+## Artifact status
+
+- `official`: maintained by the paper authors or publishing organization.
+- `community`: a useful third-party implementation or reproduction.
+- `not-found`: no qualifying artifact was found during the latest catalog review.
diff --git a/papers/README.md b/papers/README.md
new file mode 100644
index 0000000..138d308
--- /dev/null
+++ b/papers/README.md
@@ -0,0 +1,29 @@
+
+
+[← Home](../README.md) · [Browse by product](../views/by-product.md)
+
+# Paper dossiers
+
+Every catalog entry has a generated evidence dossier. The YAML catalog remains the source of truth.
+
+| System | Paper | Venue | Evidence class | Product | Artifact |
+|---|---|---|---|---|---|
+| [AGENTS.md](agents-md-impact-2026.md) | On the Impact of AGENTS.md Files on the Efficiency of AI Coding Agents | ICSE JAWs 2026 | Direct comparison | Claude Code
Codex CLI | Not found during the latest review |
+| [ARTEMIS](artemis-2026.md) | Comparing AI Agents to Cybersecurity Professionals in Real-World Penetration Testing | ICLR 2026 | Direct comparison | Claude Code (Claude Sonnet 4)
Codex CLI (GPT-5) | [Official artifact](https://github.com/Stanford-Trinity/ARTEMIS) |
+| [Agentic Harness Engineering](agentic-harness-engineering-2026.md) | Agentic Harness Engineering: Observability-Driven Automatic Evolution of Coding-Agent Harnesses | arXiv 2026 | Direct comparison | Codex CLI | [Official artifact](https://github.com/china-qijizhifeng/agentic-harness-engineering) |
+| [Bug taxonomy](engineering-pitfalls-2026.md) | Engineering Pitfalls in AI Coding Tools: An Empirical Study of Bugs in Claude Code, Codex, and Gemini CLI | FSE Industry Track 2026 | Evaluation only | Claude Code
Codex CLI | Not found during the latest review |
+| [Co-Coder](co-coder-2026.md) | When Parallelism Pays Off: Cohesion-Aware Task Partitioning for Multi-Agent Coding | arXiv 2026 | Direct comparison | Claude Code (Agent Teams) | [Official artifact](https://github.com/Flitternie/CoCoder) |
+| [CodeGrep](codegrep-2026.md) | CodeGrep: An RL-Trained Retrieval Agent for LLM Coding Agents | arXiv 2026 | Related method | Claude Code (Claude-Code-like agent) | Not found during the latest review |
+| [FormAct](formact-2026.md) | FormAct: Agentic Source Editing for Rich-Format Document Generation | ICML 2026 | Direct comparison | Codex CLI | Not found during the latest review |
+| [Prefactory](prefactory-2026.md) | Prefactory: Automated Discovery and Application of Library-Adoption Refactorings | arXiv 2026 | Direct comparison | Codex CLI | Not found during the latest review |
+| [QLCoder](qlcoder-2026.md) | QLCoder: A Query Synthesizer For Static Analysis of Security Vulnerabilities | ICLR 2026 | Direct comparison | Claude Code (Claude Sonnet 4, 1.0.120) | [Official artifact](https://github.com/neuralprogram/qlcoder) |
+| [RepoOMP](repoomp-2026.md) | RepoOMP: Repository-Aware Hotspot OpenMP Parallelization via Dependency-Aware Context Reduction | arXiv 2026 | Direct comparison | Claude Code | [Official artifact](https://github.com/Qlalq/RepoOMP_Simplified) |
+| [RustPrint](rustprint-2026.md) | RustPrint: Documentation-Guided Agentic Codebase Migration from C to Rust | arXiv 2026 | Direct comparison | Claude Code | Not found during the latest review |
+| [SCATE](scate-2026.md) | SCATE: Learning to Supervise Coding Agents for Cost-Effective Test Generation | arXiv 2026 | Related method | Claude Code | Not found during the latest review |
+| [Terminal-Bench 2.0](terminal-bench-2-2026.md) | Terminal-Bench 2.0: Advancing Agentic Terminal Intelligence through Better Evaluation and Verification | ICLR 2026 | Evaluation only | Claude Code (Claude Opus 4.5)
Codex CLI (GPT-5.2) | [Official artifact](https://github.com/harbor-framework/terminal-bench-2) |
+| [WebDesignIter](webdesigniter-2026.md) | WebDesignIter: Co-Evolving Design Knowledge for Repository-Level Front-End Code Generation | arXiv 2026 | Direct comparison | Claude Code
Codex CLI | [Official artifact](https://github.com/SYSUSELab/WebDesignIter) |
+| [ZeroRepo](rpg-zerorepo-2026.md) | RPG: A Repository Planning Graph for Unified and Scalable Codebase Generation | ICLR 2026 | Direct comparison | Claude Code (Claude 4 Sonnet)
Codex CLI (o3-pro) | [Official artifact](https://github.com/microsoft/RPG-ZeroRepo) |
+| [icat-agent](icat-agent-2026.md) | Unlocking Model Potentials Through Adaptive Multi-Agent Scaffolding for Efficient Issue Resolution | arXiv 2026 | Direct comparison | Claude Code | Not found during the latest review |
+| [BountyBench](bountybench-2025.md) | BountyBench: Dollar Impact of AI Agent Attackers and Defenders on Real-World Cybersecurity Systems | NeurIPS 2025 | Evaluation only | Claude Code
Codex CLI (o3-high and o4-mini) | [Official artifact](https://github.com/bountybench/bountybench) |
+| [MLZero](mlzero-2025.md) | MLZero: A Multi-Agent System for End-to-end Machine Learning Automation | NeurIPS 2025 | Direct comparison | Codex CLI (GPT-4.1 and o4-mini) | [Official artifact](https://github.com/autogluon/autogluon-assistant) |
+| [TiCoder](ticoder-2022.md) | Interactive Code Generation via Test-Driven User-Intent Formalization | AST at ICSE 2022 | Historical model | OpenAI Codex model (OpenAI Codex) | [Official artifact](https://github.com/microsoft/TiCoder) |
diff --git a/papers/agentic-harness-engineering-2026.md b/papers/agentic-harness-engineering-2026.md
new file mode 100644
index 0000000..7773547
--- /dev/null
+++ b/papers/agentic-harness-engineering-2026.md
@@ -0,0 +1,60 @@
+
+
+[← Paper index](README.md) · [Home](../README.md)
+
+# Agentic Harness Engineering
+
+## Agentic Harness Engineering: Observability-Driven Automatic Evolution of Coding-Agent Harnesses
+
+| Field | Value |
+|---|---|
+| Authors | Jiahang Lin, Shichun Liu, Chengjun Pan, Lizhi Lin, Shihan Dou, Zhiheng Xi, Xuanjing Huang, Hang Yan, Zhenhua Han, Tao Gui, Yu-Gang Jiang |
+| Venue | arXiv 2026 (preprint) |
+| Evidence class | Direct comparison |
+| First published | 2026-04-28 |
+| Identifiers | [arXiv:2604.25850](https://arxiv.org/abs/2604.25850) |
+| Artifact | [Official artifact](https://github.com/china-qijizhifeng/agentic-harness-engineering) |
+
+## Product configuration
+
+| Product | Role | Model | Product version |
+|---|---|---|---|
+| Codex CLI | baseline | not-reported | not-reported |
+
+## Task
+
+**Task.** Automatic evolution of coding-agent harnesses
+
+**Benchmark or scale.** Terminal-Bench 2 and SWE-bench Verified
+
+## Method
+
+Closed-loop harness evolution using component, trajectory, and decision observability with evidence-backed edits and rollback
+
+**Tags.** `harness-evolution` `observability` `memory` `verifier-loop`
+
+## Reported evidence
+
+Terminal-Bench 2 pass@1 improves from 69.7 to 77.0, above a human-designed Codex-CLI harness at 71.9; transfer uses 12% fewer tokens on SWE-bench Verified.
+
+| Control | Recorded value |
+|---|---|
+| Same model | unknown |
+| Same budget | unknown |
+| Evidence strength | medium |
+| Claim type | mixed |
+| Comparison scope | product-level |
+| Source location | Abstract and Terminal-Bench 2 transfer and ablation results |
+
+## Caveat
+
+Preprint; model, harness, and evaluation-budget effects require careful separation.
+
+## Primary links
+
+- [Paper](https://arxiv.org/abs/2604.25850)
+- [Artifact](https://github.com/china-qijizhifeng/agentic-harness-engineering)
+
+---
+
+Catalog ID: `agentic-harness-engineering-2026` · Metadata last reviewed with catalog release.
diff --git a/papers/agents-md-impact-2026.md b/papers/agents-md-impact-2026.md
new file mode 100644
index 0000000..fc9b515
--- /dev/null
+++ b/papers/agents-md-impact-2026.md
@@ -0,0 +1,61 @@
+
+
+[← Paper index](README.md) · [Home](../README.md)
+
+# AGENTS.md
+
+## On the Impact of AGENTS.md Files on the Efficiency of AI Coding Agents
+
+| Field | Value |
+|---|---|
+| Authors | Jai Lal Lulla, Seyedmoein Mohsenimofidi, Matthias Galster, Jie M. Zhang, Sebastian Baltes, Christoph Treude |
+| Venue | ICSE JAWs 2026 (workshop) |
+| Evidence class | Direct comparison |
+| First published | 2026-01-28 |
+| Identifiers | [arXiv:2601.20404](https://arxiv.org/abs/2601.20404) |
+| Artifact | Not found during the latest review |
+
+## Product configuration
+
+| Product | Role | Model | Product version |
+|---|---|---|---|
+| Claude Code | baseline | not-reported | not-reported |
+| Codex CLI | baseline | not-reported | not-reported |
+
+## Task
+
+**Task.** Real pull-request tasks with and without repository instructions
+
+**Benchmark or scale.** 10 repositories and 124 pull requests
+
+## Method
+
+Persistent repository-level build, test, style, and directory guidance in AGENTS.md
+
+**Tags.** `repository-instructions` `structured-state` `context-reduction`
+
+## Reported evidence
+
+Median runtime falls 28.64% and output tokens fall 16.58% with comparable completion behavior.
+
+| Control | Recorded value |
+|---|---|
+| Same model | yes |
+| Same budget | yes |
+| Evidence strength | high |
+| Claim type | efficiency |
+| Comparison scope | configuration-ablation |
+| Source location | Abstract and with-versus-without AGENTS.md results |
+
+## Caveat
+
+Workshop study and configuration intervention rather than a standalone agent.
+
+## Primary links
+
+- [Paper](https://conf.researchr.org/details/icse-2026/jaws-2026-papers/31/On-the-Impact-of-AGENTS-md-Files-on-the-Efficiency-of-AI-Coding-Agents)
+- No official artifact was found during the latest review.
+
+---
+
+Catalog ID: `agents-md-impact-2026` · Metadata last reviewed with catalog release.
diff --git a/papers/artemis-2026.md b/papers/artemis-2026.md
new file mode 100644
index 0000000..4d227ea
--- /dev/null
+++ b/papers/artemis-2026.md
@@ -0,0 +1,61 @@
+
+
+[← Paper index](README.md) · [Home](../README.md)
+
+# ARTEMIS
+
+## Comparing AI Agents to Cybersecurity Professionals in Real-World Penetration Testing
+
+| Field | Value |
+|---|---|
+| Authors | Justin W. Lin, Eliot Krzysztof Jones, Donovan Julian Jasper, Ethan Jun-shen Ho, Anna Wu, Arnold Tianyi Yang, Neil Perry, Andy Zou, Matt Fredrikson, J. Zico Kolter, Percy Liang, Dan Boneh, Daniel E. Ho |
+| Venue | ICLR 2026 (main) |
+| Evidence class | Direct comparison |
+| First published | 2025-12-10 |
+| Identifiers | [arXiv:2512.09882](https://arxiv.org/abs/2512.09882) |
+| Artifact | [Official artifact](https://github.com/Stanford-Trinity/ARTEMIS) |
+
+## Product configuration
+
+| Product | Role | Model | Product version |
+|---|---|---|---|
+| Claude Code | baseline | Claude Sonnet 4 | not-reported |
+| Codex CLI | baseline | GPT-5 | not-reported |
+
+## Task
+
+**Task.** Long-horizon penetration testing in a live enterprise network
+
+**Benchmark or scale.** Approximately 8,000 hosts across 12 subnets
+
+## Method
+
+Supervisor with dynamic expert agents, recursive task decomposition, parallel exploration, context management, triage, and reporting
+
+**Tags.** `multi-agent` `dynamic-specialization` `parallelism` `memory` `verifier-loop`
+
+## Reported evidence
+
+ARTEMIS found 9 valid vulnerabilities with an 82% valid-submission rate, ranked second overall, and outperformed 9 of 10 human participants.
+
+| Control | Recorded value |
+|---|---|
+| Same model | unknown |
+| Same budget | no |
+| Evidence strength | medium |
+| Claim type | mixed |
+| Comparison scope | product-level |
+| Source location | Abstract and main penetration-testing results |
+
+## Caveat
+
+Runtime limits, model combinations, safety refusals, and product policies differ across systems.
+
+## Primary links
+
+- [Paper](https://arxiv.org/abs/2512.09882)
+- [Artifact](https://github.com/Stanford-Trinity/ARTEMIS)
+
+---
+
+Catalog ID: `artemis-2026` · Metadata last reviewed with catalog release.
diff --git a/papers/bountybench-2025.md b/papers/bountybench-2025.md
new file mode 100644
index 0000000..9405baf
--- /dev/null
+++ b/papers/bountybench-2025.md
@@ -0,0 +1,61 @@
+
+
+[← Paper index](README.md) · [Home](../README.md)
+
+# BountyBench
+
+## BountyBench: Dollar Impact of AI Agent Attackers and Defenders on Real-World Cybersecurity Systems
+
+| Field | Value |
+|---|---|
+| Authors | Andy K. Zhang, Joey Ji, Celeste Menders, Riya Dulepet, Thomas Qin, Ron Y. Wang, Junrong Wu, Kyleen Liao, Jiliang Li, Jinghan Hu, Sara Hong, Nardos Demilew, Shivatmica Murgai, Jason Tran, Nishka Kacheria, Ethan Ho, Denis Liu, Lauren McLane, Olivia Bruvik, Dai-Rong Han, Seungwoo Kim, Akhil Vyas, Cuiyuanxiu Chen, Ryan Li, Weiran Xu, Jonathan Z. Ye, Prerit Choudhary, Siddharth M. Bhatia, Vikram Sivashankar, Yuxuan Bao, Dawn Song, Dan Boneh, Daniel E. Ho, Percy Liang |
+| Venue | NeurIPS 2025 (dataset-benchmark) |
+| Evidence class | Evaluation only |
+| First published | 2025-05-21 |
+| Identifiers | [arXiv:2505.15216](https://arxiv.org/abs/2505.15216) |
+| Artifact | [Official artifact](https://github.com/bountybench/bountybench) |
+
+## Product configuration
+
+| Product | Role | Model | Product version |
+|---|---|---|---|
+| Claude Code | evaluated | not-reported | not-reported |
+| Codex CLI | evaluated | o3-high and o4-mini | not-reported |
+
+## Task
+
+**Task.** Vulnerability detection, exploitation, and patching
+
+**Benchmark or scale.** BountyBench
+
+## Method
+
+Evaluation-only benchmark of offensive and defensive cybersecurity agents
+
+**Tags.** `verifier-loop`
+
+## Reported evidence
+
+Codex o3-high detects 12.5% and patches 90%; Claude Code patches 87.5% and exploits 57.5% in the reported settings.
+
+| Control | Recorded value |
+|---|---|
+| Same model | unknown |
+| Same budget | unknown |
+| Evidence strength | high |
+| Claim type | quality |
+| Comparison scope | benchmark-only |
+| Source location | Official abstract and agent task-results tables |
+
+## Caveat
+
+Benchmark-only evidence across different tasks and model configurations.
+
+## Primary links
+
+- [Paper](https://proceedings.neurips.cc/paper_files/paper/2025/hash/faed4276b52ef762879db4142655c699-Abstract-Datasets_and_Benchmarks_Track.html)
+- [Artifact](https://github.com/bountybench/bountybench)
+
+---
+
+Catalog ID: `bountybench-2025` · Metadata last reviewed with catalog release.
diff --git a/papers/co-coder-2026.md b/papers/co-coder-2026.md
new file mode 100644
index 0000000..56bcafb
--- /dev/null
+++ b/papers/co-coder-2026.md
@@ -0,0 +1,60 @@
+
+
+[← Paper index](README.md) · [Home](../README.md)
+
+# Co-Coder
+
+## When Parallelism Pays Off: Cohesion-Aware Task Partitioning for Multi-Agent Coding
+
+| Field | Value |
+|---|---|
+| Authors | Xu Yang, Lunyiu Nie, Ethan Chandra, Stanislav Gannutin, Fangru Lin, Swarat Chaudhuri |
+| Venue | arXiv 2026 (preprint) |
+| Evidence class | Direct comparison |
+| First published | 2026-05-31 |
+| Identifiers | [arXiv:2606.00953](https://arxiv.org/abs/2606.00953) |
+| Artifact | [Official artifact](https://github.com/Flitternie/CoCoder) |
+
+## Product configuration
+
+| Product | Role | Model | Product version |
+|---|---|---|---|
+| Claude Code | baseline | not-reported | Agent Teams |
+
+## Task
+
+**Task.** Parallel multi-agent coding
+
+**Benchmark or scale.** 28 DevEval and CodeProjectEval tasks
+
+## Method
+
+Static dependency graph partitioning, structural hub isolation, community detection, and dependency-aware scheduling
+
+**Tags.** `repository-graph` `dependency-aware-planning` `multi-agent` `parallelism`
+
+## Reported evidence
+
+Pass rate improves 14.0%, wall-clock speedup reaches 2.10x, and API cost falls 35% relative to Claude Code Agent Teams and simpler schedules.
+
+| Control | Recorded value |
+|---|---|
+| Same model | unknown |
+| Same budget | unknown |
+| Evidence strength | medium |
+| Claim type | mixed |
+| Comparison scope | product-level |
+| Source location | Abstract and DevEval and CodeProjectEval comparisons |
+
+## Caveat
+
+Small 28-task evaluation and preprint status.
+
+## Primary links
+
+- [Paper](https://arxiv.org/abs/2606.00953)
+- [Artifact](https://github.com/Flitternie/CoCoder)
+
+---
+
+Catalog ID: `co-coder-2026` · Metadata last reviewed with catalog release.
diff --git a/papers/codegrep-2026.md b/papers/codegrep-2026.md
new file mode 100644
index 0000000..65c2be3
--- /dev/null
+++ b/papers/codegrep-2026.md
@@ -0,0 +1,60 @@
+
+
+[← Paper index](README.md) · [Home](../README.md)
+
+# CodeGrep
+
+## CodeGrep: An RL-Trained Retrieval Agent for LLM Coding Agents
+
+| Field | Value |
+|---|---|
+| Authors | Wuya Chen, Yihao Yang, Yang Cao, Yue Lin |
+| Venue | arXiv 2026 (preprint) |
+| Evidence class | Related method |
+| First published | 2026-08-06 |
+| Identifiers | [arXiv:2608.05886](https://arxiv.org/abs/2608.05886) |
+| Artifact | Not found during the latest review |
+
+## Product configuration
+
+| Product | Role | Model | Product version |
+|---|---|---|---|
+| Claude Code | evaluated | not-reported | Claude-Code-like agent |
+
+## Task
+
+**Task.** Repository retrieval for issue resolution
+
+**Benchmark or scale.** SWE-bench Verified
+
+## Method
+
+GRPO-trained 14B retrieval agent that issues multi-turn parallel grep, glob, and read operations for a frozen downstream coding agent
+
+**Tags.** `retrieval` `context-reduction` `parallelism`
+
+## Reported evidence
+
+Resolve rate 27.0% versus 25.8% without retrieval; resolved tasks use 15% fewer rounds and 19% fewer tokens.
+
+| Control | Recorded value |
+|---|---|
+| Same model | yes |
+| Same budget | unknown |
+| Evidence strength | contextual |
+| Claim type | mixed |
+| Comparison scope | component-level |
+| Source location | Abstract and frozen-agent retrieval comparison |
+
+## Caveat
+
+The clean numeric comparison is against a frozen no-retrieval downstream agent, not necessarily the released Claude Code CLI.
+
+## Primary links
+
+- [Paper](https://arxiv.org/abs/2608.05886)
+- No official artifact was found during the latest review.
+
+---
+
+Catalog ID: `codegrep-2026` · Metadata last reviewed with catalog release.
diff --git a/papers/engineering-pitfalls-2026.md b/papers/engineering-pitfalls-2026.md
new file mode 100644
index 0000000..b32c63f
--- /dev/null
+++ b/papers/engineering-pitfalls-2026.md
@@ -0,0 +1,61 @@
+
+
+[← Paper index](README.md) · [Home](../README.md)
+
+# Bug taxonomy
+
+## Engineering Pitfalls in AI Coding Tools: An Empirical Study of Bugs in Claude Code, Codex, and Gemini CLI
+
+| Field | Value |
+|---|---|
+| Authors | Ruixin Zhang, Wuyang Dai, Hung Viet Pham, Gias Uddin, Jinqiu Yang, Song Wang |
+| Venue | FSE Industry Track 2026 (conference) |
+| Evidence class | Evaluation only |
+| First published | 2026-03-21 |
+| Identifiers | [arXiv:2603.20847](https://arxiv.org/abs/2603.20847) · [DOI:10.1145/3803437.3805213](https://doi.org/10.1145/3803437.3805213) |
+| Artifact | Not found during the latest review |
+
+## Product configuration
+
+| Product | Role | Model | Product version |
+|---|---|---|---|
+| Claude Code | evaluated | not-applicable | multiple |
+| Codex CLI | evaluated | not-applicable | multiple |
+
+## Task
+
+**Task.** Empirical analysis of product bug reports
+
+**Benchmark or scale.** More than 3,800 reported bugs
+
+## Method
+
+Evaluation-only manual taxonomy of functionality, integration, invocation, and command-execution failures
+
+**Tags.** `observability`
+
+## Reported evidence
+
+More than 67% of studied bugs concern functionality; API, integration, and configuration account for 36.9% of root causes.
+
+| Control | Recorded value |
+|---|---|
+| Same model | unknown |
+| Same budget | unknown |
+| Evidence strength | high |
+| Claim type | diagnostic |
+| Comparison scope | benchmark-only |
+| Source location | Abstract and root-cause, symptom, and workflow-stage findings |
+
+## Caveat
+
+Failure-taxonomy study; it does not propose an agent that outperforms the products.
+
+## Primary links
+
+- [Paper](https://conf.researchr.org/details/fse-2026/fse-2026-industry-papers/12/Engineering-Pitfalls-in-AI-Coding-Tools-An-Empirical-Study-of-Bugs-in-Claude-Code-C)
+- No official artifact was found during the latest review.
+
+---
+
+Catalog ID: `engineering-pitfalls-2026` · Metadata last reviewed with catalog release.
diff --git a/papers/formact-2026.md b/papers/formact-2026.md
new file mode 100644
index 0000000..8eb1e02
--- /dev/null
+++ b/papers/formact-2026.md
@@ -0,0 +1,60 @@
+
+
+[← Paper index](README.md) · [Home](../README.md)
+
+# FormAct
+
+## FormAct: Agentic Source Editing for Rich-Format Document Generation
+
+| Field | Value |
+|---|---|
+| Authors | Eugene J. Yu, Xingxing Zhang, Yuan Xia, Tao Ge, Xun Wang, FNU Kartik, Vishwas Suryanarayanan, Cheng Yang, Amanda Jiang, Jiayu Ding, Xiangyu Wong, Tengchao Lv, Lei Cui, Si-Qing Chen, Furu Wei, Sujian Li |
+| Venue | ICML 2026 (main) |
+| Evidence class | Direct comparison |
+| First published | Not recorded |
+| Identifiers | Not recorded |
+| Artifact | Not found during the latest review |
+
+## Product configuration
+
+| Product | Role | Model | Product version |
+|---|---|---|---|
+| Codex CLI | baseline | not-reported | not-reported |
+
+## Task
+
+**Task.** Rich-format document generation and editing
+
+**Benchmark or scale.** RichDocBench and RichDocFuzz
+
+## Method
+
+HTML source editor, template retrieval, rendered-page review agent, iterative refinement, and edit-triggered context compression
+
+**Tags.** `retrieval` `context-reduction` `verifier-loop` `visual-review`
+
+## Reported evidence
+
+Render correctness 4.81 versus 4.39 for multi-pass Codex; human rank-1 rate 0.760 versus 0.140.
+
+| Control | Recorded value |
+|---|---|
+| Same model | unknown |
+| Same budget | unknown |
+| Evidence strength | high |
+| Claim type | mixed |
+| Comparison scope | product-level |
+| Source location | Main comparison table and review-refinement ablation |
+
+## Caveat
+
+Codex has slightly higher content alignment, and the task is document formatting rather than source-code repair.
+
+## Primary links
+
+- [Paper](https://openreview.net/forum?id=n7Ta0YEcgw)
+- No official artifact was found during the latest review.
+
+---
+
+Catalog ID: `formact-2026` · Metadata last reviewed with catalog release.
diff --git a/papers/icat-agent-2026.md b/papers/icat-agent-2026.md
new file mode 100644
index 0000000..3fb790a
--- /dev/null
+++ b/papers/icat-agent-2026.md
@@ -0,0 +1,60 @@
+
+
+[← Paper index](README.md) · [Home](../README.md)
+
+# icat-agent
+
+## Unlocking Model Potentials Through Adaptive Multi-Agent Scaffolding for Efficient Issue Resolution
+
+| Field | Value |
+|---|---|
+| Authors | Yang Chen, Aliya Ahmad, Yiheng Zhou, Reyhaneh Jabbarvand |
+| Venue | arXiv 2026 (preprint) |
+| Evidence class | Direct comparison |
+| First published | 2026-06-24 |
+| Identifiers | [arXiv:2606.25514](https://arxiv.org/abs/2606.25514) |
+| Artifact | Not found during the latest review |
+
+## Product configuration
+
+| Product | Role | Model | Product version |
+|---|---|---|---|
+| Claude Code | baseline | not-reported | not-reported |
+
+## Task
+
+**Task.** Repository issue resolution
+
+**Benchmark or scale.** SWE-bench Verified and SWE-bench Pro
+
+## Method
+
+Event-based agent communication, issue-quality routing, parallel patching and validation, and exploratory fallback
+
+**Tags.** `multi-agent` `dynamic-specialization` `parallelism` `test-feedback`
+
+## Reported evidence
+
+SWE-bench Verified improves 3.6 to 8.4 percentage points and SWE-bench Pro 6.3 to 18.5 points; average cost is $1.18 lower than multi-agent Claude Code.
+
+| Control | Recorded value |
+|---|---|
+| Same model | unknown |
+| Same budget | unknown |
+| Evidence strength | medium |
+| Claim type | mixed |
+| Comparison scope | product-level |
+| Source location | Abstract and SWE-bench Verified and Pro comparisons |
+
+## Caveat
+
+Exact model parity varies by reported configuration; preprint.
+
+## Primary links
+
+- [Paper](https://arxiv.org/abs/2606.25514)
+- No official artifact was found during the latest review.
+
+---
+
+Catalog ID: `icat-agent-2026` · Metadata last reviewed with catalog release.
diff --git a/papers/mlzero-2025.md b/papers/mlzero-2025.md
new file mode 100644
index 0000000..7e3bc54
--- /dev/null
+++ b/papers/mlzero-2025.md
@@ -0,0 +1,60 @@
+
+
+[← Paper index](README.md) · [Home](../README.md)
+
+# MLZero
+
+## MLZero: A Multi-Agent System for End-to-end Machine Learning Automation
+
+| Field | Value |
+|---|---|
+| Authors | Haoyang Fang, Boran Han, Nick Erickson, Xiyuan Zhang, Su Zhou, Anirudh Dagar, Jiani Zhang, Ali Caner Turkmen, Cuixiong Hu, Huzefa Rangwala, Ying Nian Wu, Bernie Wang, George Karypis |
+| Venue | NeurIPS 2025 (main) |
+| Evidence class | Direct comparison |
+| First published | 2025-05-20 |
+| Identifiers | [arXiv:2505.13941](https://arxiv.org/abs/2505.13941) |
+| Artifact | [Official artifact](https://github.com/autogluon/autogluon-assistant) |
+
+## Product configuration
+
+| Product | Role | Model | Product version |
+|---|---|---|---|
+| Codex CLI | baseline | GPT-4.1 and o4-mini | not-reported |
+
+## Task
+
+**Task.** End-to-end machine learning automation
+
+**Benchmark or scale.** MLE-Bench Lite and Multimodal AutoML Agent Benchmark
+
+## Method
+
+Hierarchical agents with multimodal perception, semantic and episodic memory, and a planning-coding-evaluation loop
+
+**Tags.** `multi-agent` `memory` `verifier-loop` `test-feedback`
+
+## Reported evidence
+
+MLZero led MLE-Bench Lite with six gold medals and achieved 0.92 success on its multimodal benchmark; one appendix setting reports 6.5% errors versus 26.9% for Codex CLI.
+
+| Control | Recorded value |
+|---|---|
+| Same model | unknown |
+| Same budget | unknown |
+| Evidence strength | medium |
+| Claim type | quality |
+| Comparison scope | product-level |
+| Source location | Abstract, main benchmark results, and appendix error analysis |
+
+## Caveat
+
+Results concern machine-learning automation and should not be generalized to all software-engineering tasks.
+
+## Primary links
+
+- [Paper](https://papers.neurips.cc/paper_files/paper/2025/hash/63ed15a46a143ff57484b38cd6b85d91-Abstract-Conference.html)
+- [Artifact](https://github.com/autogluon/autogluon-assistant)
+
+---
+
+Catalog ID: `mlzero-2025` · Metadata last reviewed with catalog release.
diff --git a/papers/prefactory-2026.md b/papers/prefactory-2026.md
new file mode 100644
index 0000000..529b2c8
--- /dev/null
+++ b/papers/prefactory-2026.md
@@ -0,0 +1,60 @@
+
+
+[← Paper index](README.md) · [Home](../README.md)
+
+# Prefactory
+
+## Prefactory: Automated Discovery and Application of Library-Adoption Refactorings
+
+| Field | Value |
+|---|---|
+| Authors | Islem Bouzenia, Michael Pradel |
+| Venue | arXiv 2026 (preprint) |
+| Evidence class | Direct comparison |
+| First published | 2026-07-19 |
+| Identifiers | [arXiv:2607.17211](https://arxiv.org/abs/2607.17211) |
+| Artifact | Not found during the latest review |
+
+## Product configuration
+
+| Product | Role | Model | Product version |
+|---|---|---|---|
+| Codex CLI | baseline | not-reported | not-reported |
+
+## Task
+
+**Task.** Discovery and application of library-adoption refactorings
+
+**Benchmark or scale.** PrefactoryBench
+
+## Method
+
+LLM-synthesized lexical and structural search heuristics, deterministic candidate ranking, targeted refactoring, and differential tests
+
+**Tags.** `deterministic-search` `static-analysis` `test-feedback` `verifier-loop`
+
+## Reported evidence
+
+File-level detection 75 versus Codex 35; function-level detection 56 versus 32; 40 of 56 candidates yield test-validated refactorings.
+
+| Control | Recorded value |
+|---|---|
+| Same model | unknown |
+| Same budget | unknown |
+| Evidence strength | medium |
+| Claim type | quality |
+| Comparison scope | product-level |
+| Source location | Abstract and PrefactoryBench detection and validation results |
+
+## Caveat
+
+The strongest comparison concerns candidate detection, not universal end-to-end task success.
+
+## Primary links
+
+- [Paper](https://arxiv.org/abs/2607.17211)
+- No official artifact was found during the latest review.
+
+---
+
+Catalog ID: `prefactory-2026` · Metadata last reviewed with catalog release.
diff --git a/papers/qlcoder-2026.md b/papers/qlcoder-2026.md
new file mode 100644
index 0000000..f8308d6
--- /dev/null
+++ b/papers/qlcoder-2026.md
@@ -0,0 +1,60 @@
+
+
+[← Paper index](README.md) · [Home](../README.md)
+
+# QLCoder
+
+## QLCoder: A Query Synthesizer For Static Analysis of Security Vulnerabilities
+
+| Field | Value |
+|---|---|
+| Authors | Claire Wang, Ziyang Li, Saikat Dutta, Mayur Naik |
+| Venue | ICLR 2026 (main) |
+| Evidence class | Direct comparison |
+| First published | 2025-11-11 |
+| Identifiers | [arXiv:2511.08462](https://arxiv.org/abs/2511.08462) |
+| Artifact | [Official artifact](https://github.com/neuralprogram/qlcoder) |
+
+## Product configuration
+
+| Product | Role | Model | Product version |
+|---|---|---|---|
+| Claude Code | baseline | Claude Sonnet 4 | 1.0.120 |
+
+## Task
+
+**Task.** CodeQL query synthesis from CVE metadata
+
+**Benchmark or scale.** 176 CVEs across 111 Java projects
+
+## Method
+
+CVE-grounded retrieval, AST guidance, CodeQL LSP tools, and iterative execution feedback
+
+**Tags.** `retrieval` `verifier-loop` `static-analysis`
+
+## Reported evidence
+
+Correct-query rate 53.4% versus 10% for Claude Code-only; F1 0.70 versus 0.048 for IRIS and 0.073 for CodeQL suites.
+
+| Control | Recorded value |
+|---|---|
+| Same model | unknown |
+| Same budget | unknown |
+| Evidence strength | high |
+| Claim type | quality |
+| Comparison scope | product-level |
+| Source location | Official abstract, main evaluation, and artifact paper environment |
+
+## Caveat
+
+Security-query synthesis is narrower than general software engineering.
+
+## Primary links
+
+- [Paper](https://proceedings.iclr.cc/paper_files/paper/2026/hash/2adf01ab15adde8820622f7f24bd516b-Abstract-Conference.html)
+- [Artifact](https://github.com/neuralprogram/qlcoder)
+
+---
+
+Catalog ID: `qlcoder-2026` · Metadata last reviewed with catalog release.
diff --git a/papers/repoomp-2026.md b/papers/repoomp-2026.md
new file mode 100644
index 0000000..d5ffd9d
--- /dev/null
+++ b/papers/repoomp-2026.md
@@ -0,0 +1,60 @@
+
+
+[← Paper index](README.md) · [Home](../README.md)
+
+# RepoOMP
+
+## RepoOMP: Repository-Aware Hotspot OpenMP Parallelization via Dependency-Aware Context Reduction
+
+| Field | Value |
+|---|---|
+| Authors | Yongjie Qian, Ke Gao, Zhibin Zhang, Shaohui Peng, Ling Li |
+| Venue | arXiv 2026 (preprint) |
+| Evidence class | Direct comparison |
+| First published | 2026-08-06 |
+| Identifiers | [arXiv:2608.05855](https://arxiv.org/abs/2608.05855) |
+| Artifact | [Official artifact](https://github.com/Qlalq/RepoOMP_Simplified) |
+
+## Product configuration
+
+| Product | Role | Model | Product version |
+|---|---|---|---|
+| Claude Code | baseline | not-reported | not-reported |
+
+## Task
+
+**Task.** Repository-aware OpenMP hotspot parallelization
+
+**Benchmark or scale.** 951 hotspots including 330 accepted real-world hotspots
+
+## Method
+
+Multi-granularity dependency graph, rule-or-LLM hotspot routing, reduced transformation context, and compile/workload/speedup validation
+
+**Tags.** `repository-graph` `context-reduction` `deterministic-search` `performance-feedback` `verifier-loop`
+
+## Reported evidence
+
+Across nine detailed kernels, speedup improves 18% to 28% and token cost falls 47% to 68% relative to unstructured Claude Code.
+
+| Control | Recorded value |
+|---|---|
+| Same model | unknown |
+| Same budget | unknown |
+| Evidence strength | medium |
+| Claim type | mixed |
+| Comparison scope | product-level |
+| Source location | Abstract and nine-kernel cross-backbone comparison |
+
+## Caveat
+
+Domain-specific high-performance-computing task and preprint status.
+
+## Primary links
+
+- [Paper](https://arxiv.org/abs/2608.05855)
+- [Artifact](https://github.com/Qlalq/RepoOMP_Simplified)
+
+---
+
+Catalog ID: `repoomp-2026` · Metadata last reviewed with catalog release.
diff --git a/papers/rpg-zerorepo-2026.md b/papers/rpg-zerorepo-2026.md
new file mode 100644
index 0000000..7307305
--- /dev/null
+++ b/papers/rpg-zerorepo-2026.md
@@ -0,0 +1,61 @@
+
+
+[← Paper index](README.md) · [Home](../README.md)
+
+# ZeroRepo
+
+## RPG: A Repository Planning Graph for Unified and Scalable Codebase Generation
+
+| Field | Value |
+|---|---|
+| Authors | Jane Luo, Xin Zhang, Steven Liu, Jie Wu, Jianfeng Liu, Yiming Huang, Yangyu Huang, Chengyu Yin, Ying Xin, Yuefeng Zhan, Hao Sun, Qi Chen, Scarlett Li, Mao Yang |
+| Venue | ICLR 2026 (main) |
+| Evidence class | Direct comparison |
+| First published | 2025-09-19 |
+| Identifiers | [arXiv:2509.16198](https://arxiv.org/abs/2509.16198) |
+| Artifact | [Official artifact](https://github.com/microsoft/RPG-ZeroRepo) |
+
+## Product configuration
+
+| Product | Role | Model | Product version |
+|---|---|---|---|
+| Claude Code | baseline | Claude 4 Sonnet | not-reported |
+| Codex CLI | baseline | o3-pro | not-reported |
+
+## Task
+
+**Task.** Repository-level codebase generation
+
+**Benchmark or scale.** RepoCraft
+
+## Method
+
+Persistent graph over features, functions, files, interfaces, and data flow with graph-guided generation and validation
+
+**Tags.** `structured-state` `repository-graph` `dependency-aware-planning` `test-feedback`
+
+## Reported evidence
+
+ZeroRepo coverage 81.5% versus Claude Code 54.2% and Codex 28.4%; pass rate 69.7% versus Claude Code 33.9%.
+
+| Control | Recorded value |
+|---|---|
+| Same model | no |
+| Same budget | unknown |
+| Evidence strength | high |
+| Claim type | quality |
+| Comparison scope | product-level |
+| Source location | Section 4.3 and RepoCraft main-results table |
+
+## Caveat
+
+ZeroRepo uses o3-mini while the Claude Code baseline uses Claude 4 Sonnet, so model and harness effects are mixed.
+
+## Primary links
+
+- [Paper](https://proceedings.iclr.cc/paper_files/paper/2026/file/9482f45fdd89aba9130bb04c44f788a9-Paper-Conference.pdf)
+- [Artifact](https://github.com/microsoft/RPG-ZeroRepo)
+
+---
+
+Catalog ID: `rpg-zerorepo-2026` · Metadata last reviewed with catalog release.
diff --git a/papers/rustprint-2026.md b/papers/rustprint-2026.md
new file mode 100644
index 0000000..debb26c
--- /dev/null
+++ b/papers/rustprint-2026.md
@@ -0,0 +1,60 @@
+
+
+[← Paper index](README.md) · [Home](../README.md)
+
+# RustPrint
+
+## RustPrint: Documentation-Guided Agentic Codebase Migration from C to Rust
+
+| Field | Value |
+|---|---|
+| Authors | Minh Le-Anh, Anh Nguyen Hoang, Bach Le, Nghi D. Q. Bui |
+| Venue | arXiv 2026 (preprint) |
+| Evidence class | Direct comparison |
+| First published | 2026-05-14 |
+| Identifiers | [arXiv:2605.14634](https://arxiv.org/abs/2605.14634) |
+| Artifact | Not found during the latest review |
+
+## Product configuration
+
+| Product | Role | Model | Product version |
+|---|---|---|---|
+| Claude Code | baseline | not-reported | not-reported |
+
+## Task
+
+**Task.** Whole-codebase migration from C to Rust
+
+**Benchmark or scale.** Eight real C repositories from 11K to 84K lines of code
+
+## Method
+
+Architecture documentation blueprint, module and data-flow planning, compile feedback, documentation mismatch repair, and source-test translation
+
+**Tags.** `structured-state` `dependency-aware-planning` `verifier-loop` `test-feedback`
+
+## Reported evidence
+
+In the reported Kimi-K2-Instruct setting, feature preservation is 93.26% versus 52.52% for agentic Claude Code; cross-evaluation test pass is 95.17% versus 79.85%.
+
+| Control | Recorded value |
+|---|---|
+| Same model | no |
+| Same budget | unknown |
+| Evidence strength | medium |
+| Claim type | quality |
+| Comparison scope | product-level |
+| Source location | Abstract and feature-preservation and cross-evaluation results |
+
+## Caveat
+
+The comparison mixes model and scaffold choices and is specific to C-to-Rust migration.
+
+## Primary links
+
+- [Paper](https://arxiv.org/abs/2605.14634)
+- No official artifact was found during the latest review.
+
+---
+
+Catalog ID: `rustprint-2026` · Metadata last reviewed with catalog release.
diff --git a/papers/scate-2026.md b/papers/scate-2026.md
new file mode 100644
index 0000000..3d8f614
--- /dev/null
+++ b/papers/scate-2026.md
@@ -0,0 +1,60 @@
+
+
+[← Paper index](README.md) · [Home](../README.md)
+
+# SCATE
+
+## SCATE: Learning to Supervise Coding Agents for Cost-Effective Test Generation
+
+| Field | Value |
+|---|---|
+| Authors | Sijia Gu, Noor Nashid, Ali Mesbah |
+| Venue | arXiv 2026 (preprint) |
+| Evidence class | Related method |
+| First published | 2026-07-09 |
+| Identifiers | [arXiv:2607.08983](https://arxiv.org/abs/2607.08983) |
+| Artifact | Not found during the latest review |
+
+## Product configuration
+
+| Product | Role | Model | Product version |
+|---|---|---|---|
+| Claude Code | evaluated | not-reported | not-reported |
+
+## Task
+
+**Task.** Automated test generation
+
+**Benchmark or scale.** Repository-level test-generation benchmark
+
+## Method
+
+Contextual-bandit supervisor that selects testing actions from current coverage and testability state
+
+**Tags.** `test-feedback` `verifier-loop`
+
+## Reported evidence
+
+Line coverage improves 32.3% and branch coverage 30.9% over agent-only Gemini CLI; adaptation is also evaluated with Claude Code.
+
+| Control | Recorded value |
+|---|---|
+| Same model | unknown |
+| Same budget | unknown |
+| Evidence strength | contextual |
+| Claim type | quality |
+| Comparison scope | component-level |
+| Source location | Abstract and cross-agent adaptation experiment |
+
+## Caveat
+
+The abstract does not provide an equally complete numeric head-to-head against Claude Code.
+
+## Primary links
+
+- [Paper](https://arxiv.org/abs/2607.08983)
+- No official artifact was found during the latest review.
+
+---
+
+Catalog ID: `scate-2026` · Metadata last reviewed with catalog release.
diff --git a/papers/terminal-bench-2-2026.md b/papers/terminal-bench-2-2026.md
new file mode 100644
index 0000000..8e60e2f
--- /dev/null
+++ b/papers/terminal-bench-2-2026.md
@@ -0,0 +1,61 @@
+
+
+[← Paper index](README.md) · [Home](../README.md)
+
+# Terminal-Bench 2.0
+
+## Terminal-Bench 2.0: Advancing Agentic Terminal Intelligence through Better Evaluation and Verification
+
+| Field | Value |
+|---|---|
+| Authors | Mike A. Merrill, Alexander G. Shaw, Nicholas Carlini, Boxuan Li, Harsh Raj, Ivan Bercovich, Lin Shi, Jeong Yeon Shin, Thomas Walshe, E. Kelly Buchanan, Junhong Shen, Guanghao Ye, Haowei Lin, Jason Poulos, Maoyu Wang, Marianna Nezhurina, Jenia Jitsev, Di Lu, Orfeas Menis Mastromichalakis, Zhiwei Xu, Zizhao Chen, Yue Liu, Robert Zhang, Leon Liangyu Chen, Anurag Kashyap, Jan-Lucas Uslu, Jeffrey Li, Jianbo Wu, Minghao Yan, Song Bian, Vedang Sharma, Ke Sun, Steven Dillmann, Akshay Anand, Andrew Lanpouthakoun, Bardia Koopah, Changran Hu, Etash Guha, Gabriel H. S. Dreiman, Jiacheng Zhu, Karl Krauth, Li Zhong, Niklas Muennighoff, Robert Amanfu, Shangyin Tan, Shreyas Pimpalgaonkar, Tushar Aggarwal, Xiangning Lin, Xin Lan, Xuandong Zhao, Yiqing Liang, Yuanli Wang, Zilong Wang, Changzhi Zhou, David Heineman, Hange Liu, Harsh Trivedi, John Yang, Junhong Lin, Manish Shetty, Michael Yang, Nabil Omi, Negin Raoof, Shanda Li, Terry Yue Zhuo, Wuwei Lin, Yiwei Dai, Yuxin Wang, Wenhao Chai, Shang Zhou, Dariush Wahdany, Ziyu She, Jiaming Hu, Zhikang Dong, Yuxuan Zhu, Sasha Cui, Ahson Saiyed, Arinbjörn Kolbeinsson, Jesse Hu, Christopher Michael Rytting, Ryan Marten, Yixin Wang, Alex Dimakis, Andy Konwinski, Ludwig Schmidt |
+| Venue | ICLR 2026 (main) |
+| Evidence class | Evaluation only |
+| First published | 2026-01-17 |
+| Identifiers | [arXiv:2601.11868](https://arxiv.org/abs/2601.11868) |
+| Artifact | [Official artifact](https://github.com/harbor-framework/terminal-bench-2) |
+
+## Product configuration
+
+| Product | Role | Model | Product version |
+|---|---|---|---|
+| Claude Code | evaluated | Claude Opus 4.5 | not-reported |
+| Codex CLI | evaluated | GPT-5.2 | not-reported |
+
+## Task
+
+**Task.** General terminal-agent evaluation
+
+**Benchmark or scale.** Terminal-Bench 2.0
+
+## Method
+
+Evaluation-only benchmark with stronger task verification and a neutral terminal scaffold
+
+**Tags.** `verifier-loop`
+
+## Reported evidence
+
+GPT-5.2 with Codex CLI scores 62.9; Claude Opus 4.5 scores 57.8 with Terminus 2 and 52.1 with Claude Code.
+
+| Control | Recorded value |
+|---|---|
+| Same model | unknown |
+| Same budget | unknown |
+| Evidence strength | high |
+| Claim type | mixed |
+| Comparison scope | benchmark-only |
+| Source location | Main agent and model results table |
+
+## Caveat
+
+Benchmark result; differences can reflect both model and harness, and no new method is proposed to beat the products.
+
+## Primary links
+
+- [Paper](https://proceedings.iclr.cc/paper_files/paper/2026/file/444a3737adaee10d86ad2ef5f74468e6-Paper-Conference.pdf)
+- [Artifact](https://github.com/harbor-framework/terminal-bench-2)
+
+---
+
+Catalog ID: `terminal-bench-2-2026` · Metadata last reviewed with catalog release.
diff --git a/papers/ticoder-2022.md b/papers/ticoder-2022.md
new file mode 100644
index 0000000..652d52e
--- /dev/null
+++ b/papers/ticoder-2022.md
@@ -0,0 +1,60 @@
+
+
+[← Paper index](README.md) · [Home](../README.md)
+
+# TiCoder
+
+## Interactive Code Generation via Test-Driven User-Intent Formalization
+
+| Field | Value |
+|---|---|
+| Authors | Shuvendu K. Lahiri, Sarah Fakhoury, Aaditya Naik, Georgios Sakkas, Saikat Chakraborty, Madanlal Musuvathi, Piali Choudhury, Curtis von Veh, Jeevana Priya Inala, Chenglong Wang, Jianfeng Gao |
+| Venue | AST at ICSE 2022 (workshop) |
+| Evidence class | Historical model |
+| First published | 2022-08-11 |
+| Identifiers | [arXiv:2208.05950](https://arxiv.org/abs/2208.05950) |
+| Artifact | [Official artifact](https://github.com/microsoft/TiCoder) |
+
+## Product configuration
+
+| Product | Role | Model | Product version |
+|---|---|---|---|
+| OpenAI Codex model | historical-baseline | OpenAI Codex | not-reported |
+
+## Task
+
+**Task.** Interactive function-level code generation
+
+**Benchmark or scale.** MBPP and HumanEval
+
+## Method
+
+Generate tests to formalize user intent, collect lightweight feedback, and rank or prune candidate programs
+
+**Tags.** `test-feedback` `verifier-loop`
+
+## Reported evidence
+
+With one to five simulated user queries, Codex pass@1 gains 22.49 to 37.71 points on MBPP and 24.79 to 53.98 points on HumanEval.
+
+| Control | Recorded value |
+|---|---|
+| Same model | yes |
+| Same budget | no |
+| Evidence strength | high |
+| Claim type | quality |
+| Comparison scope | historical-model |
+| Source location | MBPP and HumanEval evaluation section |
+
+## Caveat
+
+This is the historical OpenAI Codex model, not the modern Codex CLI product.
+
+## Primary links
+
+- [Paper](https://arxiv.org/abs/2208.05950)
+- [Artifact](https://github.com/microsoft/TiCoder)
+
+---
+
+Catalog ID: `ticoder-2022` · Metadata last reviewed with catalog release.
diff --git a/papers/webdesigniter-2026.md b/papers/webdesigniter-2026.md
new file mode 100644
index 0000000..6467eb2
--- /dev/null
+++ b/papers/webdesigniter-2026.md
@@ -0,0 +1,61 @@
+
+
+[← Paper index](README.md) · [Home](../README.md)
+
+# WebDesignIter
+
+## WebDesignIter: Co-Evolving Design Knowledge for Repository-Level Front-End Code Generation
+
+| Field | Value |
+|---|---|
+| Authors | Zheng Pei, Mingwei Liu, Zhenxi Chen, Zihao Wang, Yanlin Wang |
+| Venue | arXiv 2026 (preprint) |
+| Evidence class | Direct comparison |
+| First published | 2026-07-12 |
+| Identifiers | [arXiv:2607.10621](https://arxiv.org/abs/2607.10621) |
+| Artifact | [Official artifact](https://github.com/SYSUSELab/WebDesignIter) |
+
+## Product configuration
+
+| Product | Role | Model | Product version |
+|---|---|---|---|
+| Claude Code | baseline | not-reported | not-reported |
+| Codex CLI | baseline | not-reported | not-reported |
+
+## Task
+
+**Task.** Repository-level front-end code generation
+
+**Benchmark or scale.** Web-Bench
+
+## Method
+
+Persistent architecture and design knowledge graph, design-informed planning, targeted patches, and sandbox repair
+
+**Tags.** `structured-state` `repository-graph` `retrieval` `test-feedback`
+
+## Reported evidence
+
+Average Pass@2 improves by 9.55 percentage points over existing baselines; removing design knowledge reduces Pass@1 by 11.4 points.
+
+| Control | Recorded value |
+|---|---|
+| Same model | unknown |
+| Same budget | unknown |
+| Evidence strength | medium |
+| Claim type | quality |
+| Comparison scope | product-level |
+| Source location | Abstract, main Web-Bench comparison, and design-knowledge ablation |
+
+## Caveat
+
+Front-end-specific benchmark and preprint status.
+
+## Primary links
+
+- [Paper](https://arxiv.org/abs/2607.10621)
+- [Artifact](https://github.com/SYSUSELab/WebDesignIter)
+
+---
+
+Catalog ID: `webdesigniter-2026` · Metadata last reviewed with catalog release.
diff --git a/requirements-dev.txt b/requirements-dev.txt
index ce63a7e..8bd350f 100644
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@@ -1,4 +1,5 @@
PyYAML==6.0.2
+certifi==2026.7.22
jsonschema==4.25.0
pytest==8.4.1
ruff==0.12.9
diff --git a/scripts/__init__.py b/scripts/__init__.py
new file mode 100644
index 0000000..a781929
--- /dev/null
+++ b/scripts/__init__.py
@@ -0,0 +1 @@
+"""Catalog maintenance scripts."""
diff --git a/scripts/build_readme.py b/scripts/build_readme.py
index 15349af..8ccf92d 100644
--- a/scripts/build_readme.py
+++ b/scripts/build_readme.py
@@ -1,11 +1,13 @@
#!/usr/bin/env python3
-"""Generate catalog tables in README.md from data/papers.yaml."""
+"""Build README tables, paper dossiers, research views, and JSON export."""
from __future__ import annotations
import argparse
+import json
import re
import sys
+from collections import defaultdict
from pathlib import Path
import yaml
@@ -13,6 +15,9 @@
ROOT = Path(__file__).resolve().parents[1]
CATALOG_PATH = ROOT / "data" / "papers.yaml"
README_PATH = ROOT / "README.md"
+README_ZH_PATH = ROOT / "README.zh-CN.md"
+
+GENERATED_NOTICE = ""
SECTIONS = {
"direct": ("", ""),
@@ -27,18 +32,47 @@
),
}
+STATS_MARKERS = ("", "")
+
PRODUCT_LABELS = {
"claude-code": "Claude Code",
"codex-cli": "Codex CLI",
"openai-codex-model": "OpenAI Codex model",
}
+CLASS_LABELS = {
+ "direct": "Direct comparison",
+ "related": "Related method",
+ "evaluation": "Evaluation only",
+ "historical": "Historical model",
+}
+
+CLASS_LABELS_ZH = {
+ "direct": "直接对比",
+ "related": "相关方法",
+ "evaluation": "仅评测",
+ "historical": "旧模型",
+}
+
CONTROL_LABELS = {
"yes": "same",
"no": "different",
"unknown": "unknown",
}
+TABLE_HEADERS = {
+ "en": [
+ "System / paper",
+ "Venue",
+ "Product baseline",
+ "Task",
+ "What changed",
+ "Reported evidence",
+ "Controls",
+ ],
+ "zh": ["系统 / 论文", "会议", "产品 baseline", "任务", "新增方法", "论文结果", "可比性"],
+}
+
def load_catalog() -> dict:
with CATALOG_PATH.open(encoding="utf-8") as handle:
@@ -49,13 +83,20 @@ def escape_cell(value: object) -> str:
return str(value).replace("|", "\\|").replace("\n", " ").strip()
-def format_paper(paper: dict) -> str:
- title = escape_cell(paper["title"])
- paper_link = f"[{title}]({paper['paper_url']})"
- artifact_url = paper.get("artifact_url")
- if artifact_url:
- paper_link += f"
[artifact]({artifact_url})"
- return paper_link
+def slug(value: str) -> str:
+ return re.sub(r"[^a-z0-9]+", "-", value.lower()).strip("-")
+
+
+def paper_detail_link(paper: dict, root_prefix: str = "") -> str:
+ return f"{root_prefix}papers/{paper['id']}.md"
+
+
+def format_system_paper(paper: dict, root_prefix: str = "") -> str:
+ system = escape_cell(paper["system"])
+ links = [f"[paper]({paper['paper_url']})"]
+ if paper.get("artifact_url"):
+ links.append(f"[artifact]({paper['artifact_url']})")
+ return f"**[{system}]({paper_detail_link(paper, root_prefix)})**
" + " · ".join(links)
def format_products(paper: dict) -> str:
@@ -94,53 +135,417 @@ def format_controls(paper: dict) -> str:
return f"model: {model}
budget: {budget}
evidence: {strength}"
-def generate_table(papers: list[dict]) -> str:
+def generate_table(papers: list[dict], language: str = "en", root_prefix: str = "") -> str:
if not papers:
return "_No reviewed papers in this class yet._"
- header = (
- "| Paper | Venue | Product baseline | Task / benchmark | "
- "Method | Reported result | Controls |\n"
- "|---|---|---|---|---|---|---|"
- )
+ headers = TABLE_HEADERS[language]
+ header = "| " + " | ".join(headers) + " |\n" + "|" + "---|" * len(headers)
rows = []
- for paper in sorted(papers, key=lambda item: (-item["year"], item["title"].lower())):
- task = (
- f"{escape_cell(paper['task']['summary'])}"
- f"
{escape_cell(paper['task']['benchmark'])}"
+ for paper in sorted(papers, key=lambda item: (-item["year"], item["system"].lower())):
+ row = (
+ "| "
+ + " | ".join(
+ [
+ format_system_paper(paper, root_prefix),
+ format_venue(paper),
+ format_products(paper),
+ escape_cell(paper["task"]["summary"]),
+ format_method(paper),
+ escape_cell(paper["evidence"]["result"]),
+ format_controls(paper),
+ ]
+ )
+ + " |"
)
- row = "| " + " | ".join(
- [
- format_paper(paper),
- format_venue(paper),
- format_products(paper),
- task,
- format_method(paper),
- escape_cell(paper["evidence"]["result"]),
- format_controls(paper),
- ]
- ) + " |"
rows.append(row)
return header + "\n" + "\n".join(rows)
-def replace_section(readme: str, start: str, end: str, content: str) -> str:
+def generate_stats(catalog: dict) -> str:
+ papers = catalog["papers"]
+ direct = sum(paper["classification"] == "direct" for paper in papers)
+ artifacts = sum(paper["artifact_status"] == "official" for paper in papers)
+ venues = len({paper["venue"] for paper in papers})
+ reviewed = catalog["reviewed_at"]
+ badges = [
+ ("papers", len(papers), "0f766e"),
+ ("direct comparisons", direct, "dc6b46"),
+ ("official artifacts", artifacts, "2563eb"),
+ ("venues", venues, "7c3aed"),
+ ("reviewed", reviewed, "475569"),
+ ]
+ images = []
+ for label, value, color in badges:
+ encoded_label = str(label).replace(" ", "%20")
+ encoded_value = str(value).replace("-", "--")
+ images.append(
+ f'![{label}: {value}]()
'
+ )
+ return '\n ' + "\n ".join(images) + "\n
"
+
+
+def replace_section(text: str, start: str, end: str, content: str) -> str:
pattern = re.compile(re.escape(start) + r".*?" + re.escape(end), re.DOTALL)
replacement = f"{start}\n{content}\n{end}"
- updated, count = pattern.subn(lambda _: replacement, readme)
+ updated, count = pattern.subn(lambda _: replacement, text)
if count != 1:
- raise RuntimeError(f"Expected one README section for {start}, found {count}")
+ raise RuntimeError(f"Expected one generated section for {start}, found {count}")
return updated
-def render_readme() -> str:
- catalog = load_catalog()
- readme = README_PATH.read_text(encoding="utf-8")
+def render_marked_readme(path: Path, catalog: dict, language: str) -> str:
+ text = path.read_text(encoding="utf-8")
+ text = replace_section(text, *STATS_MARKERS, generate_stats(catalog))
papers = catalog["papers"]
for classification, (start, end) in SECTIONS.items():
selected = [paper for paper in papers if paper["classification"] == classification]
- readme = replace_section(readme, start, end, generate_table(selected))
- return readme
+ text = replace_section(text, start, end, generate_table(selected, language))
+ return text
+
+
+def short_authors(paper: dict) -> str:
+ authors = paper["authors"]
+ if len(authors) <= 3:
+ return ", ".join(authors)
+ return f"{', '.join(authors[:3])}, et al. ({len(authors)} authors)"
+
+
+def artifact_label(paper: dict) -> str:
+ status = paper["artifact_status"]
+ if status == "not-found":
+ return "Not found during the latest review"
+ label = "Official artifact" if status == "official" else "Community artifact"
+ return f"[{label}]({paper['artifact_url']})"
+
+
+def render_paper_page(paper: dict) -> str:
+ evidence = paper["evidence"]
+ identifiers = []
+ if paper.get("arxiv_id"):
+ identifiers.append(
+ f"[arXiv:{paper['arxiv_id']}](https://arxiv.org/abs/{paper['arxiv_id']})"
+ )
+ if paper.get("doi"):
+ identifiers.append(f"[DOI:{paper['doi']}](https://doi.org/{paper['doi']})")
+ if not identifiers:
+ identifiers.append("Not recorded")
+
+ product_rows = []
+ for product in paper["products"]:
+ product_rows.append(
+ "| "
+ + " | ".join(
+ [
+ PRODUCT_LABELS[product["product"]],
+ product["role"],
+ escape_cell(product["model"]),
+ escape_cell(product["version"]),
+ ]
+ )
+ + " |"
+ )
+
+ tags = " ".join(f"`{tag}`" for tag in paper["method"]["tags"]) or "None"
+ published = paper.get("published_at", "Not recorded")
+ artifact_item = (
+ f"- [Artifact]({paper['artifact_url']})"
+ if paper.get("artifact_url")
+ else "- No official artifact was found during the latest review."
+ )
+ return f"""{GENERATED_NOTICE}
+
+[← Paper index](README.md) · [Home](../README.md)
+
+# {paper["system"]}
+
+## {paper["title"]}
+
+| Field | Value |
+|---|---|
+| Authors | {escape_cell(", ".join(paper["authors"]))} |
+| Venue | {escape_cell(paper["venue"])} {paper["year"]} ({paper["publication_status"]}) |
+| Evidence class | {CLASS_LABELS[paper["classification"]]} |
+| First published | {published} |
+| Identifiers | {" · ".join(identifiers)} |
+| Artifact | {artifact_label(paper)} |
+
+## Product configuration
+
+| Product | Role | Model | Product version |
+|---|---|---|---|
+{chr(10).join(product_rows)}
+
+## Task
+
+**Task.** {paper["task"]["summary"]}
+
+**Benchmark or scale.** {paper["task"]["benchmark"]}
+
+## Method
+
+{paper["method"]["summary"]}
+
+**Tags.** {tags}
+
+## Reported evidence
+
+{evidence["result"]}
+
+| Control | Recorded value |
+|---|---|
+| Same model | {evidence["same_model"]} |
+| Same budget | {evidence["same_budget"]} |
+| Evidence strength | {evidence["strength"]} |
+| Claim type | {evidence["claim_type"]} |
+| Comparison scope | {evidence["comparison_scope"]} |
+| Source location | {escape_cell(evidence["source_location"])} |
+
+## Caveat
+
+{evidence["caveats"]}
+
+## Primary links
+
+- [Paper]({paper["paper_url"]})
+{artifact_item}
+
+---
+
+Catalog ID: `{paper["id"]}` · Metadata last reviewed with catalog release.
+"""
+
+
+def render_paper_index(catalog: dict) -> str:
+ rows = []
+ for paper in sorted(catalog["papers"], key=lambda item: (-item["year"], item["system"])):
+ rows.append(
+ "| "
+ + " | ".join(
+ [
+ f"[{escape_cell(paper['system'])}]({paper['id']}.md)",
+ escape_cell(paper["title"]),
+ f"{paper['venue']} {paper['year']}",
+ CLASS_LABELS[paper["classification"]],
+ format_products(paper),
+ artifact_label(paper),
+ ]
+ )
+ + " |"
+ )
+ return f"""{GENERATED_NOTICE}
+
+[← Home](../README.md) · [Browse by product](../views/by-product.md)
+
+# Paper dossiers
+
+Every catalog entry has a generated evidence dossier. The YAML catalog remains the source of truth.
+
+| System | Paper | Venue | Evidence class | Product | Artifact |
+|---|---|---|---|---|---|
+{chr(10).join(rows)}
+"""
+
+
+def view_paper_link(paper: dict) -> str:
+ return f"[{escape_cell(paper['system'])}](../papers/{paper['id']}.md)"
+
+
+def render_views_index() -> str:
+ return f"""{GENERATED_NOTICE}
+
+[← Home](../README.md)
+
+# Research views
+
+- [By product](by-product.md) — Claude Code and Codex CLI separately.
+- [By method](by-method.md) — repository graphs, retrieval, verification, orchestration, and more.
+- [Comparison fairness](fair-comparisons.md) — same-model and same-budget controls.
+- [By venue](by-venue.md) — conferences, workshops, and preprints.
+- [All paper dossiers](../papers/README.md) — one evidence page per paper.
+"""
+
+
+def render_by_product(catalog: dict) -> str:
+ sections = []
+ for product_id in ["claude-code", "codex-cli", "openai-codex-model"]:
+ papers = [
+ paper
+ for paper in catalog["papers"]
+ if any(item["product"] == product_id for item in paper["products"])
+ ]
+ rows = []
+ for paper in sorted(papers, key=lambda item: (-item["year"], item["system"])):
+ rows.append(
+ f"| {view_paper_link(paper)} | {paper['venue']} {paper['year']} | "
+ f"{CLASS_LABELS[paper['classification']]} | "
+ f"{escape_cell(paper['task']['summary'])} | {paper['evidence']['strength']} |"
+ )
+ sections.append(
+ f"## {PRODUCT_LABELS[product_id]} ({len(papers)})\n\n"
+ "| System | Venue | Evidence class | Task | Strength |\n"
+ "|---|---|---|---|---|\n" + "\n".join(rows)
+ )
+ return f"""{GENERATED_NOTICE}
+
+[← Research views](README.md) · [Home](../README.md)
+
+# Papers by product
+
+Product means the complete agent or harness evaluated by the paper,
+not merely a model from the same vendor.
+
+{chr(10).join(sections)}
+"""
+
+
+def render_by_method(catalog: dict) -> str:
+ grouped: dict[str, list[dict]] = defaultdict(list)
+ for paper in catalog["papers"]:
+ for tag in paper["method"]["tags"]:
+ grouped[tag].append(paper)
+
+ sections = []
+ for tag, papers in sorted(grouped.items(), key=lambda item: (-len(item[1]), item[0])):
+ rows = []
+ for paper in sorted(papers, key=lambda item: (-item["year"], item["system"])):
+ rows.append(
+ f"| {view_paper_link(paper)} | {paper['venue']} {paper['year']} | "
+ f"{escape_cell(paper['method']['summary'])} |"
+ )
+ sections.append(
+ f"## `{tag}` ({len(papers)})\n\n"
+ "| System | Venue | Method |\n|---|---|---|\n" + "\n".join(rows)
+ )
+ return f"""{GENERATED_NOTICE}
+
+[← Research views](README.md) · [Home](../README.md)
+
+# Papers by method
+
+Tags describe the intervention added around or instead of the production coding-agent baseline.
+
+{chr(10).join(sections)}
+"""
+
+
+def render_fair_comparisons(catalog: dict) -> str:
+ direct = [paper for paper in catalog["papers"] if paper["classification"] == "direct"]
+ groups = [
+ (
+ "Same model and same budget",
+ [
+ paper
+ for paper in direct
+ if paper["evidence"]["same_model"] == "yes"
+ and paper["evidence"]["same_budget"] == "yes"
+ ],
+ ),
+ (
+ "Known model or budget mismatch",
+ [
+ paper
+ for paper in direct
+ if "no" in {paper["evidence"]["same_model"], paper["evidence"]["same_budget"]}
+ ],
+ ),
+ (
+ "Control parity not fully reported",
+ [
+ paper
+ for paper in direct
+ if "unknown" in {paper["evidence"]["same_model"], paper["evidence"]["same_budget"]}
+ and "no" not in {paper["evidence"]["same_model"], paper["evidence"]["same_budget"]}
+ ],
+ ),
+ ]
+ sections = []
+ for title, papers in groups:
+ rows = []
+ for paper in sorted(papers, key=lambda item: (-item["year"], item["system"])):
+ evidence = paper["evidence"]
+ rows.append(
+ f"| {view_paper_link(paper)} | {evidence['same_model']} | "
+ f"{evidence['same_budget']} | {evidence['strength']} | "
+ f"{escape_cell(evidence['caveats'])} |"
+ )
+ table = "_No papers in this group._"
+ if rows:
+ table = (
+ "| System | Same model | Same budget | Strength | Caveat |\n"
+ "|---|---|---|---|---|\n" + "\n".join(rows)
+ )
+ sections.append(f"## {title} ({len(papers)})\n\n{table}")
+ return f"""{GENERATED_NOTICE}
+
+[← Research views](README.md) · [Home](../README.md)
+
+# Comparison fairness
+
+“Direct comparison” means a numeric head-to-head was reported.
+It does not mean the model, budget, tools, or product policy were held fixed.
+
+{chr(10).join(sections)}
+"""
+
+
+def render_by_venue(catalog: dict) -> str:
+ grouped: dict[str, list[dict]] = defaultdict(list)
+ for paper in catalog["papers"]:
+ grouped[paper["venue"]].append(paper)
+ sections = []
+ for venue, papers in sorted(grouped.items(), key=lambda item: (-len(item[1]), item[0])):
+ rows = [
+ f"| {view_paper_link(paper)} | {paper['year']} | "
+ f"{paper['publication_status']} | {CLASS_LABELS[paper['classification']]} |"
+ for paper in sorted(papers, key=lambda item: (-item["year"], item["system"]))
+ ]
+ sections.append(
+ f"## {venue} ({len(papers)})\n\n"
+ "| System | Year | Status | Evidence class |\n|---|---|---|---|\n" + "\n".join(rows)
+ )
+ return f"""{GENERATED_NOTICE}
+
+[← Research views](README.md) · [Home](../README.md)
+
+# Papers by venue
+
+Main-conference, workshop, benchmark-track, and preprint status remain explicit.
+
+{chr(10).join(sections)}
+"""
+
+
+def render_all() -> dict[Path, str]:
+ catalog = load_catalog()
+ outputs = {
+ README_PATH: render_marked_readme(README_PATH, catalog, "en"),
+ README_ZH_PATH: render_marked_readme(README_ZH_PATH, catalog, "zh"),
+ ROOT / "data" / "papers.json": json.dumps(catalog, indent=2, ensure_ascii=False) + "\n",
+ ROOT / "papers" / "README.md": render_paper_index(catalog),
+ ROOT / "views" / "README.md": render_views_index(),
+ ROOT / "views" / "by-product.md": render_by_product(catalog),
+ ROOT / "views" / "by-method.md": render_by_method(catalog),
+ ROOT / "views" / "fair-comparisons.md": render_fair_comparisons(catalog),
+ ROOT / "views" / "by-venue.md": render_by_venue(catalog),
+ }
+ for paper in catalog["papers"]:
+ outputs[ROOT / "papers" / f"{paper['id']}.md"] = render_paper_page(paper)
+ return outputs
+
+
+def generated_extras(expected: set[Path]) -> list[Path]:
+ extras = []
+ for directory in [ROOT / "papers", ROOT / "views"]:
+ if not directory.exists():
+ continue
+ for path in directory.glob("*.md"):
+ if path not in expected and path.read_text(encoding="utf-8").startswith(
+ GENERATED_NOTICE
+ ):
+ extras.append(path)
+ return extras
def main() -> int:
@@ -148,21 +553,38 @@ def main() -> int:
parser.add_argument(
"--check",
action="store_true",
- help="Fail when README.md does not match the generated catalog tables.",
+ help="Fail when any generated catalog file is missing or stale.",
)
args = parser.parse_args()
- rendered = render_readme()
- current = README_PATH.read_text(encoding="utf-8")
+ outputs = render_all()
if args.check:
- if rendered != current:
- print("README.md is stale; run `python scripts/build_readme.py`.", file=sys.stderr)
+ stale = [
+ path
+ for path, content in outputs.items()
+ if not path.exists() or path.read_text(encoding="utf-8") != content
+ ]
+ extras = generated_extras(set(outputs))
+ if stale or extras:
+ for path in stale:
+ print(f"stale or missing: {path.relative_to(ROOT)}", file=sys.stderr)
+ for path in extras:
+ print(f"unexpected generated file: {path.relative_to(ROOT)}", file=sys.stderr)
+ print("Run `python scripts/build_readme.py`.", file=sys.stderr)
return 1
- print("README.md catalog tables are current.")
+ print(f"All {len(outputs)} generated catalog files are current.")
return 0
- README_PATH.write_text(rendered, encoding="utf-8")
- print(f"Updated {README_PATH.relative_to(ROOT)}")
+ for path, content in outputs.items():
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_text(content, encoding="utf-8")
+ print(f"updated {path.relative_to(ROOT)}")
+ extras = generated_extras(set(outputs))
+ if extras:
+ print("Obsolete generated files were not deleted automatically:", file=sys.stderr)
+ for path in extras:
+ print(f"- {path.relative_to(ROOT)}", file=sys.stderr)
+ return 1
return 0
diff --git a/scripts/check_links.py b/scripts/check_links.py
new file mode 100644
index 0000000..4b87593
--- /dev/null
+++ b/scripts/check_links.py
@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+"""Fail when a relative Markdown or HTML link points to a missing local file."""
+
+from __future__ import annotations
+
+import re
+import sys
+from pathlib import Path
+from urllib.parse import unquote
+
+ROOT = Path(__file__).resolve().parents[1]
+MARKDOWN_LINK = re.compile(r"\[[^\]]*\]\(([^)]+)\)")
+HTML_LINK = re.compile(r'(?:href|src)="([^"]+)"')
+EXCLUDED_PARTS = {".git", ".venv"}
+
+
+def markdown_files() -> list[Path]:
+ return sorted(
+ path
+ for path in ROOT.rglob("*.md")
+ if not EXCLUDED_PARTS.intersection(path.relative_to(ROOT).parts)
+ )
+
+
+def normalize_target(raw_target: str) -> str:
+ target = raw_target.strip()
+ if target.startswith("<") and ">" in target:
+ return target[1 : target.index(">")]
+ return target.split(maxsplit=1)[0]
+
+
+def missing_links(path: Path) -> list[str]:
+ text = path.read_text(encoding="utf-8")
+ targets = MARKDOWN_LINK.findall(text) + HTML_LINK.findall(text)
+ missing = []
+ for raw_target in targets:
+ target = normalize_target(raw_target)
+ if target.startswith(("http://", "https://", "mailto:", "#")):
+ continue
+ local_part = unquote(target.split("#", 1)[0])
+ if not local_part:
+ continue
+ resolved = (path.parent / local_part).resolve()
+ if not resolved.exists():
+ missing.append(target)
+ return missing
+
+
+def main() -> int:
+ files = markdown_files()
+ failures = [(path, target) for path in files for target in missing_links(path)]
+ if failures:
+ print("Broken internal links:", file=sys.stderr)
+ for path, target in failures:
+ print(f"- {path.relative_to(ROOT)} -> {target}", file=sys.stderr)
+ return 1
+ print(f"Validated internal links across {len(files)} Markdown files.")
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/scripts/discover_candidates.py b/scripts/discover_candidates.py
new file mode 100644
index 0000000..fcf864a
--- /dev/null
+++ b/scripts/discover_candidates.py
@@ -0,0 +1,154 @@
+#!/usr/bin/env python3
+"""Find recent arXiv papers that mention Claude Code or Codex CLI."""
+
+from __future__ import annotations
+
+import argparse
+import os
+import re
+import ssl
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+from urllib.parse import urlencode
+from urllib.request import Request, urlopen
+from xml.etree import ElementTree
+
+import certifi
+import yaml
+
+ROOT = Path(__file__).resolve().parents[1]
+CATALOG_PATH = ROOT / "data" / "papers.yaml"
+ARXIV_API = "https://export.arxiv.org/api/query"
+ATOM = "{http://www.w3.org/2005/Atom}"
+
+
+def normalize_text(value: str) -> str:
+ return re.sub(r"\s+", " ", value).strip()
+
+
+def parse_feed(payload: bytes) -> list[dict[str, object]]:
+ root = ElementTree.fromstring(payload)
+ papers = []
+ for entry in root.findall(f"{ATOM}entry"):
+ identifier = entry.findtext(f"{ATOM}id", "").rsplit("/", 1)[-1]
+ arxiv_id = re.sub(r"v\d+$", "", identifier)
+ authors = [
+ normalize_text(author.findtext(f"{ATOM}name", ""))
+ for author in entry.findall(f"{ATOM}author")
+ ]
+ papers.append(
+ {
+ "arxiv_id": arxiv_id,
+ "title": normalize_text(entry.findtext(f"{ATOM}title", "")),
+ "summary": normalize_text(entry.findtext(f"{ATOM}summary", "")),
+ "authors": authors,
+ "published": entry.findtext(f"{ATOM}published", ""),
+ "url": f"https://arxiv.org/abs/{arxiv_id}",
+ }
+ )
+ return papers
+
+
+def fetch_candidates(max_results: int) -> list[dict[str, object]]:
+ query = 'all:"Claude Code" OR all:"Codex CLI"'
+ parameters = urlencode(
+ {
+ "search_query": query,
+ "start": 0,
+ "max_results": max_results,
+ "sortBy": "submittedDate",
+ "sortOrder": "descending",
+ }
+ )
+ request = Request(
+ f"{ARXIV_API}?{parameters}",
+ headers={"User-Agent": "awesome-claude-code-codex-papers/0.1"},
+ )
+ tls_context = ssl.create_default_context(cafile=certifi.where())
+ with urlopen(request, timeout=30, context=tls_context) as response:
+ return parse_feed(response.read())
+
+
+def existing_arxiv_ids() -> set[str]:
+ with CATALOG_PATH.open(encoding="utf-8") as handle:
+ catalog = yaml.safe_load(handle)
+ return {paper["arxiv_id"] for paper in catalog["papers"] if paper.get("arxiv_id")}
+
+
+def recent_untracked(
+ papers: list[dict[str, object]], known: set[str], days: int
+) -> list[dict[str, object]]:
+ cutoff = datetime.now(timezone.utc) - timedelta(days=days)
+ selected = []
+ for paper in papers:
+ published = datetime.fromisoformat(str(paper["published"]).replace("Z", "+00:00"))
+ if paper["arxiv_id"] not in known and published >= cutoff:
+ selected.append(paper)
+ return selected
+
+
+def render_report(papers: list[dict[str, object]], days: int, total_candidates: int) -> str:
+ header = (
+ "# Weekly paper candidates\n\n"
+ f"Automated arXiv discovery for exact mentions of `Claude Code` or `Codex CLI` "
+ f"during the last {days} days. Candidates require manual product-level evidence review.\n\n"
+ f"Showing the newest {len(papers)} of {total_candidates} untracked candidate(s).\n\n"
+ )
+ if not papers:
+ return header + "No untracked candidates were found.\n"
+
+ sections = []
+ for paper in papers:
+ authors = ", ".join(paper["authors"][:5])
+ if len(paper["authors"]) > 5:
+ authors += ", et al."
+ published = str(paper["published"])[:10]
+ summary = str(paper["summary"])
+ if len(summary) > 500:
+ summary = summary[:497].rstrip() + "..."
+ sections.append(
+ f"## [{paper['title']}]({paper['url']})\n\n"
+ f"- arXiv: `{paper['arxiv_id']}`\n"
+ f"- Published: {published}\n"
+ f"- Authors: {authors}\n\n"
+ f"{summary}\n\n"
+ "Review checklist: product actually executed · evidence class · model/version · "
+ "budget · result location · official artifact\n"
+ )
+ return header + "\n".join(sections)
+
+
+def write_github_output(count: int) -> None:
+ output_path = os.environ.get("GITHUB_OUTPUT")
+ if not output_path:
+ raise RuntimeError("GITHUB_OUTPUT is not set")
+ with Path(output_path).open("a", encoding="utf-8") as handle:
+ handle.write(f"count={count}\n")
+
+
+def main() -> int:
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--days", type=int, default=30)
+ parser.add_argument("--max-results", type=int, default=100)
+ parser.add_argument("--limit", type=int, default=25)
+ parser.add_argument("--output", type=Path, default=Path("candidate-report.md"))
+ parser.add_argument("--github-output", action="store_true")
+ args = parser.parse_args()
+
+ candidates = recent_untracked(
+ fetch_candidates(args.max_results), existing_arxiv_ids(), args.days
+ )
+ candidates.sort(key=lambda paper: str(paper["published"]), reverse=True)
+ visible = candidates[: args.limit]
+ args.output.write_text(render_report(visible, args.days, len(candidates)), encoding="utf-8")
+ if args.github_output:
+ write_github_output(len(candidates))
+ print(
+ f"Found {len(candidates)} untracked candidate(s); "
+ f"wrote the newest {len(visible)} to {args.output}."
+ )
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/scripts/validate.py b/scripts/validate.py
index 0fa63ee..3a03bea 100644
--- a/scripts/validate.py
+++ b/scripts/validate.py
@@ -48,10 +48,18 @@ def invariant_errors(catalog: dict) -> list[str]:
errors.append(f"duplicate paper id: {duplicate}")
for duplicate in duplicate_values([paper.get("title", "") for paper in papers]):
errors.append(f"duplicate paper title: {duplicate}")
+ arxiv_ids = [paper["arxiv_id"] for paper in papers if paper.get("arxiv_id")]
+ for duplicate in duplicate_values(arxiv_ids):
+ errors.append(f"duplicate arXiv id: {duplicate}")
+ dois = [paper["doi"].lower() for paper in papers if paper.get("doi")]
+ for duplicate in duplicate_values(dois):
+ errors.append(f"duplicate DOI: {duplicate}")
for paper in papers:
paper_id = paper.get("id", "")
classification = paper.get("classification")
+ artifact_status = paper.get("artifact_status")
+ artifact_url = paper.get("artifact_url")
products = paper.get("products", [])
product_ids = [item.get("product") for item in products]
@@ -76,6 +84,24 @@ def invariant_errors(catalog: dict) -> list[str]:
if classification != "historical" and has_historical_product:
errors.append(f"{paper_id}: historical products must use historical classification")
+ if artifact_status in {"official", "community"} and not artifact_url:
+ errors.append(f"{paper_id}: {artifact_status} artifact requires artifact_url")
+ if artifact_status == "not-found" and artifact_url:
+ errors.append(f"{paper_id}: not-found artifact cannot include artifact_url")
+
+ comparison_scope = paper.get("evidence", {}).get("comparison_scope")
+ expected_scopes = {
+ "direct": {"product-level", "configuration-ablation"},
+ "related": {"component-level", "configuration-ablation"},
+ "evaluation": {"benchmark-only"},
+ "historical": {"historical-model"},
+ }
+ if comparison_scope not in expected_scopes.get(classification, set()):
+ errors.append(
+ f"{paper_id}: comparison scope {comparison_scope!r} does not match "
+ f"classification {classification!r}"
+ )
+
return errors
diff --git a/tests/test_catalog.py b/tests/test_catalog.py
index ee73d38..980f4d3 100644
--- a/tests/test_catalog.py
+++ b/tests/test_catalog.py
@@ -3,6 +3,9 @@
import subprocess
import sys
from pathlib import Path
+from xml.etree import ElementTree
+
+from scripts.discover_candidates import parse_feed
ROOT = Path(__file__).resolve().parents[1]
@@ -25,3 +28,28 @@ def test_catalog_is_valid() -> None:
def test_generated_readme_is_current() -> None:
result = run_script("build_readme.py", "--check")
assert result.returncode == 0, result.stderr
+
+
+def test_arxiv_feed_parser_strips_version_and_normalizes_text() -> None:
+ feed = ElementTree.fromstring(
+ """
+
+ https://arxiv.org/abs/2608.12345v2
+ A Product-Level\n Study
+ Tests Claude Code.
+ 2026-08-20T00:00:00Z
+ Ada Researcher
+
+ """
+ )
+ parsed = parse_feed(ElementTree.tostring(feed))
+ assert parsed == [
+ {
+ "arxiv_id": "2608.12345",
+ "title": "A Product-Level Study",
+ "summary": "Tests Claude Code.",
+ "authors": ["Ada Researcher"],
+ "published": "2026-08-20T00:00:00Z",
+ "url": "https://arxiv.org/abs/2608.12345",
+ }
+ ]
diff --git a/views/README.md b/views/README.md
new file mode 100644
index 0000000..e0fb9d3
--- /dev/null
+++ b/views/README.md
@@ -0,0 +1,11 @@
+
+
+[← Home](../README.md)
+
+# Research views
+
+- [By product](by-product.md) — Claude Code and Codex CLI separately.
+- [By method](by-method.md) — repository graphs, retrieval, verification, orchestration, and more.
+- [Comparison fairness](fair-comparisons.md) — same-model and same-budget controls.
+- [By venue](by-venue.md) — conferences, workshops, and preprints.
+- [All paper dossiers](../papers/README.md) — one evidence page per paper.
diff --git a/views/by-method.md b/views/by-method.md
new file mode 100644
index 0000000..aa6b33d
--- /dev/null
+++ b/views/by-method.md
@@ -0,0 +1,142 @@
+
+
+[← Research views](README.md) · [Home](../README.md)
+
+# Papers by method
+
+Tags describe the intervention added around or instead of the production coding-agent baseline.
+
+## `verifier-loop` (12)
+
+| System | Venue | Method |
+|---|---|---|
+| [ARTEMIS](../papers/artemis-2026.md) | ICLR 2026 | Supervisor with dynamic expert agents, recursive task decomposition, parallel exploration, context management, triage, and reporting |
+| [Agentic Harness Engineering](../papers/agentic-harness-engineering-2026.md) | arXiv 2026 | Closed-loop harness evolution using component, trajectory, and decision observability with evidence-backed edits and rollback |
+| [FormAct](../papers/formact-2026.md) | ICML 2026 | HTML source editor, template retrieval, rendered-page review agent, iterative refinement, and edit-triggered context compression |
+| [Prefactory](../papers/prefactory-2026.md) | arXiv 2026 | LLM-synthesized lexical and structural search heuristics, deterministic candidate ranking, targeted refactoring, and differential tests |
+| [QLCoder](../papers/qlcoder-2026.md) | ICLR 2026 | CVE-grounded retrieval, AST guidance, CodeQL LSP tools, and iterative execution feedback |
+| [RepoOMP](../papers/repoomp-2026.md) | arXiv 2026 | Multi-granularity dependency graph, rule-or-LLM hotspot routing, reduced transformation context, and compile/workload/speedup validation |
+| [RustPrint](../papers/rustprint-2026.md) | arXiv 2026 | Architecture documentation blueprint, module and data-flow planning, compile feedback, documentation mismatch repair, and source-test translation |
+| [SCATE](../papers/scate-2026.md) | arXiv 2026 | Contextual-bandit supervisor that selects testing actions from current coverage and testability state |
+| [Terminal-Bench 2.0](../papers/terminal-bench-2-2026.md) | ICLR 2026 | Evaluation-only benchmark with stronger task verification and a neutral terminal scaffold |
+| [BountyBench](../papers/bountybench-2025.md) | NeurIPS 2025 | Evaluation-only benchmark of offensive and defensive cybersecurity agents |
+| [MLZero](../papers/mlzero-2025.md) | NeurIPS 2025 | Hierarchical agents with multimodal perception, semantic and episodic memory, and a planning-coding-evaluation loop |
+| [TiCoder](../papers/ticoder-2022.md) | AST at ICSE 2022 | Generate tests to formalize user intent, collect lightweight feedback, and rank or prune candidate programs |
+## `test-feedback` (8)
+
+| System | Venue | Method |
+|---|---|---|
+| [Prefactory](../papers/prefactory-2026.md) | arXiv 2026 | LLM-synthesized lexical and structural search heuristics, deterministic candidate ranking, targeted refactoring, and differential tests |
+| [RustPrint](../papers/rustprint-2026.md) | arXiv 2026 | Architecture documentation blueprint, module and data-flow planning, compile feedback, documentation mismatch repair, and source-test translation |
+| [SCATE](../papers/scate-2026.md) | arXiv 2026 | Contextual-bandit supervisor that selects testing actions from current coverage and testability state |
+| [WebDesignIter](../papers/webdesigniter-2026.md) | arXiv 2026 | Persistent architecture and design knowledge graph, design-informed planning, targeted patches, and sandbox repair |
+| [ZeroRepo](../papers/rpg-zerorepo-2026.md) | ICLR 2026 | Persistent graph over features, functions, files, interfaces, and data flow with graph-guided generation and validation |
+| [icat-agent](../papers/icat-agent-2026.md) | arXiv 2026 | Event-based agent communication, issue-quality routing, parallel patching and validation, and exploratory fallback |
+| [MLZero](../papers/mlzero-2025.md) | NeurIPS 2025 | Hierarchical agents with multimodal perception, semantic and episodic memory, and a planning-coding-evaluation loop |
+| [TiCoder](../papers/ticoder-2022.md) | AST at ICSE 2022 | Generate tests to formalize user intent, collect lightweight feedback, and rank or prune candidate programs |
+## `context-reduction` (4)
+
+| System | Venue | Method |
+|---|---|---|
+| [AGENTS.md](../papers/agents-md-impact-2026.md) | ICSE JAWs 2026 | Persistent repository-level build, test, style, and directory guidance in AGENTS.md |
+| [CodeGrep](../papers/codegrep-2026.md) | arXiv 2026 | GRPO-trained 14B retrieval agent that issues multi-turn parallel grep, glob, and read operations for a frozen downstream coding agent |
+| [FormAct](../papers/formact-2026.md) | ICML 2026 | HTML source editor, template retrieval, rendered-page review agent, iterative refinement, and edit-triggered context compression |
+| [RepoOMP](../papers/repoomp-2026.md) | arXiv 2026 | Multi-granularity dependency graph, rule-or-LLM hotspot routing, reduced transformation context, and compile/workload/speedup validation |
+## `multi-agent` (4)
+
+| System | Venue | Method |
+|---|---|---|
+| [ARTEMIS](../papers/artemis-2026.md) | ICLR 2026 | Supervisor with dynamic expert agents, recursive task decomposition, parallel exploration, context management, triage, and reporting |
+| [Co-Coder](../papers/co-coder-2026.md) | arXiv 2026 | Static dependency graph partitioning, structural hub isolation, community detection, and dependency-aware scheduling |
+| [icat-agent](../papers/icat-agent-2026.md) | arXiv 2026 | Event-based agent communication, issue-quality routing, parallel patching and validation, and exploratory fallback |
+| [MLZero](../papers/mlzero-2025.md) | NeurIPS 2025 | Hierarchical agents with multimodal perception, semantic and episodic memory, and a planning-coding-evaluation loop |
+## `parallelism` (4)
+
+| System | Venue | Method |
+|---|---|---|
+| [ARTEMIS](../papers/artemis-2026.md) | ICLR 2026 | Supervisor with dynamic expert agents, recursive task decomposition, parallel exploration, context management, triage, and reporting |
+| [Co-Coder](../papers/co-coder-2026.md) | arXiv 2026 | Static dependency graph partitioning, structural hub isolation, community detection, and dependency-aware scheduling |
+| [CodeGrep](../papers/codegrep-2026.md) | arXiv 2026 | GRPO-trained 14B retrieval agent that issues multi-turn parallel grep, glob, and read operations for a frozen downstream coding agent |
+| [icat-agent](../papers/icat-agent-2026.md) | arXiv 2026 | Event-based agent communication, issue-quality routing, parallel patching and validation, and exploratory fallback |
+## `repository-graph` (4)
+
+| System | Venue | Method |
+|---|---|---|
+| [Co-Coder](../papers/co-coder-2026.md) | arXiv 2026 | Static dependency graph partitioning, structural hub isolation, community detection, and dependency-aware scheduling |
+| [RepoOMP](../papers/repoomp-2026.md) | arXiv 2026 | Multi-granularity dependency graph, rule-or-LLM hotspot routing, reduced transformation context, and compile/workload/speedup validation |
+| [WebDesignIter](../papers/webdesigniter-2026.md) | arXiv 2026 | Persistent architecture and design knowledge graph, design-informed planning, targeted patches, and sandbox repair |
+| [ZeroRepo](../papers/rpg-zerorepo-2026.md) | ICLR 2026 | Persistent graph over features, functions, files, interfaces, and data flow with graph-guided generation and validation |
+## `retrieval` (4)
+
+| System | Venue | Method |
+|---|---|---|
+| [CodeGrep](../papers/codegrep-2026.md) | arXiv 2026 | GRPO-trained 14B retrieval agent that issues multi-turn parallel grep, glob, and read operations for a frozen downstream coding agent |
+| [FormAct](../papers/formact-2026.md) | ICML 2026 | HTML source editor, template retrieval, rendered-page review agent, iterative refinement, and edit-triggered context compression |
+| [QLCoder](../papers/qlcoder-2026.md) | ICLR 2026 | CVE-grounded retrieval, AST guidance, CodeQL LSP tools, and iterative execution feedback |
+| [WebDesignIter](../papers/webdesigniter-2026.md) | arXiv 2026 | Persistent architecture and design knowledge graph, design-informed planning, targeted patches, and sandbox repair |
+## `structured-state` (4)
+
+| System | Venue | Method |
+|---|---|---|
+| [AGENTS.md](../papers/agents-md-impact-2026.md) | ICSE JAWs 2026 | Persistent repository-level build, test, style, and directory guidance in AGENTS.md |
+| [RustPrint](../papers/rustprint-2026.md) | arXiv 2026 | Architecture documentation blueprint, module and data-flow planning, compile feedback, documentation mismatch repair, and source-test translation |
+| [WebDesignIter](../papers/webdesigniter-2026.md) | arXiv 2026 | Persistent architecture and design knowledge graph, design-informed planning, targeted patches, and sandbox repair |
+| [ZeroRepo](../papers/rpg-zerorepo-2026.md) | ICLR 2026 | Persistent graph over features, functions, files, interfaces, and data flow with graph-guided generation and validation |
+## `dependency-aware-planning` (3)
+
+| System | Venue | Method |
+|---|---|---|
+| [Co-Coder](../papers/co-coder-2026.md) | arXiv 2026 | Static dependency graph partitioning, structural hub isolation, community detection, and dependency-aware scheduling |
+| [RustPrint](../papers/rustprint-2026.md) | arXiv 2026 | Architecture documentation blueprint, module and data-flow planning, compile feedback, documentation mismatch repair, and source-test translation |
+| [ZeroRepo](../papers/rpg-zerorepo-2026.md) | ICLR 2026 | Persistent graph over features, functions, files, interfaces, and data flow with graph-guided generation and validation |
+## `memory` (3)
+
+| System | Venue | Method |
+|---|---|---|
+| [ARTEMIS](../papers/artemis-2026.md) | ICLR 2026 | Supervisor with dynamic expert agents, recursive task decomposition, parallel exploration, context management, triage, and reporting |
+| [Agentic Harness Engineering](../papers/agentic-harness-engineering-2026.md) | arXiv 2026 | Closed-loop harness evolution using component, trajectory, and decision observability with evidence-backed edits and rollback |
+| [MLZero](../papers/mlzero-2025.md) | NeurIPS 2025 | Hierarchical agents with multimodal perception, semantic and episodic memory, and a planning-coding-evaluation loop |
+## `deterministic-search` (2)
+
+| System | Venue | Method |
+|---|---|---|
+| [Prefactory](../papers/prefactory-2026.md) | arXiv 2026 | LLM-synthesized lexical and structural search heuristics, deterministic candidate ranking, targeted refactoring, and differential tests |
+| [RepoOMP](../papers/repoomp-2026.md) | arXiv 2026 | Multi-granularity dependency graph, rule-or-LLM hotspot routing, reduced transformation context, and compile/workload/speedup validation |
+## `dynamic-specialization` (2)
+
+| System | Venue | Method |
+|---|---|---|
+| [ARTEMIS](../papers/artemis-2026.md) | ICLR 2026 | Supervisor with dynamic expert agents, recursive task decomposition, parallel exploration, context management, triage, and reporting |
+| [icat-agent](../papers/icat-agent-2026.md) | arXiv 2026 | Event-based agent communication, issue-quality routing, parallel patching and validation, and exploratory fallback |
+## `observability` (2)
+
+| System | Venue | Method |
+|---|---|---|
+| [Agentic Harness Engineering](../papers/agentic-harness-engineering-2026.md) | arXiv 2026 | Closed-loop harness evolution using component, trajectory, and decision observability with evidence-backed edits and rollback |
+| [Bug taxonomy](../papers/engineering-pitfalls-2026.md) | FSE Industry Track 2026 | Evaluation-only manual taxonomy of functionality, integration, invocation, and command-execution failures |
+## `static-analysis` (2)
+
+| System | Venue | Method |
+|---|---|---|
+| [Prefactory](../papers/prefactory-2026.md) | arXiv 2026 | LLM-synthesized lexical and structural search heuristics, deterministic candidate ranking, targeted refactoring, and differential tests |
+| [QLCoder](../papers/qlcoder-2026.md) | ICLR 2026 | CVE-grounded retrieval, AST guidance, CodeQL LSP tools, and iterative execution feedback |
+## `harness-evolution` (1)
+
+| System | Venue | Method |
+|---|---|---|
+| [Agentic Harness Engineering](../papers/agentic-harness-engineering-2026.md) | arXiv 2026 | Closed-loop harness evolution using component, trajectory, and decision observability with evidence-backed edits and rollback |
+## `performance-feedback` (1)
+
+| System | Venue | Method |
+|---|---|---|
+| [RepoOMP](../papers/repoomp-2026.md) | arXiv 2026 | Multi-granularity dependency graph, rule-or-LLM hotspot routing, reduced transformation context, and compile/workload/speedup validation |
+## `repository-instructions` (1)
+
+| System | Venue | Method |
+|---|---|---|
+| [AGENTS.md](../papers/agents-md-impact-2026.md) | ICSE JAWs 2026 | Persistent repository-level build, test, style, and directory guidance in AGENTS.md |
+## `visual-review` (1)
+
+| System | Venue | Method |
+|---|---|---|
+| [FormAct](../papers/formact-2026.md) | ICML 2026 | HTML source editor, template retrieval, rendered-page review agent, iterative refinement, and edit-triggered context compression |
diff --git a/views/by-product.md b/views/by-product.md
new file mode 100644
index 0000000..851a72e
--- /dev/null
+++ b/views/by-product.md
@@ -0,0 +1,47 @@
+
+
+[← Research views](README.md) · [Home](../README.md)
+
+# Papers by product
+
+Product means the complete agent or harness evaluated by the paper,
+not merely a model from the same vendor.
+
+## Claude Code (14)
+
+| System | Venue | Evidence class | Task | Strength |
+|---|---|---|---|---|
+| [AGENTS.md](../papers/agents-md-impact-2026.md) | ICSE JAWs 2026 | Direct comparison | Real pull-request tasks with and without repository instructions | high |
+| [ARTEMIS](../papers/artemis-2026.md) | ICLR 2026 | Direct comparison | Long-horizon penetration testing in a live enterprise network | medium |
+| [Bug taxonomy](../papers/engineering-pitfalls-2026.md) | FSE Industry Track 2026 | Evaluation only | Empirical analysis of product bug reports | high |
+| [Co-Coder](../papers/co-coder-2026.md) | arXiv 2026 | Direct comparison | Parallel multi-agent coding | medium |
+| [CodeGrep](../papers/codegrep-2026.md) | arXiv 2026 | Related method | Repository retrieval for issue resolution | contextual |
+| [QLCoder](../papers/qlcoder-2026.md) | ICLR 2026 | Direct comparison | CodeQL query synthesis from CVE metadata | high |
+| [RepoOMP](../papers/repoomp-2026.md) | arXiv 2026 | Direct comparison | Repository-aware OpenMP hotspot parallelization | medium |
+| [RustPrint](../papers/rustprint-2026.md) | arXiv 2026 | Direct comparison | Whole-codebase migration from C to Rust | medium |
+| [SCATE](../papers/scate-2026.md) | arXiv 2026 | Related method | Automated test generation | contextual |
+| [Terminal-Bench 2.0](../papers/terminal-bench-2-2026.md) | ICLR 2026 | Evaluation only | General terminal-agent evaluation | high |
+| [WebDesignIter](../papers/webdesigniter-2026.md) | arXiv 2026 | Direct comparison | Repository-level front-end code generation | medium |
+| [ZeroRepo](../papers/rpg-zerorepo-2026.md) | ICLR 2026 | Direct comparison | Repository-level codebase generation | high |
+| [icat-agent](../papers/icat-agent-2026.md) | arXiv 2026 | Direct comparison | Repository issue resolution | medium |
+| [BountyBench](../papers/bountybench-2025.md) | NeurIPS 2025 | Evaluation only | Vulnerability detection, exploitation, and patching | high |
+## Codex CLI (11)
+
+| System | Venue | Evidence class | Task | Strength |
+|---|---|---|---|---|
+| [AGENTS.md](../papers/agents-md-impact-2026.md) | ICSE JAWs 2026 | Direct comparison | Real pull-request tasks with and without repository instructions | high |
+| [ARTEMIS](../papers/artemis-2026.md) | ICLR 2026 | Direct comparison | Long-horizon penetration testing in a live enterprise network | medium |
+| [Agentic Harness Engineering](../papers/agentic-harness-engineering-2026.md) | arXiv 2026 | Direct comparison | Automatic evolution of coding-agent harnesses | medium |
+| [Bug taxonomy](../papers/engineering-pitfalls-2026.md) | FSE Industry Track 2026 | Evaluation only | Empirical analysis of product bug reports | high |
+| [FormAct](../papers/formact-2026.md) | ICML 2026 | Direct comparison | Rich-format document generation and editing | high |
+| [Prefactory](../papers/prefactory-2026.md) | arXiv 2026 | Direct comparison | Discovery and application of library-adoption refactorings | medium |
+| [Terminal-Bench 2.0](../papers/terminal-bench-2-2026.md) | ICLR 2026 | Evaluation only | General terminal-agent evaluation | high |
+| [WebDesignIter](../papers/webdesigniter-2026.md) | arXiv 2026 | Direct comparison | Repository-level front-end code generation | medium |
+| [ZeroRepo](../papers/rpg-zerorepo-2026.md) | ICLR 2026 | Direct comparison | Repository-level codebase generation | high |
+| [BountyBench](../papers/bountybench-2025.md) | NeurIPS 2025 | Evaluation only | Vulnerability detection, exploitation, and patching | high |
+| [MLZero](../papers/mlzero-2025.md) | NeurIPS 2025 | Direct comparison | End-to-end machine learning automation | medium |
+## OpenAI Codex model (1)
+
+| System | Venue | Evidence class | Task | Strength |
+|---|---|---|---|---|
+| [TiCoder](../papers/ticoder-2022.md) | AST at ICSE 2022 | Historical model | Interactive function-level code generation | high |
diff --git a/views/by-venue.md b/views/by-venue.md
new file mode 100644
index 0000000..c3a0e3a
--- /dev/null
+++ b/views/by-venue.md
@@ -0,0 +1,55 @@
+
+
+[← Research views](README.md) · [Home](../README.md)
+
+# Papers by venue
+
+Main-conference, workshop, benchmark-track, and preprint status remain explicit.
+
+## arXiv (9)
+
+| System | Year | Status | Evidence class |
+|---|---|---|---|
+| [Agentic Harness Engineering](../papers/agentic-harness-engineering-2026.md) | 2026 | preprint | Direct comparison |
+| [Co-Coder](../papers/co-coder-2026.md) | 2026 | preprint | Direct comparison |
+| [CodeGrep](../papers/codegrep-2026.md) | 2026 | preprint | Related method |
+| [Prefactory](../papers/prefactory-2026.md) | 2026 | preprint | Direct comparison |
+| [RepoOMP](../papers/repoomp-2026.md) | 2026 | preprint | Direct comparison |
+| [RustPrint](../papers/rustprint-2026.md) | 2026 | preprint | Direct comparison |
+| [SCATE](../papers/scate-2026.md) | 2026 | preprint | Related method |
+| [WebDesignIter](../papers/webdesigniter-2026.md) | 2026 | preprint | Direct comparison |
+| [icat-agent](../papers/icat-agent-2026.md) | 2026 | preprint | Direct comparison |
+## ICLR (4)
+
+| System | Year | Status | Evidence class |
+|---|---|---|---|
+| [ARTEMIS](../papers/artemis-2026.md) | 2026 | main | Direct comparison |
+| [QLCoder](../papers/qlcoder-2026.md) | 2026 | main | Direct comparison |
+| [Terminal-Bench 2.0](../papers/terminal-bench-2-2026.md) | 2026 | main | Evaluation only |
+| [ZeroRepo](../papers/rpg-zerorepo-2026.md) | 2026 | main | Direct comparison |
+## NeurIPS (2)
+
+| System | Year | Status | Evidence class |
+|---|---|---|---|
+| [BountyBench](../papers/bountybench-2025.md) | 2025 | dataset-benchmark | Evaluation only |
+| [MLZero](../papers/mlzero-2025.md) | 2025 | main | Direct comparison |
+## AST at ICSE (1)
+
+| System | Year | Status | Evidence class |
+|---|---|---|---|
+| [TiCoder](../papers/ticoder-2022.md) | 2022 | workshop | Historical model |
+## FSE Industry Track (1)
+
+| System | Year | Status | Evidence class |
+|---|---|---|---|
+| [Bug taxonomy](../papers/engineering-pitfalls-2026.md) | 2026 | conference | Evaluation only |
+## ICML (1)
+
+| System | Year | Status | Evidence class |
+|---|---|---|---|
+| [FormAct](../papers/formact-2026.md) | 2026 | main | Direct comparison |
+## ICSE JAWs (1)
+
+| System | Year | Status | Evidence class |
+|---|---|---|---|
+| [AGENTS.md](../papers/agents-md-impact-2026.md) | 2026 | workshop | Direct comparison |
diff --git a/views/fair-comparisons.md b/views/fair-comparisons.md
new file mode 100644
index 0000000..6538642
--- /dev/null
+++ b/views/fair-comparisons.md
@@ -0,0 +1,34 @@
+
+
+[← Research views](README.md) · [Home](../README.md)
+
+# Comparison fairness
+
+“Direct comparison” means a numeric head-to-head was reported.
+It does not mean the model, budget, tools, or product policy were held fixed.
+
+## Same model and same budget (1)
+
+| System | Same model | Same budget | Strength | Caveat |
+|---|---|---|---|---|
+| [AGENTS.md](../papers/agents-md-impact-2026.md) | yes | yes | high | Workshop study and configuration intervention rather than a standalone agent. |
+## Known model or budget mismatch (3)
+
+| System | Same model | Same budget | Strength | Caveat |
+|---|---|---|---|---|
+| [ARTEMIS](../papers/artemis-2026.md) | unknown | no | medium | Runtime limits, model combinations, safety refusals, and product policies differ across systems. |
+| [RustPrint](../papers/rustprint-2026.md) | no | unknown | medium | The comparison mixes model and scaffold choices and is specific to C-to-Rust migration. |
+| [ZeroRepo](../papers/rpg-zerorepo-2026.md) | no | unknown | high | ZeroRepo uses o3-mini while the Claude Code baseline uses Claude 4 Sonnet, so model and harness effects are mixed. |
+## Control parity not fully reported (9)
+
+| System | Same model | Same budget | Strength | Caveat |
+|---|---|---|---|---|
+| [Agentic Harness Engineering](../papers/agentic-harness-engineering-2026.md) | unknown | unknown | medium | Preprint; model, harness, and evaluation-budget effects require careful separation. |
+| [Co-Coder](../papers/co-coder-2026.md) | unknown | unknown | medium | Small 28-task evaluation and preprint status. |
+| [FormAct](../papers/formact-2026.md) | unknown | unknown | high | Codex has slightly higher content alignment, and the task is document formatting rather than source-code repair. |
+| [Prefactory](../papers/prefactory-2026.md) | unknown | unknown | medium | The strongest comparison concerns candidate detection, not universal end-to-end task success. |
+| [QLCoder](../papers/qlcoder-2026.md) | unknown | unknown | high | Security-query synthesis is narrower than general software engineering. |
+| [RepoOMP](../papers/repoomp-2026.md) | unknown | unknown | medium | Domain-specific high-performance-computing task and preprint status. |
+| [WebDesignIter](../papers/webdesigniter-2026.md) | unknown | unknown | medium | Front-end-specific benchmark and preprint status. |
+| [icat-agent](../papers/icat-agent-2026.md) | unknown | unknown | medium | Exact model parity varies by reported configuration; preprint. |
+| [MLZero](../papers/mlzero-2025.md) | unknown | unknown | medium | Results concern machine-learning automation and should not be generalized to all software-engineering tasks. |