From a915826cfde516644b0ce11cd44daf23be30b1d2 Mon Sep 17 00:00:00 2001 From: Taylor Gagne Date: Thu, 15 Oct 2026 13:18:49 -0700 Subject: [PATCH 1/2] feat(bench): ContextBench Lite file precision/recall bench for CGraph (build plan item 2) Adds bench/contextbench/: a pinned, one-command run that scores CGraph's retrieved files against ContextBench's human-annotated gold contexts with ContextBench's own file-level metric (EuniAI/ContextBench @ 1436c28, HF data @ c285579, all 500 Lite instances). Retrieval reuses the ARB adapter; the snapshot lifecycle is extracted from bench/arb/cgraph_eval.py into served_snapshot so both benches share it (ARB output unchanged). Result on bin-v0.9.4-1-g43453f3: default-context file precision 0.100 (95% CI 0.089-0.112), best top-K precision 0.217 at K=2. The ContextBench half of item 2's gate (>= 0.5) fails. Co-Authored-By: Claude Opus 5.5 --- bench/arb/cgraph_eval.py | 51 ++++--- bench/contextbench/RESULTS.md | 213 ++++++++++++++++++++++++++++++ bench/contextbench/cgraph_eval.py | 160 ++++++++++++++++++++++ bench/contextbench/instances.py | 48 +++++++ bench/contextbench/run.sh | 92 +++++++++++++ bench/contextbench/score.py | 170 ++++++++++++++++++++++++ 6 files changed, 714 insertions(+), 20 deletions(-) create mode 100644 bench/contextbench/RESULTS.md create mode 100644 bench/contextbench/cgraph_eval.py create mode 100644 bench/contextbench/instances.py create mode 100755 bench/contextbench/run.sh create mode 100644 bench/contextbench/score.py diff --git a/bench/arb/cgraph_eval.py b/bench/arb/cgraph_eval.py index a5064b2..9aa30c1 100644 --- a/bench/arb/cgraph_eval.py +++ b/bench/arb/cgraph_eval.py @@ -43,8 +43,9 @@ import time from collections import defaultdict from concurrent.futures import ThreadPoolExecutor, as_completed +from contextlib import contextmanager from pathlib import Path -from typing import Any +from typing import Any, Iterator import snapshots @@ -122,6 +123,34 @@ def shutdown(self, pid: int | None) -> None: time.sleep(0.1) +@contextmanager +def served_snapshot(bin_dir: Path, cache: Path, repo: str, sha: str, root: Path, keep: bool = False) -> Iterator[tuple[Daemon, dict[str, Any], dict[str, Any]]]: + """Check ``repo@sha`` out at ``root``, build its graph and serve it; yields (daemon, build, status). + + On exit the daemon is stopped and, unless ``keep``, the snapshot is removed. + """ + if root.exists(): + snapshots.remove_tree(root) + snapshots.materialize(cache, repo, sha, root) + daemon = Daemon(bin_dir, root) + serving = False + pid: int | None = None + try: + build = daemon.build() + serving = True # the first client call spawns graphd + status = daemon.wait_ready() + pid = int(status["pid"]) + yield daemon, build, status + finally: + try: + # cgraph-client spawns a daemon when none is running, so only stop one we started. + if serving: + daemon.shutdown(pid) + finally: + if not keep: + snapshots.remove_tree(root) + + def query_text(sample: dict[str, Any]) -> str: parts: list[str] = [] for key, value in sorted((sample.get("query") or {}).items()): @@ -349,18 +378,8 @@ def evaluate_snapshot( jobs: list[tuple[str, dict[str, Any], Path]], ) -> list[tuple[str, dict[str, Any]]]: root = args.work / f"{snapshots.repo_slug(repo)}-{sha[:12]}" - if root.exists(): - snapshots.remove_tree(root) - snapshots.materialize(args.cache, repo, sha, root) - daemon = Daemon(args.cgraph_bin_dir, root) out: list[tuple[str, dict[str, Any]]] = [] - serving = False - pid: int | None = None - try: - build = daemon.build() - serving = True # the first client call spawns graphd - status = daemon.wait_ready() - pid = int(status["pid"]) + with served_snapshot(args.cgraph_bin_dir, args.cache, repo, sha, root, args.keep_snapshots) as (daemon, build, status): index = SnapshotIndex(root) corpus_texts: dict[Path, dict[str, str]] = {} for task, sample, chunks_path in jobs: @@ -406,14 +425,6 @@ def evaluate_snapshot( }, } out.append((task, detail)) - finally: - try: - # cgraph-client spawns a daemon when none is running, so only stop one we started. - if serving: - daemon.shutdown(pid) - finally: - if not args.keep_snapshots: - snapshots.remove_tree(root) return out diff --git a/bench/contextbench/RESULTS.md b/bench/contextbench/RESULTS.md new file mode 100644 index 0000000..f0d6bce --- /dev/null +++ b/bench/contextbench/RESULTS.md @@ -0,0 +1,213 @@ +# CGraph on ContextBench Lite: file-level precision and recall + +This measures item 2 of the CGraph build plan, **"ContextBench file precision >= 0.5"**. +[ContextBench](https://arxiv.org/abs/2602.05892) (v3, Apache-2.0) gives each issue a +human-annotated gold context. This bench scores the files CGraph retrieves for the issue +against that gold, using ContextBench's own metric code. Every instance's repository is +checked out at its base commit, and the real `cgraph`, `graphd` and `cgraph-client` +binaries index and serve it. + +## Verdict + +**Fail.** CGraph's default context has macro file precision **0.100** (95% CI 0.089-0.112) +on 499 of the 500 Lite instances. The gate is 0.5. No cutoff gets close: the best point on +the full top-K curve is K=2, with 0.217 (95% CI 0.191-0.243). The bash-only agent +(mini-SWE-agent with GPT-5) reaches 0.709 (paper Table 2). Item 2's ContextBench half +fails, and it fails by a wide margin. It is not a noise call. + +## Pins + +| What | Pin | +|---|---| +| Paper | arXiv 2602.05892v3 (11 Feb 2026); PDF sha256 `517140a2504959bfb94c7c875f6e80a031e899bc362dcc73c446536b45e18a46` | +| Metric code | [EuniAI/ContextBench](https://github.com/EuniAI/ContextBench) @ `1436c28a8eb95496da4ea69ad458b9f8a8eb7d61` | +| Data | HF `Contextbench/ContextBench` @ `c2855792b006af41c67202d33883fb9d46362853`, `data/contextbench_verified.parquet`, sha256 `e9dcfd504cbfb849ac815a79040c793d0d92f94eecc9b5a4ee3e1445a2f8a791` | +| Subset | ContextBench Lite: all 500 instances (Verified 174, Multi 156, Poly 116, Pro 54; 8 languages; 59 repositories) | +| Engine | `cgraph --version`: `cgraph-native 0.1.0 (bin-v0.9.4-1-g43453f3)`, built from main at 43453f3 | + +The HF config is named `contextbench_verified`, but it is the Lite set. Its 500 +`instance_id`s are exactly the 500 rows of `data/selected_500_instances.csv` in the pinned +repo, which the billed-cost bench used. Every one of its rows is byte-identical to the same +instance in `data/full.parquet` at the same revision. + +## Metric + +The metric is ContextBench's `compute_granularity_metrics` (`contextbench/metrics/compute.py`). +At the file level it is: + +- recall (ContextBench calls it coverage) `= |pred ∩ gold| / |gold|` +- precision `= |pred ∩ gold| / |pred|` + +`gold` is `Gold.files()`: the files of every gold-context span, with paths normalized by +ContextBench (`contextbench/parsers/gold.py`). Appendix H of the paper says "all reported +metrics are averaged over tasks". The headline is therefore the macro mean over instances. +The micro aggregate that ContextBench's CLI prints (`aggregate_results` in +`contextbench/evaluate.py`) is reported next to it. An empty prediction is an error in +ContextBench's evaluator (`no_context_extracted`) and leaves the average. This run had +no empty predictions. One instance has no CGraph output at all (see Findings). The 95% CIs are percentile bootstraps over instances (10,000 +resamples, seed 2602). + +## What CGraph is asked, and the cutoff + +The input is the instance's `problem_statement` only. `instances.py` exports the repo, +commit and issue text, and leaves `gold_context`, `patch`, `test_patch` and the test lists +behind. Retrieval reuses the ARB adapter (`bench/arb/cgraph_eval.py`) unchanged: + +- **default** (the gate row). This is one `context` call with `q` set to the issue text and + every other parameter at the engine default: budget 6000 tokens + (`kDefaultContextBudget`, `src/engine/daemon_ops.cpp`). The prediction is that bundle's + distinct files. It is what CGraph hands an agent that asks about the issue. The cutoff is + CGraph's own default, fixed before any scoring. +- **top-K**. This is the ARB recipe's fused ranking: `context` at budget 8000, `query` with + limit 100, and `context` around each file the issue text names, merged by reciprocal-rank + fusion. It is restricted to files in the checked-out tree. Precision and recall are + reported for every K from 1 to 20, so no K is chosen from the labels. + +The agents' numbers are not measured at a cutoff. They score the context each agent +declares at the end of its run (paper §2.5), so ContextBench has no rank cutoff to copy. + +## Results (500 instances) + +Engine `bin-v0.9.4-1-g43453f3`. n is the number of Lite instances; "valid" is the number +with a non-empty prediction. The one missing instance is +`SWE-Bench-Pro__javascript__maintenance__bugfix__5c001746`: its graph build did not finish, +see Findings. The last two columns score it as 0, and they move no overall number by more +than 0.001. + +| Prediction | Group | n (valid) | Mean files | Precision [95% CI] | Recall [95% CI] | F1 | Micro P | Micro R | P, errors as 0 | R, errors as 0 | +|---|---|---:|---:|---|---|---:|---:|---:|---:|---:| +| **default** | overall | 500 (499) | 12.54 | **0.100** [0.089, 0.112] | 0.371 [0.337, 0.407] | 0.134 | 0.072 | 0.260 | 0.100 | 0.371 | +| default | Verified | 174 (174) | 12.38 | 0.098 [0.082, 0.114] | 0.492 [0.428, 0.556] | 0.148 | 0.073 | 0.449 | 0.098 | 0.492 | +| default | Pro | 54 (53) | 10.42 | 0.168 [0.120, 0.219] | 0.246 [0.173, 0.326] | 0.171 | 0.136 | 0.178 | 0.165 | 0.241 | +| default | Poly | 116 (116) | 12.05 | 0.096 [0.073, 0.121] | 0.306 [0.243, 0.371] | 0.123 | 0.069 | 0.284 | 0.096 | 0.306 | +| default | Multi | 156 (156) | 13.81 | 0.083 [0.066, 0.102] | 0.329 [0.269, 0.391] | 0.113 | 0.056 | 0.195 | 0.083 | 0.329 | +| top-1 | overall | 500 (499) | 1.00 | 0.214 [0.178, 0.251] | 0.106 [0.083, 0.129] | 0.126 | 0.214 | 0.062 | 0.214 | 0.105 | +| top-2 | overall | 500 (499) | 2.00 | 0.217 [0.191, 0.243] | 0.200 [0.173, 0.229] | 0.183 | 0.217 | 0.125 | 0.217 | 0.200 | +| top-3 | overall | 500 (499) | 3.00 | 0.192 [0.172, 0.213] | 0.262 [0.231, 0.295] | 0.194 | 0.192 | 0.166 | 0.192 | 0.261 | +| top-5 | overall | 500 (499) | 5.00 | 0.162 [0.146, 0.178] | 0.347 [0.313, 0.382] | 0.194 | 0.162 | 0.233 | 0.162 | 0.346 | +| top-10 | overall | 500 (499) | 10.00 | 0.110 [0.100, 0.120] | 0.441 [0.406, 0.477] | 0.158 | 0.110 | 0.316 | 0.110 | 0.440 | +| top-20 | overall | 500 (499) | 19.97 | 0.070 [0.065, 0.076] | 0.555 [0.520, 0.591] | 0.116 | 0.070 | 0.404 | 0.070 | 0.554 | + +Per-source rows for every K are in `runs//results.md`. The highest precision on any +row is Pro at top-1: 0.377 (CI 0.245-0.509, n=53). + +Full curve (overall, macro): + +| K | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 12 | 15 | 20 | +|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| +| Precision | 0.214 | 0.217 | 0.192 | 0.176 | 0.162 | 0.147 | 0.135 | 0.125 | 0.116 | 0.110 | 0.099 | 0.086 | 0.070 | +| Recall | 0.106 | 0.200 | 0.262 | 0.307 | 0.347 | 0.367 | 0.395 | 0.411 | 0.423 | 0.441 | 0.477 | 0.513 | 0.555 | + +## Published baselines (file level, ContextBench Lite) + +These come from the paper, arXiv 2602.05892v3. The paper gives point estimates only. The +pinned repo ships no per-instance results, so no CI can be computed for them. + +| System | Recall | Precision | F1 | Source | +|---|---:|---:|---:|---| +| mini-SWE-agent + GPT-5 (the bash-only agent) | 0.682 | **0.709** | 0.634 | Table 2 | +| Agentless + GPT-5 | 0.609 | 0.352 | 0.390 | Table 2 | +| SWE-agent + GPT-5 | 0.726 | 0.537 | 0.544 | Table 2 | +| OpenHands + GPT-5 | 0.733 | 0.400 | 0.463 | Table 2 | +| Prometheus + GPT-5 (graph-based retrieval) | 0.717 | 0.336 | 0.403 | Table 2 | +| mini-SWE-agent + Claude Sonnet 4.5 | 0.720 | 0.665 | 0.624 | Table 3 | +| mini-SWE-agent + Gemini 2.5 Pro | 0.587 | 0.752 | 0.600 | Table 3 | +| mini-SWE-agent + Devstral 2 | 0.660 | 0.693 | 0.615 | Table 3 | + +The build plan's line "graph retrieval found the most gold files, but with half the +precision of a plain bash agent" matches Table 2. Prometheus had recall 0.717 and precision +0.336. mini-SWE-agent had recall 0.682 and precision 0.709. + +**Comparability.** The agents are not the same kind of system as CGraph. Each agent runs a +full GPT-5 session, reads code, writes a patch, and then declares the files it used. CGraph +answers the issue text once, with no model and no iteration. The agent numbers show where +the 0.5 gate sits. They are not a like-for-like baseline. + +## Findings + +1. **The default bundle is too wide for this gate.** A default `context` call returns 12.54 + distinct files on average (median 10, range 1 to 64). The gold context has a mean of 3.47 + files and a median of 2. Suppose every gold file were inside the bundle. Macro precision + at the bundle sizes CGraph actually returned would still be at most 0.332, so the default + cannot reach 0.5 however good its ranking is. That ceiling is computed after scoring from + gold sizes. It is a diagnostic only and was not used to pick anything. +2. **Ranking is the larger gap.** Even a 2-file answer is right only about one time in + five (top-2 precision 0.217). The default bundle contains at least one gold file on 291 + of the 499 instances. +3. **Recall is also below the agents.** Default recall is 0.371 and top-20 recall is 0.555. + The agents' file recall is 0.587 to 0.733. The plan's "most gold files" line describes + Prometheus, a different graph-based agent. It does not describe CGraph's single-shot + retrieval. +4. **Engine bug: one build never finished.** The build of `element-hq/element-web@c9d9c421` + ran 2 h 9 min and was killed. That covered 2 h 1 min of CPU on one thread (`ps`: + `ELAPSED 02:09:22, TIME 02:01:25`). Building each of the snapshot's 2,100 JS/TS files on + its own with a 10 s timeout isolated one file: `docs/lib/mermaid.min.js`, a 2,935,907-byte + minified bundle. The thread's stack was: + `ts_node_parent <- branch_path <- exclusive_branches <- local_values <- collect_url_template + <- analyze_client_call <- http_call_handler <- js_extra_walk <- walk_node`. This is the + HTTP-client contract extractor, and it is super-linear on minified code. It is reported + here and not fixed. `run.sh` hits it again on a rerun. +5. **ContextBench path quirk, unchanged.** `_normalize_rel_path` in + `contextbench/parsers/gold.py` ends with `p.lstrip("./")`. That call strips the leading dot + of `.github/...`, so gold `github/workflows/*.yml` matches no repository file. In total, 18 + of the 1,746 gold files on Lite are not in their base-commit tree. The others include + agent-written reproduction scripts such as `repro.js`. This affects every system scored + with ContextBench's code equally. It is reported, not patched: that would be an eval-data + change. + +## Cost and time + +- **Model and API spend: $0.** CGraph has no model in the loop. +- **Wall time.** The full run took 2 h 28 min (`time`: `16132.80s user 755.04s system 190% + cpu 2:28:03.77 total`), pinned to 8 cores (`taskset -c 0-7`, `--jobs 8`). The other 499 + instances finished within the first 51 minutes. The rest of the time was the single + stuck build in Finding 4. +- **CPU.** 4.7 CPU-hours in total, of which 2.0 were the stuck build. Over the 499 completed + instances, the graph builds took 3.08 h (median 5.5 s; maximum 664 s, on mui/material-ui) + and retrieval took 0.13 h (median 0.45 s per instance). +- **Data.** The snapshot cache is 1.4 GB of shallow bare repositories (59 repositories, 500 + commits). Ten Python repositories were seeded from local mirrors. Their commits are + identical by SHA, so the result does not depend on where they came from. + +## Reproduce + +```sh +PYTHON= bench/contextbench/run.sh --cgraph-bin-dir DIR --jobs 8 +bench/contextbench/run.sh --cgraph-bin-dir DIR --limit 30 # smoke run, not a score +``` + +`DIR` holds `cgraph`, `cgraph-client` and `graphd`. Without `--cgraph-bin-dir`, the script +uses the release that `cgraph` on PATH resolves to. The work directory defaults to +`~/.cache/cgraph-contextbench` (`--work` overrides it). The script: + +1. clones ContextBench at the pinned commit; +2. downloads the Lite parquet and checks its sha256; +3. exports the label-free inputs; +4. fetches the 500 snapshots with `bench/arb/snapshots.py`; +5. runs CGraph; +6. writes `runs/cgraph-/results.{json,md}`. + +It needs git, curl, and Python >= 3.10 with pyarrow. It needs no GPU, no Docker and no +model. ContextBench's tree-sitter pin is needed only for its symbol and span levels, which +this bench does not use. + +**Determinism.** `--limit 30` was rerun separately from the full run. For all 30 of 30 +instances it produced the same default files, the same fused ranking and the same per-op +lists. + +## Files + +- `run.sh`: the pinned one-command pipeline. +- `instances.py`: exports the label-free task inputs. +- `cgraph_eval.py`: the adapter. It imports `served_snapshot`, `retrieve`, `SnapshotIndex` + and the context helpers from `bench/arb/cgraph_eval.py`, and the snapshot cache from + `bench/arb/snapshots.py`. +- `score.py`: scores the predictions with ContextBench's `GoldLoader`, + `compute_granularity_metrics` and `aggregate_results`. + +`bench/arb/cgraph_eval.py` gained `served_snapshot`. It is the checkout, build, serve and +teardown lifecycle that `evaluate_snapshot` had inline, and both benches now share it. A +one-sample run per task, made with ARB's `trace2code` and `edit2ripple` before and after +this change, gave identical details: metrics, ranked files, per-op lists and graph size. + +Eval data changed: no. diff --git a/bench/contextbench/cgraph_eval.py b/bench/contextbench/cgraph_eval.py new file mode 100644 index 0000000..153f2d0 --- /dev/null +++ b/bench/contextbench/cgraph_eval.py @@ -0,0 +1,160 @@ +"""ContextBench adapter for CGraph. + +For every ContextBench Lite instance this checks the real repository out at the +instance's base commit, builds a CGraph graph, and asks the resident daemon +(through ``cgraph-client``) about the issue text. It reuses the Agent Retrieval +Bench adapter (``bench/arb/cgraph_eval.py``) for the snapshot lifecycle and the +retrieval recipe, so both benches measure the same CGraph behaviour. + +Two predictions are stored per instance; nothing reads the gold labels (the +input file carries only repo, commit and issue text, see ``instances.py``): + +* ``default_context_files``: the files of one ``context`` call with ``q`` = the + issue text and every other parameter at the engine default (budget 6000 + tokens). This is the context CGraph hands an agent when asked about the issue, + and the file set the gate is judged on. +* ``ranked_files``: the ARB recipe's fused file ranking (``context`` at budget + 8000, ``query`` with limit 100, and ``context`` around any file the issue text + names, merged by reciprocal-rank fusion), restricted to files in the checked-out + tree. score.py reports precision and recall of its top K for every K from 1 to 20. + +Usage: cgraph_eval.py --instances instances.jsonl --cache DIR --work DIR --out DIR [--cgraph-bin-dir DIR] [--jobs N] [--limit N] +""" + +from __future__ import annotations + +import argparse +import json +import shutil +import sys +import threading +import time +from concurrent.futures import ThreadPoolExecutor, as_completed +from pathlib import Path +from typing import Any + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "arb")) +import cgraph_eval as arb # noqa: E402 +import snapshots # noqa: E402 + + +def tree_files(cache: Path, repo: str, sha: str) -> set[str]: + """Regular files and symlinks in the commit's tree: the paths ContextBench accepts as + repository files (its evaluator drops predicted paths that are not files in the checkout).""" + out = snapshots.git("ls-tree", "-r", "-z", "--full-tree", sha, git_dir=snapshots.bare_path(cache, repo)) + files: set[str] = set() + for entry in out.split("\0"): + if not entry: + continue + meta, path = entry.split("\t", 1) + if meta.split()[1] == "blob": + files.add(path) + return files + + +def evaluate_instance(args: argparse.Namespace, inst: dict[str, Any]) -> dict[str, Any]: + repo, sha = inst["repo"], inst["base_commit"] + root = args.work / f"{snapshots.repo_slug(repo)}-{sha[:12]}" + files = tree_files(args.cache, repo, sha) + with arb.served_snapshot(args.cgraph_bin_dir, args.cache, repo, sha, root, args.keep_snapshots) as (daemon, build, status): + index = arb.SnapshotIndex(root) + prefix = str(root.resolve()) + "/" + text = inst["problem_statement"] + started = time.monotonic() + default = daemon.call("context", {"q": text}) + default_files = [p for p in arb.context_files(arb.context_nodes(default), prefix) if p in files] + sample = {"id": inst["id"], "task_type": "contextbench", "query": {"problem_statement": text}} + got = arb.retrieve(daemon, sample, index, files) + elapsed = time.monotonic() - started + return { + "instance_id": inst["id"], + "original_inst_id": inst["original_inst_id"], + "repo": repo, + "base_commit": sha, + "source": inst["source"], + "language": inst["language"], + "default_context_files": default_files, + "default_context": { + "budget": default.get("budget"), + "tokens_used": default.get("tokens_used"), + "included": len(default.get("included") or []), + "focus": (default.get("focus") or {}).get("id"), + }, + "ranked_files": got["ranked_files"][:50], + "ranked_files_returned": len(got["ranked_files"]), + "cgraph": { + "ops": got["ops"], + "lists": got["lists"], + "retrieve_seconds": elapsed, + "graph_nodes": index.node_count, + "graph_edges": index.edge_count, + "tree_files": len(files), + "files_extracted": build["stats"]["files_total"], + "build_seconds": build["build_seconds"], + "engine_revision": status.get("engine_revision"), + "engine_version": status.get("engine_version"), + }, + } + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--instances", type=Path, required=True, help="instances.jsonl from instances.py.") + parser.add_argument("--cache", type=Path, required=True, help="Bare repo cache from bench/arb/snapshots.py.") + parser.add_argument("--work", type=Path, required=True, help="Scratch dir for materialized snapshots.") + parser.add_argument("--out", type=Path, required=True, help="Output dir for details.jsonl.") + installed = shutil.which("cgraph") + parser.add_argument("--cgraph-bin-dir", type=Path, default=Path(installed).resolve().parent if installed else None, + help="Dir holding cgraph, cgraph-client and graphd (default: the release `cgraph` on PATH resolves to).") + parser.add_argument("--jobs", type=int, default=4) + parser.add_argument("--limit", type=int, default=0, help="Evaluate only the first N instances (0 = all).") + parser.add_argument("--keep-snapshots", action="store_true") + args = parser.parse_args(argv) + if args.cgraph_bin_dir is None: + parser.error("no cgraph on PATH; pass --cgraph-bin-dir") + args.cgraph_bin_dir = args.cgraph_bin_dir.resolve() + args.work = args.work.resolve() + args.cache = args.cache.resolve() + for name in ("cgraph", "cgraph-client", "graphd"): + if not (args.cgraph_bin_dir / name).is_file(): + parser.error(f"{args.cgraph_bin_dir / name} not found") + args.work.mkdir(parents=True, exist_ok=True) + args.out.mkdir(parents=True, exist_ok=True) + + details_path = args.out / "details.jsonl" + done: set[str] = set() + if details_path.exists(): + done = {json.loads(line)["instance_id"] for line in details_path.read_text().splitlines() if line.strip()} + instances = [json.loads(line) for line in args.instances.read_text(encoding="utf-8").splitlines() if line.strip()] + if args.limit: + instances = instances[: args.limit] + todo = [inst for inst in instances if inst["id"] not in done] + print(f"{len(todo)} instances to evaluate ({len(done)} already done)", file=sys.stderr) + + lock = threading.Lock() + finished = 0 + failures: list[str] = [] + with ThreadPoolExecutor(max_workers=args.jobs) as pool: + futures = {pool.submit(evaluate_instance, args, inst): inst for inst in todo} + for future in as_completed(futures): + inst = futures[future] + try: + row = future.result() + except Exception as exc: # report and keep going; the run exits non-zero + failures.append(f"{inst['id']} ({inst['repo']}@{inst['base_commit']}): {exc}") + print(f"FAILED {inst['id']}: {exc}", file=sys.stderr) + continue + with lock: + with details_path.open("a", encoding="utf-8") as handle: + handle.write(json.dumps(row, ensure_ascii=False, sort_keys=True) + "\n") + finished += 1 + print(f"[{finished}/{len(todo)}] {inst['id']} {inst['repo']}@{inst['base_commit'][:12]} " + f"build {row['cgraph']['build_seconds']:.0f}s", file=sys.stderr) + if failures: + print(f"{len(failures)} instance(s) failed:\n" + "\n".join(failures), file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/bench/contextbench/instances.py b/bench/contextbench/instances.py new file mode 100644 index 0000000..6e5e880 --- /dev/null +++ b/bench/contextbench/instances.py @@ -0,0 +1,48 @@ +"""Export the ContextBench Lite task inputs CGraph is allowed to see. + +Reads the released parquet and writes one JSON line per instance with the +repository, base commit and issue text only. ``gold_context``, ``patch``, +``test_patch`` and the test lists are deliberately not exported, so retrieval +cannot read the labels. ``repo`` is the GitHub ``owner/name`` taken from +``repo_url``: the dataset's ``repo`` column is a project label that does not +always name the repository (``fasterxml/jackson`` covers three repositories). + +Usage: instances.py PARQUET OUT.jsonl +""" + +from __future__ import annotations + +import json +import re +import sys +from pathlib import Path + +import pyarrow.parquet as pq + +COLUMNS = ["instance_id", "original_inst_id", "repo_url", "base_commit", "problem_statement", "source", "language"] +GITHUB_RE = re.compile(r"^https://github\.com/([^/]+/[^/]+?)(?:\.git)?/?$") + + +def main(argv: list[str]) -> int: + parquet, out = Path(argv[0]), Path(argv[1]) + rows = pq.read_table(parquet, columns=COLUMNS).to_pylist() + with out.open("w", encoding="utf-8") as handle: + for row in sorted(rows, key=lambda r: r["instance_id"]): + match = GITHUB_RE.match(row["repo_url"] or "") + if not match: + raise SystemExit(f"{row['instance_id']}: not a GitHub repo_url: {row['repo_url']!r}") + handle.write(json.dumps({ + "id": row["instance_id"], + "original_inst_id": row["original_inst_id"], + "repo": match.group(1), + "base_commit": row["base_commit"], + "source": row["source"], + "language": row["language"], + "problem_statement": row["problem_statement"], + }, ensure_ascii=False) + "\n") + print(f"{len(rows)} instances written to {out}", file=sys.stderr) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/bench/contextbench/run.sh b/bench/contextbench/run.sh new file mode 100755 index 0000000..8c919fe --- /dev/null +++ b/bench/contextbench/run.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +# One-command ContextBench Lite run for CGraph (file-level precision and recall). +# +# bench/contextbench/run.sh [--cgraph-bin-dir DIR] [--jobs N] [--work DIR] [--limit N] +# +# --limit N evaluates only the first N instances (a smoke test, not a score). +# +# Pins: ContextBench code at CB_COMMIT (its metric code scores everything), data at +# HF_REVISION. Steps: +# 1. clone ContextBench at the pinned commit +# 2. download the Lite parquet (`contextbench_verified`, 500 instances) and check its sha256 +# 3. export repo, commit and issue text only (instances.py; the labels stay behind) +# 4. fetch every instance's repository at its base commit (bench/arb/snapshots.py) +# 5. run CGraph on every instance (cgraph_eval.py) +# 6. score with ContextBench's file-level metric (score.py) +# +# Needs: git, curl, python3 >= 3.10 with pyarrow (set PYTHON to pick the interpreter), +# and a cgraph release (cgraph, cgraph-client, graphd in one directory; default: where +# `cgraph` on PATH resolves). +set -euo pipefail + +CB_REPO="https://github.com/EuniAI/ContextBench.git" +CB_COMMIT="1436c28a8eb95496da4ea69ad458b9f8a8eb7d61" +HF_DATASET="Contextbench/ContextBench" +HF_REVISION="c2855792b006af41c67202d33883fb9d46362853" +LITE_FILE="contextbench_verified.parquet" +LITE_SHA256="e9dcfd504cbfb849ac815a79040c793d0d92f94eecc9b5a4ee3e1445a2f8a791" + +HERE="$(cd "$(dirname "$0")" && pwd)" +PY="${PYTHON:-python3}" +WORK="${CONTEXTBENCH_WORK:-$HOME/.cache/cgraph-contextbench}" +JOBS=4 +BIN_DIR="" +LIMIT=0 +while [[ $# -gt 0 ]]; do + case "$1" in + --cgraph-bin-dir) BIN_DIR="$(cd "$2" && pwd)"; shift 2 ;; + --jobs) JOBS="$2"; shift 2 ;; + --work) WORK="$2"; shift 2 ;; + --limit) LIMIT="$2"; shift 2 ;; + *) echo "unknown argument: $1" >&2; exit 2 ;; + esac +done +if [[ -z "$BIN_DIR" ]]; then + command -v cgraph >/dev/null || { echo "no cgraph on PATH; pass --cgraph-bin-dir" >&2; exit 1; } + BIN_DIR="$(dirname "$(readlink -f "$(command -v cgraph)")")" +fi +for exe in cgraph cgraph-client graphd; do + [[ -x "$BIN_DIR/$exe" ]] || { echo "missing $BIN_DIR/$exe" >&2; exit 1; } +done +"$PY" -c 'import pyarrow' || { echo "$PY lacks pyarrow; set PYTHON to an interpreter that has it" >&2; exit 1; } +mkdir -p "$WORK" +WORK="$(cd "$WORK" && pwd)" +cd "$WORK" + +# 1. ContextBench code at the pinned commit. +if [[ ! -d contextbench/.git ]]; then + git init --quiet contextbench + git -C contextbench remote add origin "$CB_REPO" +fi +git -C contextbench fetch --quiet --depth 1 origin "$CB_COMMIT" +git -C contextbench -c advice.detachedHead=false checkout --quiet --force "$CB_COMMIT" + +# 2. Lite data, checked against the pinned hash. +mkdir -p data +if [[ ! -f "data/$LITE_FILE" ]]; then + curl -fsSL -o "data/$LITE_FILE.part" \ + "https://huggingface.co/datasets/$HF_DATASET/resolve/$HF_REVISION/data/$LITE_FILE" + mv "data/$LITE_FILE.part" "data/$LITE_FILE" +fi +echo "$LITE_SHA256 data/$LITE_FILE" | sha256sum -c --quiet - + +# 3. Task inputs without labels. +"$PY" "$HERE/instances.py" "data/$LITE_FILE" instances.jsonl + +# 4. Repository snapshots. +python3 "$HERE/../arb/snapshots.py" --cache repos --jobs "$JOBS" instances.jsonl + +# 5. CGraph. Output is keyed by engine revision so a dev build never mixes with a release run. +REV="$("$BIN_DIR/cgraph" --version | awk '{gsub(/[()]/, "", $3); print $3}')" +RUN="runs/cgraph-$REV" +if [[ "$LIMIT" != 0 ]]; then RUN="$RUN-limit$LIMIT"; fi +# A failed instance does not stop scoring: score.py counts it as an error. The script still exits non-zero. +STATUS=0 +python3 "$HERE/cgraph_eval.py" --instances instances.jsonl --cache repos --work "$WORK/snapshots" \ + --out "$RUN" --cgraph-bin-dir "$BIN_DIR" --jobs "$JOBS" --limit "$LIMIT" || STATUS=$? + +# 6. Score. +"$PY" "$HERE/score.py" --contextbench-src contextbench --gold "data/$LITE_FILE" \ + --instances instances.jsonl --limit "$LIMIT" --details "$RUN/details.jsonl" --out "$RUN/results.json" --markdown "$RUN/results.md" +echo "results: $WORK/$RUN/results.md" +exit "$STATUS" diff --git a/bench/contextbench/score.py b/bench/contextbench/score.py new file mode 100644 index 0000000..dfdc50e --- /dev/null +++ b/bench/contextbench/score.py @@ -0,0 +1,170 @@ +"""Score CGraph's ContextBench predictions with ContextBench's own file-level metric. + +Gold files come from ContextBench's ``GoldLoader``/``Gold.files()`` (the union of +every gold-context span's file, path-normalized by ContextBench). Per-instance +file coverage (recall) and precision come from ContextBench's +``compute_granularity_metrics``: ``|pred & gold| / |gold|`` and +``|pred & gold| / |pred|``. The paper (arXiv 2602.05892v3, Appendix H) averages +every metric over tasks, so the headline is the macro mean; the micro aggregate +ContextBench's CLI prints (``aggregate_results``) is reported beside it. An +instance with an empty prediction is an error in ContextBench's evaluator +(``no_context_extracted``) and is left out of its averages; so is an instance CGraph +produced no output for (``no_prediction``, e.g. a failed build). Both are counted, and +the macro means with every such instance scored 0 are reported beside the headline. + +Cutoffs are fixed before scoring and never chosen from the labels: + +* ``default``: the file set of CGraph's ``context`` op at engine defaults (gate row). +* ``top-K`` for every K in 1..20 over the fused ranking: the full curve, no K picked. + +95% confidence intervals are percentile bootstraps over instances (10,000 +resamples, fixed seed). + +Usage: score.py --contextbench-src DIR --gold PARQUET --instances instances.jsonl [--limit N] + --details details.jsonl --out results.json [--markdown results.md] +""" + +from __future__ import annotations + +import argparse +import json +import random +import sys +from pathlib import Path +from typing import Any + +BOOTSTRAP = 10_000 +SEED = 2602 +CURVE = range(1, 21) +TABLE_K = (1, 2, 3, 5, 10, 20) +GROUPS = ("overall", "Verified", "Pro", "Poly", "Multi") + + +def bootstrap_ci(values: list[float]) -> tuple[float, float]: + rng = random.Random(SEED) + n = len(values) + means = sorted(sum(values[rng.randrange(n)] for _ in range(n)) / n for _ in range(BOOTSTRAP)) + return means[int(0.025 * BOOTSTRAP)], means[int(0.975 * BOOTSTRAP) - 1] + + +def summarize(results: list[dict[str, Any]], aggregate_results) -> dict[str, Any]: + valid = [r for r in results if "error" not in r] + out: dict[str, Any] = {"n": len(results), "valid": len(valid), "errors": len(results) - len(valid)} + if not valid: + return out + for key in ("precision", "coverage"): + values = [r["final"]["file"][key] for r in valid] + lo, hi = bootstrap_ci(values) + out[key] = {"mean": sum(values) / len(values), "ci95": [lo, hi], + "mean_errors_as_zero": sum(values) / len(results)} + f1 = [] + for r in valid: + p, c = r["final"]["file"]["precision"], r["final"]["file"]["coverage"] + f1.append(2 * p * c / (p + c) if p + c > 0 else 0.0) + out["f1"] = {"mean": sum(f1) / len(f1)} + out["mean_files"] = sum(r["final"]["file"]["pred_size"] for r in valid) / len(valid) + out["micro"] = aggregate_results(results)["final_file"] + return out + + +def score_prediction(pred: list[str], gold_files: set[str], compute_granularity_metrics) -> dict[str, Any]: + if not pred: + return {"error": "no_context_extracted"} + metrics = compute_granularity_metrics(set(pred), set(), {}, gold_files, set(), {}) + return {"final": {"file": metrics["file"]}} + + +def render_markdown(result: dict[str, Any]) -> str: + lines = [ + "| Prediction | Group | n (valid) | Mean files | Precision [95% CI] | Recall [95% CI] | F1 | Micro P | Micro R | P, errors as 0 | R, errors as 0 |", + "|---|---|---:|---:|---|---|---:|---:|---:|---:|---:|", + ] + for name in ["default", *(f"top-{k}" for k in TABLE_K)]: + for group in GROUPS: + row = result["predictions"][name][group] + if not row.get("valid"): + continue + p, r = row["precision"], row["coverage"] + lines.append( + f"| {name} | {group} | {row['n']} ({row['valid']}) | {row['mean_files']:.2f} " + f"| {p['mean']:.3f} [{p['ci95'][0]:.3f}, {p['ci95'][1]:.3f}] " + f"| {r['mean']:.3f} [{r['ci95'][0]:.3f}, {r['ci95'][1]:.3f}] | {row['f1']['mean']:.3f} " + f"| {row['micro']['precision']:.3f} | {row['micro']['coverage']:.3f} " + f"| {p['mean_errors_as_zero']:.3f} | {r['mean_errors_as_zero']:.3f} |" + ) + if result["no_prediction"]: + lines += ["", "No CGraph output (scored as errors): " + ", ".join(result["no_prediction"])] + lines += ["", "Full curve (overall, macro):", "", "| K | Precision | Recall | F1 |", "|---:|---:|---:|---:|"] + for k in CURVE: + row = result["predictions"][f"top-{k}"]["overall"] + lines.append(f"| {k} | {row['precision']['mean']:.3f} | {row['coverage']['mean']:.3f} | {row['f1']['mean']:.3f} |") + return "\n".join(lines) + "\n" + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--contextbench-src", type=Path, required=True, help="Checkout of EuniAI/ContextBench.") + parser.add_argument("--gold", type=Path, required=True, help="ContextBench gold parquet.") + parser.add_argument("--instances", type=Path, required=True, help="instances.jsonl from instances.py: the instances to score.") + parser.add_argument("--limit", type=int, default=0, help="Score only the first N instances (as cgraph_eval.py --limit).") + parser.add_argument("--details", type=Path, required=True) + parser.add_argument("--out", type=Path, required=True) + parser.add_argument("--markdown", type=Path) + args = parser.parse_args(argv) + sys.path.insert(0, str(args.contextbench_src)) + from contextbench.evaluate import aggregate_results # noqa: PLC0415 + from contextbench.metrics import compute_granularity_metrics # noqa: PLC0415 + from contextbench.parsers import GoldLoader # noqa: PLC0415 + + gold = GoldLoader(str(args.gold)) + universe = [json.loads(line) for line in args.instances.read_text(encoding="utf-8").splitlines() if line.strip()] + if args.limit: + universe = universe[: args.limit] + details = [json.loads(line) for line in args.details.read_text(encoding="utf-8").splitlines() if line.strip()] + by_id = {d["instance_id"]: d for d in details} + unknown = set(by_id) - {u["id"] for u in universe} + if unknown: + raise SystemExit(f"details name instances not in {args.instances}: {sorted(unknown)[:5]}") + predictions: dict[str, Any] = {} + per_instance: dict[str, list[dict[str, Any]]] = {} + + def preds(d: dict[str, Any]) -> dict[str, list[str]]: + out = {"default": d["default_context_files"]} + for k in CURVE: + out[f"top-{k}"] = d["ranked_files"][:k] + return out + + missing = [] + for u in universe: + d = by_id.get(u["id"]) + if d is None: + missing.append(u["id"]) + for name in ("default", *(f"top-{k}" for k in CURVE)): + per_instance.setdefault(name, []).append({"error": "no_prediction", "source": u["source"]}) + continue + gold_files = set(gold.get(u["id"]).files()) + for name, pred in preds(d).items(): + row = score_prediction(pred, gold_files, compute_granularity_metrics) + row["source"] = u["source"] + per_instance.setdefault(name, []).append(row) + for name, rows in per_instance.items(): + predictions[name] = { + group: summarize(rows if group == "overall" else [r for r in rows if r["source"] == group], aggregate_results) + for group in GROUPS + } + result = {"instances": len(universe), "no_prediction": missing, "predictions": predictions} + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8") + if args.markdown: + args.markdown.write_text(render_markdown(result), encoding="utf-8") + for name in ("default", *(f"top-{k}" for k in TABLE_K)): + row = predictions[name]["overall"] + if row.get("valid"): + print(f"{name:8s} n={row['valid']}/{row['n']} files={row['mean_files']:.2f} " + f"P={row['precision']['mean']:.3f} {row['precision']['ci95']} " + f"R={row['coverage']['mean']:.3f} {row['coverage']['ci95']}", file=sys.stderr) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From ea271bafc199ecf3bfd6f20d8b59f39c93f62f5f Mon Sep 17 00:00:00 2001 From: Taylor Gagne Date: Fri, 16 Oct 2026 18:00:46 -0700 Subject: [PATCH 2/2] docs(bench): note #204 fixed the element-web hang; exact Appendix H quote - RESULTS.md finding 4: the minified-bundle hang is fixed on main by #204; numbers stay pinned to 43453f3, no 500/500 rerun yet. - Quote Appendix H exactly ("averaged over evaluated tasks"). - Per-source rows for every K live in results.json, not results.md. - run.sh: steps 4 and 5 use $PY like the other steps. Co-Authored-By: Claude Opus 5.5 --- bench/contextbench/RESULTS.md | 10 ++++++---- bench/contextbench/run.sh | 4 ++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/bench/contextbench/RESULTS.md b/bench/contextbench/RESULTS.md index f0d6bce..7b2ba83 100644 --- a/bench/contextbench/RESULTS.md +++ b/bench/contextbench/RESULTS.md @@ -40,7 +40,7 @@ At the file level it is: `gold` is `Gold.files()`: the files of every gold-context span, with paths normalized by ContextBench (`contextbench/parsers/gold.py`). Appendix H of the paper says "all reported -metrics are averaged over tasks". The headline is therefore the macro mean over instances. +metrics are averaged over evaluated tasks". The headline is therefore the macro mean over instances. The micro aggregate that ContextBench's CLI prints (`aggregate_results` in `contextbench/evaluate.py`) is reported next to it. An empty prediction is an error in ContextBench's evaluator (`no_context_extracted`) and leaves the average. This run had @@ -88,7 +88,7 @@ than 0.001. | top-10 | overall | 500 (499) | 10.00 | 0.110 [0.100, 0.120] | 0.441 [0.406, 0.477] | 0.158 | 0.110 | 0.316 | 0.110 | 0.440 | | top-20 | overall | 500 (499) | 19.97 | 0.070 [0.065, 0.076] | 0.555 [0.520, 0.591] | 0.116 | 0.070 | 0.404 | 0.070 | 0.554 | -Per-source rows for every K are in `runs//results.md`. The highest precision on any +Per-source rows for every K are in `runs//results.json`. The highest precision on any row is Pro at top-1: 0.377 (CI 0.245-0.509, n=53). Full curve (overall, macro): @@ -145,8 +145,10 @@ the 0.5 gate sits. They are not a like-for-like baseline. minified bundle. The thread's stack was: `ts_node_parent <- branch_path <- exclusive_branches <- local_values <- collect_url_template <- analyze_client_call <- http_call_handler <- js_extra_walk <- walk_node`. This is the - HTTP-client contract extractor, and it is super-linear on minified code. It is reported - here and not fixed. `run.sh` hits it again on a rerun. + HTTP-client contract extractor, and it was super-linear on minified code. #204 (a41425e) + fixed it on main after this run: the same file now builds in 17-28 s with an identical + graph. The numbers here stay pinned to 43453f3; a 500/500 rerun on a build with #204 + has not been done. 5. **ContextBench path quirk, unchanged.** `_normalize_rel_path` in `contextbench/parsers/gold.py` ends with `p.lstrip("./")`. That call strips the leading dot of `.github/...`, so gold `github/workflows/*.yml` matches no repository file. In total, 18 diff --git a/bench/contextbench/run.sh b/bench/contextbench/run.sh index 8c919fe..9533641 100755 --- a/bench/contextbench/run.sh +++ b/bench/contextbench/run.sh @@ -74,7 +74,7 @@ echo "$LITE_SHA256 data/$LITE_FILE" | sha256sum -c --quiet - "$PY" "$HERE/instances.py" "data/$LITE_FILE" instances.jsonl # 4. Repository snapshots. -python3 "$HERE/../arb/snapshots.py" --cache repos --jobs "$JOBS" instances.jsonl +"$PY" "$HERE/../arb/snapshots.py" --cache repos --jobs "$JOBS" instances.jsonl # 5. CGraph. Output is keyed by engine revision so a dev build never mixes with a release run. REV="$("$BIN_DIR/cgraph" --version | awk '{gsub(/[()]/, "", $3); print $3}')" @@ -82,7 +82,7 @@ RUN="runs/cgraph-$REV" if [[ "$LIMIT" != 0 ]]; then RUN="$RUN-limit$LIMIT"; fi # A failed instance does not stop scoring: score.py counts it as an error. The script still exits non-zero. STATUS=0 -python3 "$HERE/cgraph_eval.py" --instances instances.jsonl --cache repos --work "$WORK/snapshots" \ +"$PY" "$HERE/cgraph_eval.py" --instances instances.jsonl --cache repos --work "$WORK/snapshots" \ --out "$RUN" --cgraph-bin-dir "$BIN_DIR" --jobs "$JOBS" --limit "$LIMIT" || STATUS=$? # 6. Score.