-
Notifications
You must be signed in to change notification settings - Fork 268
benchmarks: Add zvec-grep to benchmarks #268
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
Merged
Changes from all commits
Commits
Show all changes
8 commits
Select commit
Hold shift + click to select a range
c8bcbd7
bench: add zvec-grep as a benchmark baseline
Pringled e51ea65
Merge remote-tracking branch 'origin/main' into add-zvec-grep-baseline
Pringled 755ac4f
bench: give the speed plots more room instead of special-casing labels
Pringled 3e58d9d
bench: tighten the cold plot's label offset for the larger figure
Pringled babd057
bench: offset plot labels from the marker edge instead of in data space
Pringled 0a0b802
Update readme
Pringled 14bf36d
bench: pin and record the zvec-grep version, fix stale model references
Pringled 4087dae
bench: measure zvec-grep with the speed benchmark harness
Pringled File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,214 @@ | ||
| import argparse | ||
| import json | ||
| import re | ||
| import shutil | ||
| import subprocess | ||
| import sys | ||
| import time | ||
| from dataclasses import dataclass | ||
| from pathlib import Path | ||
|
|
||
| from benchmarks.data import ( | ||
| RepoSpec, | ||
| Task, | ||
| add_filter_args, | ||
| grouped_tasks, | ||
| load_filtered_tasks, | ||
| save_results, | ||
| ) | ||
| from benchmarks.metrics import file_rank, ndcg_at_k | ||
|
|
||
| _ZG = "zg" | ||
| _EMBEDDING = "local/potion-code-16m-v2" | ||
| _TOP_K = 10 | ||
| _LATENCY_RUNS = 3 | ||
| _INDEX_TIMEOUT = 1800 | ||
| _SEARCH_TIMEOUT = 60 | ||
| _HIT_RE = re.compile(r"^#\d+\s+matchedBy=\S+\s+(?P<path>.+):(?P<start>\d+)-(?P<end>\d+)$") | ||
|
|
||
|
|
||
| @dataclass(frozen=True) | ||
| class RepoResult: | ||
| """Per-repo benchmark result.""" | ||
|
|
||
| repo: str | ||
| language: str | ||
| ndcg10: float | ||
| p50_ms: float | ||
| index_ms: float | ||
|
|
||
|
|
||
| def _cleanup_index(benchmark_dir: Path) -> None: | ||
| shutil.rmtree(benchmark_dir / ".zvec-grep", ignore_errors=True) | ||
|
|
||
|
|
||
| def _build_index(benchmark_dir: Path) -> tuple[bool, float]: | ||
| """Build a zvec-grep hybrid (FTS + vector) index for a repo; return (success, elapsed_ms).""" | ||
| _cleanup_index(benchmark_dir) | ||
| started = time.perf_counter() | ||
| try: | ||
| proc = subprocess.run( | ||
| [_ZG, "index", str(benchmark_dir), "--embedding", _EMBEDDING, "--mode", "direct"], | ||
| capture_output=True, | ||
| text=True, | ||
| timeout=_INDEX_TIMEOUT, | ||
| ) | ||
| except subprocess.TimeoutExpired: | ||
| print(f" WARNING: zg index timed out after {_INDEX_TIMEOUT}s", file=sys.stderr) | ||
| return False, (time.perf_counter() - started) * 1000 | ||
| elapsed_ms = (time.perf_counter() - started) * 1000 | ||
| if proc.returncode != 0: | ||
| print(f" WARNING: zg index failed: {proc.stderr.strip()[:300]}", file=sys.stderr) | ||
| return False, elapsed_ms | ||
| return True, elapsed_ms | ||
|
|
||
|
|
||
| def _run_search(query: str, benchmark_dir: Path, *, top_k: int) -> list[str]: | ||
| """Return absolute file paths ranked by zvec-grep's hybrid (FTS + vector) result order.""" | ||
| cmd = [_ZG, "query", query, "--limit", str(top_k), "--mode", "direct", "--refresh", "off"] | ||
| try: | ||
| proc = subprocess.run(cmd, cwd=benchmark_dir, capture_output=True, text=True, timeout=_SEARCH_TIMEOUT) | ||
| except subprocess.TimeoutExpired: | ||
| return [] | ||
| if proc.returncode != 0: | ||
| return [] | ||
| seen: dict[str, None] = {} | ||
| for line in proc.stdout.splitlines(): | ||
| match = _HIT_RE.match(line.strip()) | ||
| if not match: | ||
| continue | ||
| abs_path = str((benchmark_dir / match.group("path")).resolve()) | ||
| seen[abs_path] = None | ||
| return list(seen)[:top_k] | ||
|
|
||
|
|
||
| def _evaluate_repo( | ||
| tasks: list[Task], | ||
| benchmark_dir: Path, | ||
| *, | ||
| verbose: bool = False, | ||
| ) -> tuple[float, float]: | ||
| """Return (mean ndcg@10, p50 latency ms) for a list of tasks.""" | ||
| ndcg10_sum = 0.0 | ||
| latencies: list[float] = [] | ||
|
|
||
| for task in tasks: | ||
| query_latencies: list[float] = [] | ||
| file_paths: list[str] = [] | ||
| for _ in range(_LATENCY_RUNS): | ||
| started = time.perf_counter() | ||
| file_paths = _run_search(task.query, benchmark_dir, top_k=_TOP_K) | ||
| query_latencies.append((time.perf_counter() - started) * 1000) | ||
| latencies.append(sorted(query_latencies)[_LATENCY_RUNS // 2]) | ||
|
|
||
| relevant_ranks = [rank for t in task.all_relevant if (rank := file_rank(file_paths, t.path)) is not None] | ||
| q_ndcg10 = ndcg_at_k(relevant_ranks, len(task.all_relevant), _TOP_K) | ||
| ndcg10_sum += q_ndcg10 | ||
|
|
||
| if verbose: | ||
| print( | ||
| f" ndcg@10={q_ndcg10:.3f} ranks={relevant_ranks} n_rel={len(task.all_relevant)} q={task.query!r}", | ||
| file=sys.stderr, | ||
| ) | ||
| print(f" targets: {', '.join(t.path for t in task.all_relevant)}", file=sys.stderr) | ||
| print(f" top-5: {[Path(fp).name for fp in file_paths[:5]]}", file=sys.stderr) | ||
|
|
||
| latencies.sort() | ||
| return ndcg10_sum / len(tasks), latencies[len(latencies) // 2] | ||
|
|
||
|
|
||
| def _run_repo(spec: RepoSpec, tasks: list[Task], *, verbose: bool) -> RepoResult | None: | ||
| """Index, evaluate, and clean up a single repo.""" | ||
| benchmark_dir = spec.benchmark_dir | ||
| ok, index_ms = _build_index(benchmark_dir) | ||
| if not ok: | ||
| print(f" SKIP: {spec.name} — zg index failed", file=sys.stderr) | ||
| _cleanup_index(benchmark_dir) | ||
| return None | ||
|
|
||
| try: | ||
| ndcg10, p50_ms = _evaluate_repo(tasks, benchmark_dir, verbose=verbose) | ||
| finally: | ||
| _cleanup_index(benchmark_dir) | ||
|
|
||
| return RepoResult(repo=spec.name, language=spec.language, ndcg10=ndcg10, p50_ms=p50_ms, index_ms=index_ms) | ||
|
|
||
|
|
||
| def _zg_version() -> str: | ||
| """Return the installed zvec-grep version, or 'unknown' if zg cannot be queried.""" | ||
| try: | ||
| proc = subprocess.run([_ZG, "--version"], capture_output=True, text=True, timeout=30) | ||
| except (OSError, subprocess.TimeoutExpired): | ||
| return "unknown" | ||
| return proc.stdout.strip() or "unknown" if proc.returncode == 0 else "unknown" | ||
|
|
||
|
|
||
| def _parse_args() -> argparse.Namespace: | ||
| parser = argparse.ArgumentParser(description="Benchmark zvec-grep on the semble benchmark suite.") | ||
| add_filter_args(parser, verbose=True) | ||
| return parser.parse_args() | ||
|
|
||
|
|
||
| def main() -> None: | ||
| """Run the zvec-grep baseline benchmark.""" | ||
| args = _parse_args() | ||
| repo_specs, tasks = load_filtered_tasks(args.repo or None, args.language or None) | ||
|
|
||
| print(f"zvec-grep (hybrid: FTS + {_EMBEDDING} vector search)", file=sys.stderr) | ||
| print(f"{'Repo':<22} {'Language':<12} {'Index':>9} {'NDCG@10':>8} {'p50':>8}", file=sys.stderr) | ||
| print(f"{'-' * 22} {'-' * 12} {'-' * 9} {'-' * 8} {'-' * 8}", file=sys.stderr) | ||
|
|
||
| results: list[RepoResult] = [] | ||
| for repo, repo_task_list in sorted(grouped_tasks(tasks).items()): | ||
| spec = repo_specs[repo] | ||
| if args.verbose: | ||
| print(f"\n--- {repo} ---", file=sys.stderr) | ||
| result = _run_repo(spec, repo_task_list, verbose=args.verbose) | ||
| if result is None: | ||
| continue | ||
| results.append(result) | ||
| print( | ||
| f"{repo:<22} {spec.language:<12} {result.index_ms:>8.0f}ms {result.ndcg10:>8.3f} {result.p50_ms:>7.1f}ms", | ||
| file=sys.stderr, | ||
| ) | ||
|
|
||
| if not results: | ||
| return | ||
|
|
||
| avg_ndcg10 = sum(r.ndcg10 for r in results) / len(results) | ||
| avg_p50 = sum(r.p50_ms for r in results) / len(results) | ||
| avg_index = sum(r.index_ms for r in results) / len(results) | ||
| print(f"{'-' * 22} {'-' * 12} {'-' * 9} {'-' * 8} {'-' * 8}", file=sys.stderr) | ||
| avg_label = f"Average ({len(results)})" | ||
| print( | ||
| f"{avg_label:<22} {'':<12} {avg_index:>8.0f}ms {avg_ndcg10:>8.3f} {avg_p50:>7.1f}ms", | ||
| file=sys.stderr, | ||
| ) | ||
|
|
||
| summary = { | ||
| "tool": "zvec-grep", | ||
| "version": _zg_version(), | ||
| "note": f"hybrid FTS + {_EMBEDDING} vector search", | ||
| "repos": [ | ||
| { | ||
| "repo": r.repo, | ||
| "language": r.language, | ||
| "ndcg10": round(r.ndcg10, 4), | ||
| "p50_ms": round(r.p50_ms, 1), | ||
| "index_ms": round(r.index_ms, 0), | ||
| } | ||
| for r in results | ||
| ], | ||
| "avg_ndcg10": round(avg_ndcg10, 4), | ||
| "avg_p50_ms": round(avg_p50, 1), | ||
| "avg_index_ms": round(avg_index, 0), | ||
| } | ||
| print(json.dumps(summary, indent=2)) | ||
|
|
||
| if not args.repo and not args.language: | ||
| out = save_results("zvecgrep", summary) | ||
| print(f"\nResults saved to {out}", file=sys.stderr) | ||
|
|
||
|
|
||
| if __name__ == "__main__": | ||
| main() |
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.