diff --git a/assets/images/speed_vs_ndcg_cold.png b/assets/images/speed_vs_ndcg_cold.png index 2d2fa5f1..ebc55d54 100644 Binary files a/assets/images/speed_vs_ndcg_cold.png and b/assets/images/speed_vs_ndcg_cold.png differ diff --git a/assets/images/speed_vs_ndcg_warm.png b/assets/images/speed_vs_ndcg_warm.png index 33327318..768193c2 100644 Binary files a/assets/images/speed_vs_ndcg_warm.png and b/assets/images/speed_vs_ndcg_warm.png differ diff --git a/benchmarks/README.md b/benchmarks/README.md index 7501600e..e1928492 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -21,6 +21,7 @@ Quality and speed across all methods. | CodeRankEmbed | 0.839 | 116 s | 16 ms | | ColGREP | 0.693 | 5.4 s | 122 ms | | BM25 | 0.673 | 47 ms | 0.17 ms | +| zvec-grep | 0.670 | 3.4 s | 391 ms | | ck | 0.642 | 96 s | 187 ms | | codebase-memory-mcp | 0.630 | 454 ms | 46 ms | | grepai | 0.561 | 35 s | 48 ms | @@ -71,28 +72,28 @@ semble returns the top-50 ranked chunks. `ripgrep+read` splits the query into ke NDCG@10 per language, sorted by CodeRankEmbed (CRE in the table). Best score per row is bolded. -| Language | semble | CRE | ColGREP | ck | cbm | grepai | probe | cs | ripgrep | -| ----------- | --------: | --------: | --------: | --------: | --------: | --------: | --------: | --------: | --------: | -| javascript | 0.917 | **0.925** | 0.823 | 0.772 | 0.770 | 0.675 | 0.588 | 0.171 | 0.176 | -| scala | 0.909 | **0.925** | 0.765 | 0.717 | 0.704 | 0.330 | 0.392 | 0.111 | 0.180 | -| zig | **0.913** | 0.911 | 0.474 | 0.511 | 0.766 | 0.755 | 0.369 | 0.121 | 0.000 | -| ruby | **0.909** | 0.905 | 0.708 | 0.738 | 0.689 | 0.643 | 0.382 | 0.255 | 0.230 | -| cpp | **0.915** | 0.897 | 0.626 | 0.687 | 0.630 | 0.731 | 0.375 | 0.262 | 0.126 | -| elixir | **0.894** | 0.893 | 0.808 | 0.786 | 0.506 | 0.669 | 0.412 | 0.397 | 0.134 | -| python | 0.867 | **0.878** | 0.777 | 0.721 | 0.643 | 0.634 | 0.488 | 0.305 | 0.202 | -| csharp | **0.885** | 0.848 | 0.614 | 0.548 | 0.775 | 0.277 | 0.392 | 0.248 | 0.117 | -| php | **0.858** | 0.847 | 0.663 | 0.615 | 0.608 | 0.402 | 0.340 | 0.180 | 0.123 | -| swift | **0.860** | 0.845 | 0.710 | 0.672 | 0.630 | 0.429 | 0.280 | 0.151 | 0.160 | -| bash | 0.825 | **0.834** | 0.706 | 0.677 | 0.768 | 0.723 | 0.226 | 0.170 | 0.000 | -| lua | 0.823 | **0.829** | 0.798 | 0.738 | 0.591 | 0.699 | 0.336 | 0.050 | 0.000 | -| kotlin | 0.821 | **0.823** | 0.637 | 0.587 | 0.611 | 0.478 | 0.335 | 0.170 | 0.166 | -| haskell | 0.765 | **0.811** | 0.683 | 0.733 | 0.624 | 0.483 | 0.313 | 0.160 | 0.000 | -| java | **0.849** | 0.790 | 0.641 | 0.606 | 0.554 | 0.386 | 0.536 | 0.136 | 0.198 | -| c | 0.741 | **0.771** | 0.676 | 0.606 | 0.655 | 0.555 | 0.384 | 0.175 | 0.000 | -| rust | **0.856** | 0.754 | 0.662 | 0.419 | 0.454 | 0.519 | 0.242 | 0.193 | 0.162 | -| go | **0.895** | 0.713 | 0.785 | 0.458 | 0.506 | 0.722 | 0.410 | 0.183 | 0.133 | -| typescript | **0.706** | 0.671 | 0.430 | 0.456 | 0.455 | 0.394 | 0.354 | 0.145 | 0.128 | -| **overall** | **0.854** | 0.839 | **0.693** | **0.634** | **0.630** | **0.561** | **0.387** | **0.200** | **0.126** | +| Language | semble | CRE | ColGREP | zvec-grep | ck | cbm | grepai | probe | cs | ripgrep | +| ----------- | --------: | --------: | --------: | ---------: | --------: | --------: | --------: | --------: | --------: | --------: | +| javascript | 0.917 | **0.925** | 0.823 | 0.760 | 0.772 | 0.770 | 0.675 | 0.588 | 0.171 | 0.176 | +| scala | 0.909 | **0.925** | 0.765 | 0.799 | 0.717 | 0.704 | 0.330 | 0.392 | 0.111 | 0.180 | +| zig | **0.913** | 0.911 | 0.474 | 0.664 | 0.511 | 0.766 | 0.755 | 0.369 | 0.121 | 0.000 | +| ruby | **0.909** | 0.905 | 0.708 | 0.758 | 0.738 | 0.689 | 0.643 | 0.382 | 0.255 | 0.230 | +| cpp | **0.915** | 0.897 | 0.626 | 0.677 | 0.687 | 0.630 | 0.731 | 0.375 | 0.262 | 0.126 | +| elixir | **0.894** | 0.893 | 0.808 | 0.811 | 0.786 | 0.506 | 0.669 | 0.412 | 0.397 | 0.134 | +| python | 0.867 | **0.878** | 0.777 | 0.704 | 0.721 | 0.643 | 0.634 | 0.488 | 0.305 | 0.202 | +| csharp | **0.885** | 0.848 | 0.614 | 0.609 | 0.548 | 0.775 | 0.277 | 0.392 | 0.248 | 0.117 | +| php | **0.858** | 0.847 | 0.663 | 0.583 | 0.615 | 0.608 | 0.402 | 0.340 | 0.180 | 0.123 | +| swift | **0.860** | 0.845 | 0.710 | 0.709 | 0.672 | 0.630 | 0.429 | 0.280 | 0.151 | 0.160 | +| bash | 0.825 | **0.834** | 0.706 | 0.725 | 0.677 | 0.768 | 0.723 | 0.226 | 0.170 | 0.000 | +| lua | 0.823 | **0.829** | 0.798 | 0.736 | 0.738 | 0.591 | 0.699 | 0.336 | 0.050 | 0.000 | +| kotlin | 0.821 | **0.823** | 0.637 | 0.628 | 0.587 | 0.611 | 0.478 | 0.335 | 0.170 | 0.166 | +| haskell | 0.765 | **0.811** | 0.683 | 0.689 | 0.733 | 0.624 | 0.483 | 0.313 | 0.160 | 0.000 | +| java | **0.849** | 0.790 | 0.641 | 0.685 | 0.606 | 0.554 | 0.386 | 0.536 | 0.136 | 0.198 | +| c | 0.741 | **0.771** | 0.676 | 0.598 | 0.606 | 0.655 | 0.555 | 0.384 | 0.175 | 0.000 | +| rust | **0.856** | 0.754 | 0.662 | 0.541 | 0.419 | 0.454 | 0.519 | 0.242 | 0.193 | 0.162 | +| go | **0.895** | 0.713 | 0.785 | 0.512 | 0.458 | 0.506 | 0.722 | 0.410 | 0.183 | 0.133 | +| typescript | **0.706** | 0.671 | 0.430 | 0.476 | 0.456 | 0.455 | 0.394 | 0.354 | 0.145 | 0.128 | +| **overall** | **0.854** | 0.839 | **0.693** | **0.670** | **0.634** | **0.630** | **0.561** | **0.387** | **0.200** | **0.126** | cbm = [codebase-memory-mcp](#methods). @@ -102,9 +103,9 @@ cbm = [codebase-memory-mcp](#methods). | Retrieval | Raw | + ranking | | ---------------------- | ----: | --------: | -| BM25 | 0.675 | 0.834 | -| potion-code-16M | 0.650 | 0.821 | -| BM25 + potion-code-16M | — | **0.854** | +| BM25 | 0.675 | 0.834 | +| potion-code-16M-v2 | 0.650 | 0.821 | +| BM25 + potion-code-16M-v2 | — | **0.854** |
By query category @@ -147,8 +148,9 @@ cbm = [codebase-memory-mcp](#methods). - **[grepai](https://github.com/nicholasgasior/grepai)**: semantic search using [nomic-embed-text](https://huggingface.co/nomic-ai/nomic-embed-text-v1) (137M params) via a local Ollama daemon. - **[codebase-memory-mcp](https://github.com/DeusData/codebase-memory-mcp)**: code intelligence engine that indexes a repo into a SQLite/graph store. We benchmark its `search_graph` tool in `fast` mode, which does BM25 full-text search with structural boosting. - **[ck](https://github.com/BeaconBay/ck)**: hybrid regex + semantic search using [BAAI/bge-small-en-v1.5](https://huggingface.co/BAAI/bge-small-en-v1.5). +- **[zvec-grep](https://github.com/zvec-ai/zvec-grep)**: [potion-code-16M-v2](https://huggingface.co/minishlab/potion-code-16M-v2) static embeddings + BM25. Benchmarked in `direct` mode. - **[CodeRankEmbed](https://huggingface.co/nomic-ai/CodeRankEmbed)**: 137M-param transformer embedding model for code retrieval, used for semantic-only dense search. -- **[semble](https://github.com/your-repo/semble)**: this library. [potion-code-16M](https://huggingface.co/minishlab/potion-code-16M) static embeddings + BM25 + the semble reranking stack. +- **[semble](https://github.com/your-repo/semble)**: this library. [potion-code-16M-v2](https://huggingface.co/minishlab/potion-code-16M-v2) static embeddings + BM25 + the semble reranking stack. ## Excluded methods @@ -168,7 +170,7 @@ uv run python -m benchmarks.sync_repos # clone / update uv run python -m benchmarks.sync_repos --check # verify only ``` -All tools run CPU-only. semble uses `minishlab/potion-code-16M`; CodeRankEmbed uses `nomic-ai/CodeRankEmbed` (137M params). The speed benchmark touches one repo per language with a cold-start index and 5 query runs per repo. +All tools run CPU-only. semble uses `minishlab/potion-code-16M-v2`; CodeRankEmbed uses `nomic-ai/CodeRankEmbed` (137M params). The speed benchmark touches one repo per language with a cold-start index and 5 query runs per repo.
semble @@ -188,7 +190,8 @@ Full runs write to `benchmarks/results/semble-hybrid-.json`. ```bash uv run python -m benchmarks.speed_benchmark -uv run python -m benchmarks.speed_benchmark --semble-only # skip the baselines +uv run python -m benchmarks.speed_benchmark --tools semble zvec-grep # subset of tools +uv run python -m benchmarks.speed_benchmark --repo flask --repo axios # subset of repos ``` Writes to `benchmarks/results/speed-.json`. @@ -280,6 +283,18 @@ uv run python -m benchmarks.baselines.ck --repo fastapi --repo axios
+
+zvec-grep + +Needs `zg` on `$PATH` (`npm install -g @zvec/zvec-grep@0.2.1`, requires Node.js 22+). Downloads the `potion-code-16m-v2` embedding model on first index. The published numbers were measured with 0.2.1; the runner records the installed version in its results file, and pinning matters because it parses `zg`'s human-readable output. + +```bash +uv run python -m benchmarks.baselines.zvecgrep +uv run python -m benchmarks.baselines.zvecgrep --repo fastapi --repo axios +``` + +
+
ripgrep diff --git a/benchmarks/baselines/zvecgrep.py b/benchmarks/baselines/zvecgrep.py new file mode 100644 index 00000000..8f531e74 --- /dev/null +++ b/benchmarks/baselines/zvecgrep.py @@ -0,0 +1,214 @@ +import argparse +import json +import re +import shutil +import subprocess +import sys +import time +from dataclasses import dataclass +from pathlib import Path + +from benchmarks.data import ( + RepoSpec, + Task, + add_filter_args, + grouped_tasks, + load_filtered_tasks, + save_results, +) +from benchmarks.metrics import file_rank, ndcg_at_k + +_ZG = "zg" +_EMBEDDING = "local/potion-code-16m-v2" +_TOP_K = 10 +_LATENCY_RUNS = 3 +_INDEX_TIMEOUT = 1800 +_SEARCH_TIMEOUT = 60 +_HIT_RE = re.compile(r"^#\d+\s+matchedBy=\S+\s+(?P.+):(?P\d+)-(?P\d+)$") + + +@dataclass(frozen=True) +class RepoResult: + """Per-repo benchmark result.""" + + repo: str + language: str + ndcg10: float + p50_ms: float + index_ms: float + + +def _cleanup_index(benchmark_dir: Path) -> None: + shutil.rmtree(benchmark_dir / ".zvec-grep", ignore_errors=True) + + +def _build_index(benchmark_dir: Path) -> tuple[bool, float]: + """Build a zvec-grep hybrid (FTS + vector) index for a repo; return (success, elapsed_ms).""" + _cleanup_index(benchmark_dir) + started = time.perf_counter() + try: + proc = subprocess.run( + [_ZG, "index", str(benchmark_dir), "--embedding", _EMBEDDING, "--mode", "direct"], + capture_output=True, + text=True, + timeout=_INDEX_TIMEOUT, + ) + except subprocess.TimeoutExpired: + print(f" WARNING: zg index timed out after {_INDEX_TIMEOUT}s", file=sys.stderr) + return False, (time.perf_counter() - started) * 1000 + elapsed_ms = (time.perf_counter() - started) * 1000 + if proc.returncode != 0: + print(f" WARNING: zg index failed: {proc.stderr.strip()[:300]}", file=sys.stderr) + return False, elapsed_ms + return True, elapsed_ms + + +def _run_search(query: str, benchmark_dir: Path, *, top_k: int) -> list[str]: + """Return absolute file paths ranked by zvec-grep's hybrid (FTS + vector) result order.""" + cmd = [_ZG, "query", query, "--limit", str(top_k), "--mode", "direct", "--refresh", "off"] + try: + proc = subprocess.run(cmd, cwd=benchmark_dir, capture_output=True, text=True, timeout=_SEARCH_TIMEOUT) + except subprocess.TimeoutExpired: + return [] + if proc.returncode != 0: + return [] + seen: dict[str, None] = {} + for line in proc.stdout.splitlines(): + match = _HIT_RE.match(line.strip()) + if not match: + continue + abs_path = str((benchmark_dir / match.group("path")).resolve()) + seen[abs_path] = None + return list(seen)[:top_k] + + +def _evaluate_repo( + tasks: list[Task], + benchmark_dir: Path, + *, + verbose: bool = False, +) -> tuple[float, float]: + """Return (mean ndcg@10, p50 latency ms) for a list of tasks.""" + ndcg10_sum = 0.0 + latencies: list[float] = [] + + for task in tasks: + query_latencies: list[float] = [] + file_paths: list[str] = [] + for _ in range(_LATENCY_RUNS): + started = time.perf_counter() + file_paths = _run_search(task.query, benchmark_dir, top_k=_TOP_K) + query_latencies.append((time.perf_counter() - started) * 1000) + latencies.append(sorted(query_latencies)[_LATENCY_RUNS // 2]) + + relevant_ranks = [rank for t in task.all_relevant if (rank := file_rank(file_paths, t.path)) is not None] + q_ndcg10 = ndcg_at_k(relevant_ranks, len(task.all_relevant), _TOP_K) + ndcg10_sum += q_ndcg10 + + if verbose: + print( + f" ndcg@10={q_ndcg10:.3f} ranks={relevant_ranks} n_rel={len(task.all_relevant)} q={task.query!r}", + file=sys.stderr, + ) + print(f" targets: {', '.join(t.path for t in task.all_relevant)}", file=sys.stderr) + print(f" top-5: {[Path(fp).name for fp in file_paths[:5]]}", file=sys.stderr) + + latencies.sort() + return ndcg10_sum / len(tasks), latencies[len(latencies) // 2] + + +def _run_repo(spec: RepoSpec, tasks: list[Task], *, verbose: bool) -> RepoResult | None: + """Index, evaluate, and clean up a single repo.""" + benchmark_dir = spec.benchmark_dir + ok, index_ms = _build_index(benchmark_dir) + if not ok: + print(f" SKIP: {spec.name} — zg index failed", file=sys.stderr) + _cleanup_index(benchmark_dir) + return None + + try: + ndcg10, p50_ms = _evaluate_repo(tasks, benchmark_dir, verbose=verbose) + finally: + _cleanup_index(benchmark_dir) + + return RepoResult(repo=spec.name, language=spec.language, ndcg10=ndcg10, p50_ms=p50_ms, index_ms=index_ms) + + +def _zg_version() -> str: + """Return the installed zvec-grep version, or 'unknown' if zg cannot be queried.""" + try: + proc = subprocess.run([_ZG, "--version"], capture_output=True, text=True, timeout=30) + except (OSError, subprocess.TimeoutExpired): + return "unknown" + return proc.stdout.strip() or "unknown" if proc.returncode == 0 else "unknown" + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Benchmark zvec-grep on the semble benchmark suite.") + add_filter_args(parser, verbose=True) + return parser.parse_args() + + +def main() -> None: + """Run the zvec-grep baseline benchmark.""" + args = _parse_args() + repo_specs, tasks = load_filtered_tasks(args.repo or None, args.language or None) + + print(f"zvec-grep (hybrid: FTS + {_EMBEDDING} vector search)", file=sys.stderr) + print(f"{'Repo':<22} {'Language':<12} {'Index':>9} {'NDCG@10':>8} {'p50':>8}", file=sys.stderr) + print(f"{'-' * 22} {'-' * 12} {'-' * 9} {'-' * 8} {'-' * 8}", file=sys.stderr) + + results: list[RepoResult] = [] + for repo, repo_task_list in sorted(grouped_tasks(tasks).items()): + spec = repo_specs[repo] + if args.verbose: + print(f"\n--- {repo} ---", file=sys.stderr) + result = _run_repo(spec, repo_task_list, verbose=args.verbose) + if result is None: + continue + results.append(result) + print( + f"{repo:<22} {spec.language:<12} {result.index_ms:>8.0f}ms {result.ndcg10:>8.3f} {result.p50_ms:>7.1f}ms", + file=sys.stderr, + ) + + if not results: + return + + avg_ndcg10 = sum(r.ndcg10 for r in results) / len(results) + avg_p50 = sum(r.p50_ms for r in results) / len(results) + avg_index = sum(r.index_ms for r in results) / len(results) + print(f"{'-' * 22} {'-' * 12} {'-' * 9} {'-' * 8} {'-' * 8}", file=sys.stderr) + avg_label = f"Average ({len(results)})" + print( + f"{avg_label:<22} {'':<12} {avg_index:>8.0f}ms {avg_ndcg10:>8.3f} {avg_p50:>7.1f}ms", + file=sys.stderr, + ) + + summary = { + "tool": "zvec-grep", + "version": _zg_version(), + "note": f"hybrid FTS + {_EMBEDDING} vector search", + "repos": [ + { + "repo": r.repo, + "language": r.language, + "ndcg10": round(r.ndcg10, 4), + "p50_ms": round(r.p50_ms, 1), + "index_ms": round(r.index_ms, 0), + } + for r in results + ], + "avg_ndcg10": round(avg_ndcg10, 4), + "avg_p50_ms": round(avg_p50, 1), + "avg_index_ms": round(avg_index, 0), + } + print(json.dumps(summary, indent=2)) + + if not args.repo and not args.language: + out = save_results("zvecgrep", summary) + print(f"\nResults saved to {out}", file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/plot.py b/benchmarks/plot.py index d177684d..4801611c 100644 --- a/benchmarks/plot.py +++ b/benchmarks/plot.py @@ -61,6 +61,14 @@ class _Method(TypedDict): "color": "#4a7ba3", "params_m": 33, }, + { + "name": "zvec-grep", + "ndcg10": 0.6701, + "index_ms": 3402.0, + "query_p50_ms": 391.09, + "color": "#c23b6e", + "params_m": 16, + }, { "name": "BM25", "ndcg10": 0.673, @@ -103,10 +111,9 @@ class _Method(TypedDict): }, ] -# Fixed label offset in cube-root(ms) space — gives a consistent visual gap at every x-position. -# The warm plot spans ~0.01–500 ms so needs a much smaller delta than the cold plot (~100 ms–100 s). -_CBRT_LABEL_DELTA_COLD = 2.0 -_CBRT_LABEL_DELTA_WARM = 0.2 +# Gap between a marker's edge and its label, in points. Offsetting from the edge (rather than in data +# space) keeps the gap identical for every marker size, axis range and figure size. +_LABEL_GAP_POINTS = 3.0 # Frontier methods per mode. # Cold: incumbent prior-art curve (ripgrep → BM25 → ColGREP → CodeRankEmbed); semble floats above it. @@ -148,9 +155,8 @@ def _make_plot(out_path: Path, *, warm: bool = False) -> None: :param warm: If True, use per-query latency (index pre-built). If False, use index + query latency. """ mode = "warm" if warm else "cold" - cbrt_label_delta = _CBRT_LABEL_DELTA_WARM if warm else _CBRT_LABEL_DELTA_COLD - fig, ax = plt.subplots(figsize=(8, 5)) + fig, ax = plt.subplots(figsize=(9, 5.6)) fig.patch.set_facecolor("white") ax.set_facecolor("white") @@ -173,7 +179,7 @@ def _make_plot(out_path: Path, *, warm: bool = False) -> None: if m["name"] in _FRONTIER_NAMES[mode] ] ) - xlim = (0.01, 500) if warm else (5, 200_000) + xlim = (0.01, 1_000) if warm else (5, 200_000) # Shade the incumbent zone: region below the frontier, closed to the plot edges. shade_xs = [xlim[0]] + [p[0] for p in frontier] + [xlim[1]] shade_ys = [frontier[0][1]] + [p[1] for p in frontier] + [frontier[-1][1]] @@ -190,10 +196,11 @@ def _make_plot(out_path: Path, *, warm: bool = False) -> None: for m in _METHODS: x = m["query_p50_ms"] if warm else m["index_ms"] + m["query_p50_ms"] y = m["ndcg10"] + size = _marker_size(m["params_m"]) ax.scatter( x, y, - s=_marker_size(m["params_m"]), + s=size, color=m["color"], marker="o", zorder=3, @@ -201,12 +208,13 @@ def _make_plot(out_path: Path, *, warm: bool = False) -> None: edgecolors="white", ) - x_label = (x ** (1 / 3) + cbrt_label_delta) ** 3 - ax.text( - x_label, - y, + # scatter sizes are areas in points², so the marker's radius is half of sqrt(size). + ax.annotate( m["name"], - fontsize=8.5, + (x, y), + textcoords="offset points", + xytext=(size**0.5 / 2 + _LABEL_GAP_POINTS, 0), + fontsize=9.5, fontweight="bold" if m["name"] == "semble" else "normal", color=m["color"], ha="left", @@ -220,8 +228,8 @@ def _make_plot(out_path: Path, *, warm: bool = False) -> None: ax.set_xlabel("Query latency", fontsize=10, color="#444444") if warm: - ax.set_xlim(0.01, 500) - ax.set_xticks([0.1, 1, 10, 100]) + ax.set_xlim(0.01, 1_000) + ax.set_xticks([0.1, 1, 10, 100, 1_000]) ax.set_title("Code search quality vs. latency (warm)", fontsize=12, color="#222222", pad=12) else: ax.set_xlim(5, 200_000) diff --git a/benchmarks/results/speed-14bf36d03f0c.json b/benchmarks/results/speed-14bf36d03f0c.json new file mode 100644 index 00000000..0f0f477e --- /dev/null +++ b/benchmarks/results/speed-14bf36d03f0c.json @@ -0,0 +1,2152 @@ +{ + "repos": [ + "nvm", + "libuv", + "nlohmann-json", + "messagepack-csharp", + "phoenix", + "gin", + "aeson", + "gson", + "axios", + "ktor", + "telescope.nvim", + "monolog", + "flask", + "rack", + "axum", + "http4s", + "alamofire", + "trpc", + "zls" + ], + "summary": { + "zvec-grep": { + "avg_index_ms": 3401.8, + "avg_p50_ms": 391.09, + "avg_p90_ms": 430.67, + "avg_p95_ms": 462.63, + "avg_p99_ms": 652.37 + } + }, + "results": [ + { + "repo": "nvm", + "language": "bash", + "tool": "zvec-grep", + "index_ms": 4463.5, + "p50_ms": 433.43, + "p90_ms": 470.35, + "p95_ms": 494.3, + "p99_ms": 679.01, + "latencies_ms": [ + 696.964, + 663.683, + 678.83, + 628.273, + 458.825, + 502.74, + 489.498, + 440.985, + 437.585, + 431.321, + 433.818, + 433.169, + 427.818, + 451.239, + 451.114, + 459.158, + 458.994, + 460.963, + 433.343, + 427.29, + 421.654, + 421.529, + 423.378, + 423.269, + 422.921, + 423.655, + 423.703, + 443.0, + 470.852, + 455.997, + 426.665, + 425.776, + 427.545, + 424.424, + 427.607, + 430.736, + 431.347, + 428.611, + 424.945, + 426.027, + 428.041, + 429.328, + 426.474, + 450.005, + 444.504, + 443.074, + 427.015, + 431.322, + 425.814, + 426.354, + 441.695, + 493.851, + 450.524, + 429.019, + 430.657, + 436.434, + 438.106, + 435.143, + 451.124, + 432.151, + 433.233, + 434.51, + 433.713, + 436.974, + 434.671, + 431.667, + 441.16, + 440.119, + 432.8, + 425.163, + 431.337, + 427.512, + 427.898, + 466.963, + 471.932, + 462.525, + 429.807, + 430.156, + 433.525, + 440.516, + 437.502, + 431.757, + 427.79, + 429.839, + 430.665, + 434.193, + 431.437, + 423.857, + 432.162, + 429.496, + 432.384, + 433.989, + 436.525, + 436.769, + 433.867, + 438.762, + 470.295, + 481.064, + 460.034, + 438.986 + ] + }, + { + "repo": "libuv", + "language": "c", + "tool": "zvec-grep", + "index_ms": 3597.3, + "p50_ms": 394.95, + "p90_ms": 416.91, + "p95_ms": 428.21, + "p99_ms": 465.52, + "latencies_ms": [ + 426.309, + 395.15, + 398.477, + 392.142, + 393.911, + 406.973, + 399.729, + 390.088, + 391.645, + 388.826, + 393.835, + 412.295, + 455.955, + 416.391, + 408.444, + 395.292, + 393.488, + 391.654, + 392.328, + 391.153, + 395.714, + 397.459, + 394.59, + 392.399, + 389.821, + 392.597, + 393.461, + 394.934, + 395.075, + 391.421, + 394.652, + 392.216, + 391.907, + 393.624, + 392.761, + 396.904, + 415.736, + 427.728, + 437.437, + 413.308, + 394.99, + 394.108, + 391.442, + 394.554, + 394.718, + 393.869, + 395.717, + 1187.673, + 410.131, + 391.978, + 392.561, + 396.483, + 392.668, + 395.872, + 415.314, + 413.556, + 414.071, + 393.382, + 392.843, + 411.716, + 438.394, + 413.24, + 423.419, + 393.87, + 399.205, + 403.049, + 392.477, + 393.308, + 392.289, + 393.477, + 397.674, + 394.965, + 395.153, + 423.073, + 398.118, + 394.262, + 392.631, + 396.819, + 393.889, + 397.699, + 395.623, + 390.008, + 394.191, + 393.152, + 409.983, + 458.226, + 421.55, + 412.525, + 391.352, + 395.554, + 393.117, + 392.106, + 394.892, + 397.238, + 400.451, + 391.496, + 396.235, + 395.138, + 394.362, + 393.576 + ] + }, + { + "repo": "nlohmann-json", + "language": "cpp", + "tool": "zvec-grep", + "index_ms": 3053.3, + "p50_ms": 388.61, + "p90_ms": 405.83, + "p95_ms": 415.98, + "p99_ms": 446.04, + "latencies_ms": [ + 390.493, + 410.891, + 413.226, + 448.227, + 405.411, + 389.7, + 387.628, + 388.687, + 390.673, + 391.088, + 388.799, + 386.42, + 391.234, + 390.043, + 390.974, + 390.77, + 390.287, + 389.306, + 390.374, + 391.087, + 401.577, + 394.255, + 385.826, + 384.053, + 387.06, + 386.762, + 387.408, + 411.361, + 422.383, + 418.115, + 387.528, + 394.307, + 387.866, + 385.844, + 390.851, + 391.417, + 384.767, + 388.262, + 391.788, + 393.817, + 392.284, + 433.52, + 405.148, + 395.251, + 390.404, + 388.562, + 381.539, + 398.613, + 389.366, + 391.507, + 395.45, + 382.644, + 402.182, + 446.022, + 405.294, + 403.3, + 390.168, + 389.249, + 389.473, + 386.917, + 387.843, + 387.71, + 391.201, + 382.589, + 387.363, + 382.894, + 383.396, + 382.32, + 382.872, + 381.422, + 391.284, + 381.874, + 381.643, + 378.554, + 380.954, + 388.665, + 381.344, + 385.502, + 409.588, + 415.871, + 402.785, + 384.004, + 380.401, + 384.904, + 383.406, + 385.485, + 385.84, + 381.669, + 381.683, + 382.372, + 386.68, + 382.317, + 380.921, + 384.978, + 381.883, + 385.651, + 382.234, + 380.516, + 381.551, + 384.679 + ] + }, + { + "repo": "messagepack-csharp", + "language": "csharp", + "tool": "zvec-grep", + "index_ms": 3504.6, + "p50_ms": 389.36, + "p90_ms": 404.47, + "p95_ms": 412.75, + "p99_ms": 437.91, + "latencies_ms": [ + 390.837, + 386.27, + 390.424, + 392.988, + 388.761, + 394.428, + 386.295, + 391.628, + 385.927, + 428.485, + 390.778, + 391.804, + 386.159, + 387.879, + 388.432, + 393.996, + 387.05, + 388.577, + 386.76, + 384.812, + 403.928, + 438.327, + 404.398, + 405.157, + 389.099, + 395.394, + 385.545, + 388.667, + 392.179, + 391.51, + 394.476, + 391.108, + 389.381, + 395.901, + 391.743, + 392.696, + 388.764, + 384.336, + 390.667, + 390.025, + 387.054, + 388.569, + 389.956, + 389.521, + 389.4, + 400.82, + 404.256, + 437.909, + 410.794, + 390.087, + 389.832, + 387.121, + 386.506, + 386.578, + 385.336, + 387.967, + 384.301, + 381.714, + 387.469, + 384.462, + 388.888, + 391.248, + 392.067, + 388.516, + 391.65, + 387.997, + 390.499, + 387.205, + 390.074, + 387.557, + 386.963, + 406.157, + 426.678, + 403.129, + 394.7, + 388.919, + 389.336, + 385.31, + 385.575, + 386.298, + 386.316, + 385.428, + 386.65, + 385.82, + 387.311, + 386.02, + 386.453, + 387.841, + 386.697, + 389.842, + 393.166, + 399.247, + 387.426, + 383.948, + 387.027, + 390.133, + 396.048, + 406.663, + 412.557, + 416.348 + ] + }, + { + "repo": "phoenix", + "language": "elixir", + "tool": "zvec-grep", + "index_ms": 3130.3, + "p50_ms": 383.94, + "p90_ms": 402.14, + "p95_ms": 411.35, + "p99_ms": 424.04, + "latencies_ms": [ + 382.979, + 383.696, + 383.911, + 384.393, + 383.5, + 385.283, + 383.157, + 381.622, + 382.837, + 380.328, + 382.932, + 380.905, + 384.164, + 381.612, + 400.504, + 410.835, + 415.365, + 392.963, + 378.886, + 380.222, + 393.816, + 380.887, + 387.453, + 381.486, + 382.779, + 387.517, + 379.862, + 386.317, + 386.626, + 383.726, + 398.944, + 390.773, + 380.346, + 385.777, + 380.975, + 380.594, + 384.201, + 382.519, + 384.366, + 383.433, + 400.346, + 426.005, + 400.571, + 402.109, + 383.444, + 387.7, + 385.68, + 386.127, + 385.257, + 382.56, + 385.614, + 386.46, + 383.089, + 380.559, + 423.917, + 386.765, + 380.658, + 383.941, + 383.51, + 379.405, + 382.166, + 384.434, + 382.709, + 386.014, + 381.864, + 383.956, + 411.038, + 412.074, + 406.429, + 388.7, + 386.645, + 383.172, + 382.219, + 386.052, + 382.745, + 383.894, + 383.953, + 379.66, + 384.645, + 386.418, + 385.602, + 379.755, + 380.833, + 382.611, + 383.169, + 383.548, + 378.641, + 382.18, + 379.507, + 382.373, + 386.041, + 388.959, + 402.168, + 420.095, + 407.848 + ] + }, + { + "repo": "gin", + "language": "go", + "tool": "zvec-grep", + "index_ms": 3786.3, + "p50_ms": 387.55, + "p90_ms": 404.35, + "p95_ms": 413.21, + "p99_ms": 417.94, + "latencies_ms": [ + 396.824, + 394.296, + 397.943, + 398.109, + 394.528, + 393.619, + 398.719, + 399.706, + 394.882, + 394.396, + 398.699, + 391.978, + 410.76, + 429.311, + 417.821, + 413.139, + 391.938, + 391.612, + 394.026, + 392.011, + 394.194, + 386.964, + 397.982, + 393.81, + 393.79, + 389.039, + 405.096, + 391.65, + 385.637, + 385.784, + 395.02, + 380.969, + 389.394, + 387.371, + 384.672, + 384.713, + 385.271, + 398.572, + 404.265, + 406.067, + 404.147, + 385.797, + 381.338, + 387.564, + 384.033, + 384.626, + 383.886, + 385.303, + 388.846, + 382.52, + 383.835, + 385.233, + 384.028, + 384.952, + 386.891, + 384.566, + 386.424, + 386.02, + 384.297, + 385.88, + 387.503, + 386.097, + 387.402, + 397.885, + 414.516, + 415.923, + 400.211, + 386.683, + 383.936, + 386.55, + 389.195, + 388.564, + 385.212, + 384.189, + 387.686, + 387.381, + 385.077, + 385.035, + 382.375, + 386.778, + 381.644, + 385.661, + 383.666, + 384.73, + 387.7, + 387.533, + 384.5, + 386.804, + 385.451, + 402.902, + 417.401, + 410.042, + 392.366, + 383.945, + 381.359, + 392.96, + 383.895, + 390.622, + 386.151, + 388.011 + ] + }, + { + "repo": "aeson", + "language": "haskell", + "tool": "zvec-grep", + "index_ms": 3071.5, + "p50_ms": 377.58, + "p90_ms": 392.61, + "p95_ms": 400.71, + "p99_ms": 415.15, + "latencies_ms": [ + 378.104, + 378.182, + 376.455, + 372.644, + 374.515, + 375.518, + 373.008, + 397.641, + 402.406, + 398.816, + 386.988, + 374.916, + 372.28, + 379.104, + 376.192, + 379.982, + 380.24, + 384.954, + 382.618, + 383.218, + 382.963, + 383.549, + 384.01, + 382.581, + 382.42, + 381.95, + 381.409, + 375.821, + 380.441, + 380.878, + 379.753, + 382.252, + 384.516, + 391.139, + 418.036, + 400.625, + 386.537, + 374.517, + 375.07, + 377.02, + 377.534, + 378.388, + 376.697, + 375.05, + 375.187, + 379.674, + 375.86, + 382.515, + 379.661, + 383.666, + 376.699, + 376.515, + 377.503, + 377.388, + 377.644, + 376.137, + 374.783, + 377.195, + 374.991, + 384.875, + 394.739, + 409.593, + 390.506, + 374.851, + 375.76, + 374.948, + 376.214, + 375.96, + 378.76, + 376.395, + 376.577, + 375.681, + 382.64, + 373.294, + 415.118, + 375.267, + 377.627, + 374.848, + 375.881, + 375.173, + 382.035, + 374.126, + 374.572, + 375.415, + 373.539, + 384.808, + 392.371, + 407.754, + 395.197, + 379.62, + 377.427, + 376.426, + 373.909, + 374.959, + 371.298, + 377.765, + 372.048, + 379.14, + 374.8, + 376.071 + ] + }, + { + "repo": "gson", + "language": "java", + "tool": "zvec-grep", + "index_ms": 4785.9, + "p50_ms": 409.96, + "p90_ms": 428.74, + "p95_ms": 437.49, + "p99_ms": 449.14, + "latencies_ms": [ + 429.952, + 454.546, + 424.805, + 409.953, + 412.625, + 411.747, + 406.828, + 411.131, + 408.77, + 406.34, + 419.087, + 408.85, + 417.631, + 408.326, + 408.279, + 410.363, + 406.487, + 408.074, + 405.897, + 406.261, + 406.511, + 405.136, + 409.372, + 417.663, + 424.92, + 445.682, + 427.976, + 407.333, + 410.989, + 410.511, + 411.049, + 412.03, + 405.218, + 406.588, + 407.078, + 409.975, + 414.292, + 404.489, + 404.904, + 412.551, + 405.713, + 407.904, + 413.018, + 406.089, + 406.903, + 408.147, + 409.801, + 407.615, + 434.907, + 439.781, + 429.194, + 412.534, + 407.311, + 407.772, + 408.703, + 419.752, + 410.487, + 410.607, + 413.616, + 406.575, + 407.542, + 415.975, + 413.748, + 408.95, + 405.729, + 412.672, + 406.072, + 408.453, + 408.853, + 411.079, + 410.71, + 408.751, + 428.692, + 449.081, + 428.684, + 418.873, + 413.187, + 408.986, + 407.841, + 409.727, + 412.108, + 406.753, + 410.47, + 406.548, + 411.541, + 406.441, + 410.559, + 407.317, + 404.605, + 413.362, + 407.876, + 407.146, + 408.831, + 407.432, + 411.26, + 422.111, + 432.135, + 437.367, + 441.701, + 417.335 + ] + }, + { + "repo": "axios", + "language": "javascript", + "tool": "zvec-grep", + "index_ms": 3076.1, + "p50_ms": 381.52, + "p90_ms": 399.46, + "p95_ms": 408.27, + "p99_ms": 421.12, + "latencies_ms": [ + 380.556, + 381.961, + 422.354, + 382.519, + 382.455, + 382.101, + 378.07, + 384.761, + 384.251, + 380.845, + 379.999, + 376.863, + 377.328, + 379.514, + 411.06, + 404.064, + 394.778, + 390.892, + 382.712, + 379.393, + 380.053, + 380.46, + 379.569, + 379.205, + 379.574, + 384.034, + 382.617, + 378.565, + 382.483, + 381.299, + 384.167, + 383.957, + 381.219, + 384.808, + 378.237, + 381.824, + 376.547, + 382.938, + 386.008, + 379.843, + 407.972, + 399.41, + 397.389, + 408.865, + 380.782, + 380.171, + 381.895, + 380.32, + 389.408, + 383.587, + 380.756, + 380.961, + 379.646, + 381.24, + 383.451, + 382.231, + 380.47, + 378.039, + 380.381, + 378.047, + 382.652, + 382.561, + 378.674, + 378.951, + 379.624, + 379.674, + 408.235, + 407.716, + 397.002, + 395.284, + 383.981, + 381.661, + 385.631, + 377.376, + 388.379, + 380.879, + 381.742, + 380.887, + 380.516, + 381.678, + 421.104, + 381.103, + 378.311, + 384.136, + 381.371, + 385.216, + 379.078, + 380.886, + 380.824, + 379.253, + 379.589, + 388.695, + 399.958, + 411.554, + 396.346, + 389.19, + 381.026, + 379.077, + 379.224, + 379.117 + ] + }, + { + "repo": "ktor", + "language": "kotlin", + "tool": "zvec-grep", + "index_ms": 3125.7, + "p50_ms": 390.89, + "p90_ms": 406.73, + "p95_ms": 414.36, + "p99_ms": 423.98, + "latencies_ms": [ + 392.143, + 390.017, + 400.273, + 389.868, + 392.948, + 391.23, + 387.581, + 396.416, + 393.139, + 404.727, + 417.033, + 414.222, + 406.272, + 389.523, + 390.125, + 394.597, + 390.871, + 390.805, + 393.61, + 386.613, + 391.345, + 388.292, + 392.816, + 391.972, + 393.552, + 393.405, + 387.717, + 388.171, + 391.051, + 390.975, + 392.548, + 392.056, + 390.911, + 391.884, + 400.314, + 412.669, + 419.146, + 411.321, + 388.851, + 388.867, + 396.696, + 389.753, + 390.067, + 390.859, + 386.547, + 400.559, + 388.547, + 387.748, + 429.871, + 391.29, + 392.59, + 389.342, + 388.996, + 392.304, + 390.779, + 397.189, + 389.056, + 384.894, + 389.059, + 384.112, + 411.832, + 418.791, + 403.355, + 403.856, + 382.801, + 387.418, + 387.95, + 382.842, + 388.218, + 386.097, + 391.636, + 389.76, + 384.017, + 388.373, + 389.615, + 384.648, + 390.244, + 389.046, + 390.898, + 391.312, + 393.795, + 390.86, + 391.678, + 389.431, + 390.868, + 404.48, + 410.833, + 423.92, + 403.867, + 390.773, + 393.361, + 390.632, + 389.56, + 394.866, + 391.012, + 388.773, + 387.748, + 389.094, + 387.65, + 390.886 + ] + }, + { + "repo": "telescope.nvim", + "language": "lua", + "tool": "zvec-grep", + "index_ms": 3152.1, + "p50_ms": 381.19, + "p90_ms": 399.19, + "p95_ms": 410.57, + "p99_ms": 421.05, + "latencies_ms": [ + 380.848, + 382.072, + 381.189, + 399.136, + 415.914, + 396.331, + 392.816, + 379.788, + 379.151, + 385.978, + 385.565, + 381.315, + 377.274, + 378.944, + 382.177, + 381.633, + 378.924, + 420.976, + 379.457, + 380.346, + 381.947, + 379.324, + 378.304, + 377.023, + 379.227, + 380.204, + 377.408, + 376.853, + 377.58, + 405.249, + 410.299, + 393.579, + 394.441, + 378.681, + 381.15, + 381.012, + 378.822, + 383.143, + 379.631, + 377.894, + 382.595, + 379.39, + 377.517, + 378.637, + 374.509, + 380.513, + 381.202, + 382.348, + 380.113, + 379.739, + 380.128, + 379.008, + 380.992, + 383.474, + 378.756, + 399.932, + 427.946, + 399.666, + 395.435, + 379.538, + 382.229, + 382.046, + 380.038, + 385.652, + 381.433, + 381.186, + 380.609, + 379.807, + 380.459, + 381.795, + 382.8, + 379.921, + 380.985, + 389.042, + 379.427, + 383.016, + 378.385, + 383.61, + 381.797, + 382.088, + 383.237, + 399.076, + 415.75, + 401.088, + 398.186, + 382.547, + 378.903, + 383.116, + 383.293, + 379.072, + 381.811, + 380.547, + 383.279, + 380.175, + 384.365, + 420.505, + 378.633, + 382.619, + 379.515, + 379.682 + ] + }, + { + "repo": "monolog", + "language": "php", + "tool": "zvec-grep", + "index_ms": 3092.7, + "p50_ms": 386.71, + "p90_ms": 405.07, + "p95_ms": 409.68, + "p99_ms": 429.27, + "latencies_ms": [ + 419.099, + 403.069, + 390.691, + 384.301, + 385.882, + 390.662, + 383.967, + 385.556, + 383.231, + 383.997, + 386.081, + 380.311, + 389.371, + 385.27, + 384.23, + 390.84, + 385.173, + 384.72, + 388.021, + 386.469, + 386.725, + 386.525, + 385.693, + 385.679, + 401.628, + 406.65, + 432.227, + 397.215, + 387.398, + 390.047, + 387.759, + 389.938, + 387.684, + 382.013, + 387.062, + 384.337, + 385.248, + 383.101, + 387.555, + 390.295, + 387.387, + 385.584, + 385.798, + 386.824, + 384.188, + 387.986, + 383.537, + 387.048, + 387.321, + 383.398, + 407.513, + 412.234, + 408.866, + 398.945, + 386.279, + 391.065, + 382.592, + 384.918, + 400.088, + 386.009, + 396.136, + 385.571, + 389.078, + 387.073, + 429.239, + 388.174, + 382.246, + 386.569, + 384.321, + 385.663, + 387.108, + 385.653, + 382.794, + 387.11, + 387.905, + 405.025, + 409.544, + 416.722, + 405.432, + 391.888, + 386.685, + 383.481, + 386.41, + 384.999, + 390.529, + 386.342, + 389.223, + 384.23, + 388.399, + 386.261, + 389.297, + 386.141, + 385.776, + 385.2, + 385.385, + 390.028, + 385.829, + 384.779, + 383.427, + 393.126 + ] + }, + { + "repo": "flask", + "language": "python", + "tool": "zvec-grep", + "index_ms": 3212.9, + "p50_ms": 391.5, + "p90_ms": 619.81, + "p95_ms": 629.5, + "p99_ms": 646.95, + "latencies_ms": [ + 380.351, + 380.07, + 380.508, + 377.499, + 375.266, + 378.199, + 375.131, + 374.787, + 374.749, + 378.676, + 376.903, + 378.215, + 386.074, + 402.663, + 387.773, + 380.268, + 380.258, + 378.468, + 378.373, + 420.294, + 424.101, + 404.723, + 397.37, + 384.757, + 387.772, + 524.012, + 622.916, + 586.588, + 568.803, + 561.579, + 575.57, + 557.764, + 509.91, + 599.362, + 611.903, + 616.607, + 622.925, + 658.926, + 632.357, + 630.408, + 625.889, + 617.996, + 610.683, + 647.196, + 615.523, + 632.285, + 604.294, + 611.238, + 621.644, + 618.254, + 619.866, + 607.9, + 641.022, + 619.715, + 571.595, + 462.712, + 458.823, + 415.292, + 395.456, + 392.113, + 391.314, + 393.662, + 390.322, + 390.169, + 388.006, + 387.043, + 388.73, + 393.492, + 397.091, + 402.341, + 398.567, + 380.429, + 384.954, + 379.24, + 383.935, + 383.504, + 381.148, + 381.493, + 385.195, + 409.142, + 423.766, + 398.801, + 391.503, + 380.774, + 383.844, + 383.903, + 382.08, + 382.028, + 381.52, + 380.676, + 389.329, + 383.052, + 383.071, + 413.98, + 381.368, + 383.36, + 396.584, + 391.24, + 382.826, + 392.069, + 387.906, + 386.867, + 384.895, + 386.521, + 401.339 + ] + }, + { + "repo": "rack", + "language": "ruby", + "tool": "zvec-grep", + "index_ms": 2842.0, + "p50_ms": 391.21, + "p90_ms": 435.04, + "p95_ms": 575.73, + "p99_ms": 586.68, + "latencies_ms": [ + 387.569, + 387.568, + 385.998, + 386.277, + 386.902, + 385.197, + 397.866, + 406.266, + 390.351, + 417.811, + 387.04, + 390.428, + 384.353, + 386.695, + 383.725, + 383.566, + 381.36, + 379.515, + 403.519, + 414.168, + 396.628, + 383.509, + 375.531, + 376.949, + 376.02, + 375.123, + 376.879, + 379.283, + 375.262, + 379.219, + 378.554, + 379.843, + 377.886, + 379.131, + 387.637, + 382.077, + 378.238, + 383.932, + 383.253, + 380.418, + 382.664, + 399.956, + 407.21, + 403.466, + 412.321, + 413.303, + 399.722, + 406.451, + 398.436, + 404.174, + 400.868, + 385.358, + 384.817, + 385.209, + 384.891, + 386.805, + 386.122, + 382.915, + 415.427, + 384.547, + 389.464, + 383.457, + 402.486, + 403.01, + 407.487, + 398.167, + 400.251, + 403.956, + 396.998, + 418.778, + 458.783, + 415.633, + 411.408, + 404.534, + 398.591, + 395.27, + 400.85, + 391.982, + 432.407, + 402.596, + 404.91, + 412.585, + 474.428, + 583.648, + 611.242, + 576.006, + 576.173, + 575.391, + 583.152, + 514.841 + ] + }, + { + "repo": "axum", + "language": "rust", + "tool": "zvec-grep", + "index_ms": 4218.3, + "p50_ms": 408.79, + "p90_ms": 657.71, + "p95_ms": 676.83, + "p99_ms": 681.03, + "latencies_ms": [ + 712.231, + 676.721, + 680.262, + 680.332, + 680.71, + 638.322, + 606.725, + 621.245, + 610.164, + 657.593, + 673.597, + 652.257, + 626.289, + 613.474, + 403.526, + 400.187, + 396.302, + 393.227, + 388.856, + 391.37, + 426.607, + 391.408, + 429.92, + 602.745, + 606.473, + 588.353, + 586.058, + 579.125, + 595.041, + 565.807, + 653.326, + 620.307, + 591.03, + 596.043, + 598.257, + 601.106, + 599.291, + 611.946, + 604.72, + 656.03, + 662.758, + 649.549, + 658.75, + 613.505, + 636.806, + 667.918, + 678.929, + 617.511, + 594.799, + 479.475, + 401.588, + 398.35, + 390.141, + 390.223, + 387.431, + 391.89, + 395.393, + 396.722, + 390.318, + 388.346, + 392.643, + 401.403, + 388.333, + 391.426, + 392.329, + 392.861, + 393.837, + 407.593, + 431.166, + 427.75, + 405.672, + 391.901, + 389.384, + 392.685, + 392.181, + 392.407, + 392.112, + 391.275, + 392.037, + 391.493, + 392.489, + 395.087, + 391.668, + 389.972, + 396.339, + 391.368, + 392.98, + 400.478, + 409.991, + 393.291, + 393.762, + 411.239, + 407.184, + 435.12, + 420.203, + 412.857, + 391.235, + 398.575, + 403.162, + 387.646 + ] + }, + { + "repo": "http4s", + "language": "scala", + "tool": "zvec-grep", + "index_ms": 3270.7, + "p50_ms": 403.47, + "p90_ms": 436.7, + "p95_ms": 443.61, + "p99_ms": 479.66, + "latencies_ms": [ + 409.475, + 408.139, + 420.189, + 406.864, + 400.451, + 400.465, + 398.521, + 407.366, + 414.881, + 431.193, + 449.8, + 436.788, + 403.323, + 402.519, + 404.857, + 408.338, + 401.699, + 400.54, + 404.885, + 404.338, + 436.691, + 425.324, + 427.686, + 417.114, + 417.903, + 405.005, + 404.654, + 407.974, + 402.251, + 401.654, + 403.609, + 432.041, + 442.486, + 461.292, + 471.773, + 426.555, + 428.296, + 422.583, + 413.373, + 427.385, + 443.28, + 411.397, + 408.428, + 404.657, + 399.385, + 441.53, + 396.658, + 406.387, + 400.642, + 402.927, + 401.498, + 398.81, + 394.982, + 402.534, + 401.374, + 406.11, + 399.686, + 427.745, + 453.012, + 410.022, + 442.258, + 409.031, + 398.982, + 400.63, + 401.436, + 399.037, + 405.552, + 399.758, + 396.183, + 395.463, + 423.737, + 397.361, + 396.088, + 395.157, + 397.418, + 1260.204, + 394.062, + 393.9, + 393.659, + 414.192, + 429.338, + 431.459, + 411.162, + 392.697, + 397.019, + 394.919, + 397.562, + 400.348, + 398.802, + 400.744, + 398.245, + 399.72, + 398.556, + 402.652, + 400.173, + 401.384, + 398.656, + 397.992, + 398.609, + 401.496 + ] + }, + { + "repo": "alamofire", + "language": "swift", + "tool": "zvec-grep", + "index_ms": 2938.3, + "p50_ms": 383.75, + "p90_ms": 403.69, + "p95_ms": 416.83, + "p99_ms": 437.21, + "latencies_ms": [ + 403.661, + 386.918, + 385.793, + 383.265, + 383.868, + 401.663, + 385.382, + 386.639, + 381.541, + 384.62, + 384.209, + 419.119, + 397.089, + 401.069, + 381.396, + 383.302, + 381.441, + 389.971, + 383.458, + 380.806, + 384.509, + 392.599, + 429.905, + 457.218, + 437.01, + 403.969, + 379.743, + 381.736, + 382.434, + 401.193, + 398.128, + 399.428, + 388.851, + 385.64, + 381.915, + 385.625, + 380.973, + 382.938, + 382.725, + 385.086, + 385.015, + 382.792, + 382.924, + 382.756, + 382.164, + 384.888, + 383.094, + 381.171, + 397.898, + 403.652, + 416.709, + 412.787, + 383.692, + 391.504, + 383.766, + 385.891, + 386.364, + 384.761, + 382.69, + 381.919, + 383.417, + 382.709, + 383.41, + 382.043, + 381.808, + 385.8, + 382.572, + 382.251, + 381.636, + 383.468, + 383.2, + 386.126, + 384.881, + 391.117, + 410.957, + 425.989, + 405.826, + 381.716, + 380.618, + 391.794, + 384.134, + 383.515, + 383.513, + 381.168, + 381.044, + 383.391, + 381.525, + 394.567, + 383.373, + 381.873, + 385.163, + 382.971, + 383.732, + 383.01, + 384.099, + 382.514, + 381.319, + 382.967, + 383.142, + 396.566 + ] + }, + { + "repo": "trpc", + "language": "typescript", + "tool": "zvec-grep", + "index_ms": 3122.5, + "p50_ms": 393.05, + "p90_ms": 410.7, + "p95_ms": 425.75, + "p99_ms": 440.29, + "latencies_ms": [ + 394.19, + 393.495, + 394.596, + 396.476, + 399.689, + 394.8, + 389.665, + 390.746, + 394.005, + 394.008, + 397.195, + 393.581, + 395.894, + 394.488, + 393.364, + 391.882, + 393.451, + 424.607, + 440.71, + 421.728, + 395.192, + 393.521, + 393.173, + 395.658, + 392.77, + 394.658, + 392.295, + 390.588, + 390.615, + 391.775, + 394.116, + 391.626, + 393.183, + 392.073, + 391.071, + 390.982, + 389.221, + 392.244, + 389.753, + 390.557, + 392.622, + 389.645, + 405.64, + 440.133, + 417.051, + 404.966, + 392.313, + 391.935, + 392.271, + 394.96, + 396.344, + 393.693, + 392.405, + 397.457, + 391.823, + 393.826, + 425.681, + 390.597, + 390.598, + 392.927, + 389.101, + 391.274, + 399.229, + 390.919, + 394.533, + 394.713, + 389.986, + 409.719, + 426.972, + 427.572, + 401.112, + 392.771, + 392.669, + 392.435, + 394.049, + 395.54, + 391.981, + 389.887, + 388.062, + 391.028, + 392.617, + 388.542, + 387.654, + 389.045, + 388.202, + 396.048, + 388.324, + 389.191, + 391.205, + 393.885, + 390.931, + 394.152, + 404.139, + 410.393, + 440.286, + 413.416, + 389.146, + 390.825, + 390.144, + 389.372 + ] + }, + { + "repo": "zls", + "language": "zig", + "tool": "zvec-grep", + "index_ms": 3191.0, + "p50_ms": 387.27, + "p90_ms": 403.5, + "p95_ms": 415.9, + "p99_ms": 426.76, + "latencies_ms": [ + 379.605, + 384.628, + 385.214, + 383.76, + 381.233, + 380.759, + 386.522, + 379.679, + 381.01, + 381.183, + 403.204, + 422.583, + 398.399, + 387.579, + 380.187, + 381.707, + 379.789, + 381.851, + 377.84, + 377.728, + 380.068, + 375.888, + 377.507, + 376.248, + 377.068, + 377.484, + 374.878, + 376.912, + 375.403, + 377.566, + 378.39, + 379.374, + 379.715, + 374.077, + 376.592, + 374.885, + 399.408, + 415.734, + 400.635, + 384.659, + 383.118, + 380.423, + 390.026, + 401.642, + 386.732, + 409.599, + 395.934, + 385.095, + 389.315, + 393.005, + 388.46, + 385.24, + 381.211, + 393.904, + 401.601, + 402.801, + 401.636, + 398.245, + 389.194, + 381.259, + 384.795, + 406.177, + 400.664, + 426.549, + 418.985, + 392.42, + 391.89, + 389.641, + 388.365, + 390.712, + 384.96, + 388.497, + 382.416, + 389.551, + 390.966, + 396.233, + 384.02, + 387.296, + 388.376, + 390.781, + 387.236, + 385.044, + 386.225, + 383.279, + 386.595, + 392.787, + 394.123, + 420.36, + 447.938, + 409.163, + 407.279, + 391.349, + 389.733, + 388.39, + 390.004, + 389.582, + 391.031, + 384.575, + 396.166, + 390.076 + ] + } + ] +} diff --git a/benchmarks/results/zvecgrep-6bbbd322fd6e.json b/benchmarks/results/zvecgrep-6bbbd322fd6e.json new file mode 100644 index 00000000..83129f04 --- /dev/null +++ b/benchmarks/results/zvecgrep-6bbbd322fd6e.json @@ -0,0 +1,450 @@ +{ + "tool": "zvec-grep", + "note": "hybrid FTS + local/potion-code-16m-v2 vector search", + "repos": [ + { + "repo": "abseil-cpp", + "language": "cpp", + "ndcg10": 0.6546, + "p50_ms": 509.5, + "index_ms": 10431.0 + }, + { + "repo": "aeson", + "language": "haskell", + "ndcg10": 0.824, + "p50_ms": 368.3, + "index_ms": 2946.0 + }, + { + "repo": "aiohttp", + "language": "python", + "ndcg10": 0.5369, + "p50_ms": 367.8, + "index_ms": 3905.0 + }, + { + "repo": "alamofire", + "language": "swift", + "ndcg10": 0.8628, + "p50_ms": 363.9, + "index_ms": 3151.0 + }, + { + "repo": "axios", + "language": "javascript", + "ndcg10": 0.7372, + "p50_ms": 369.3, + "index_ms": 2964.0 + }, + { + "repo": "axum", + "language": "rust", + "ndcg10": 0.3621, + "p50_ms": 370.6, + "index_ms": 3514.0 + }, + { + "repo": "bash-it", + "language": "bash", + "ndcg10": 0.6521, + "p50_ms": 424.2, + "index_ms": 3612.0 + }, + { + "repo": "bats-core", + "language": "bash", + "ndcg10": 0.9346, + "p50_ms": 362.6, + "index_ms": 2820.0 + }, + { + "repo": "cats", + "language": "scala", + "ndcg10": 0.7982, + "p50_ms": 380.5, + "index_ms": 3546.0 + }, + { + "repo": "chi", + "language": "go", + "ndcg10": 0.5775, + "p50_ms": 374.2, + "index_ms": 3203.0 + }, + { + "repo": "circe", + "language": "scala", + "ndcg10": 0.7694, + "p50_ms": 364.3, + "index_ms": 3112.0 + }, + { + "repo": "click", + "language": "python", + "ndcg10": 0.9262, + "p50_ms": 363.5, + "index_ms": 3101.0 + }, + { + "repo": "cobra", + "language": "go", + "ndcg10": 0.4911, + "p50_ms": 388.5, + "index_ms": 3364.0 + }, + { + "repo": "commons-lang", + "language": "java", + "ndcg10": 0.921, + "p50_ms": 391.8, + "index_ms": 6108.0 + }, + { + "repo": "curl", + "language": "c", + "ndcg10": 0.5571, + "p50_ms": 395.5, + "index_ms": 5931.0 + }, + { + "repo": "dapper", + "language": "csharp", + "ndcg10": 0.6943, + "p50_ms": 369.1, + "index_ms": 3209.0 + }, + { + "repo": "ecto", + "language": "elixir", + "ndcg10": 0.9154, + "p50_ms": 363.9, + "index_ms": 3119.0 + }, + { + "repo": "exposed", + "language": "kotlin", + "ndcg10": 0.5447, + "p50_ms": 378.9, + "index_ms": 3243.0 + }, + { + "repo": "express", + "language": "javascript", + "ndcg10": 0.6908, + "p50_ms": 364.7, + "index_ms": 2868.0 + }, + { + "repo": "fastapi", + "language": "python", + "ndcg10": 0.7121, + "p50_ms": 369.6, + "index_ms": 3403.0 + }, + { + "repo": "flask", + "language": "python", + "ndcg10": 0.6759, + "p50_ms": 367.6, + "index_ms": 3282.0 + }, + { + "repo": "fmtlib", + "language": "cpp", + "ndcg10": 0.6858, + "p50_ms": 370.1, + "index_ms": 3534.0 + }, + { + "repo": "gin", + "language": "go", + "ndcg10": 0.467, + "p50_ms": 376.8, + "index_ms": 3584.0 + }, + { + "repo": "gson", + "language": "java", + "ndcg10": 0.5945, + "p50_ms": 396.3, + "index_ms": 5062.0 + }, + { + "repo": "guzzle", + "language": "php", + "ndcg10": 0.7296, + "p50_ms": 366.2, + "index_ms": 2698.0 + }, + { + "repo": "http4s", + "language": "scala", + "ndcg10": 0.8284, + "p50_ms": 395.0, + "index_ms": 3537.0 + }, + { + "repo": "httpx", + "language": "python", + "ndcg10": 0.8073, + "p50_ms": 369.5, + "index_ms": 3276.0 + }, + { + "repo": "jackson-databind", + "language": "java", + "ndcg10": 0.5396, + "p50_ms": 425.6, + "index_ms": 8729.0 + }, + { + "repo": "kotlinx-coroutines", + "language": "kotlin", + "ndcg10": 0.7758, + "p50_ms": 379.6, + "index_ms": 3264.0 + }, + { + "repo": "ktor", + "language": "kotlin", + "ndcg10": 0.5634, + "p50_ms": 382.2, + "index_ms": 2931.0 + }, + { + "repo": "laravel-framework", + "language": "php", + "ndcg10": 0.4005, + "p50_ms": 539.5, + "index_ms": 6189.0 + }, + { + "repo": "lazy.nvim", + "language": "lua", + "ndcg10": 0.5788, + "p50_ms": 373.6, + "index_ms": 3038.0 + }, + { + "repo": "libuv", + "language": "c", + "ndcg10": 0.5145, + "p50_ms": 380.7, + "index_ms": 3956.0 + }, + { + "repo": "messagepack-csharp", + "language": "csharp", + "ndcg10": 0.618, + "p50_ms": 382.2, + "index_ms": 3470.0 + }, + { + "repo": "mini.nvim", + "language": "lua", + "ndcg10": 0.9077, + "p50_ms": 372.7, + "index_ms": 4228.0 + }, + { + "repo": "model2vec", + "language": "python", + "ndcg10": 0.5493, + "p50_ms": 374.1, + "index_ms": 3113.0 + }, + { + "repo": "monolog", + "language": "php", + "ndcg10": 0.6183, + "p50_ms": 382.1, + "index_ms": 3267.0 + }, + { + "repo": "newtonsoft-json", + "language": "csharp", + "ndcg10": 0.5148, + "p50_ms": 392.4, + "index_ms": 4080.0 + }, + { + "repo": "nlohmann-json", + "language": "cpp", + "ndcg10": 0.6894, + "p50_ms": 376.3, + "index_ms": 3489.0 + }, + { + "repo": "nvm", + "language": "bash", + "ndcg10": 0.5882, + "p50_ms": 414.9, + "index_ms": 3546.0 + }, + { + "repo": "pandoc", + "language": "haskell", + "ndcg10": 0.5358, + "p50_ms": 389.8, + "index_ms": 4473.0 + }, + { + "repo": "phoenix", + "language": "elixir", + "ndcg10": 0.7809, + "p50_ms": 372.9, + "index_ms": 3251.0 + }, + { + "repo": "plug", + "language": "elixir", + "ndcg10": 0.7377, + "p50_ms": 367.4, + "index_ms": 3422.0 + }, + { + "repo": "pydantic", + "language": "python", + "ndcg10": 0.6067, + "p50_ms": 385.9, + "index_ms": 4221.0 + }, + { + "repo": "rack", + "language": "ruby", + "ndcg10": 0.8241, + "p50_ms": 371.7, + "index_ms": 3161.0 + }, + { + "repo": "rails", + "language": "ruby", + "ndcg10": 0.5234, + "p50_ms": 435.9, + "index_ms": 3326.0 + }, + { + "repo": "redis", + "language": "c", + "ndcg10": 0.7218, + "p50_ms": 414.2, + "index_ms": 8300.0 + }, + { + "repo": "redux", + "language": "javascript", + "ndcg10": 0.8531, + "p50_ms": 369.7, + "index_ms": 3018.0 + }, + { + "repo": "requests", + "language": "python", + "ndcg10": 0.8341, + "p50_ms": 370.9, + "index_ms": 3126.0 + }, + { + "repo": "serde", + "language": "rust", + "ndcg10": 0.5274, + "p50_ms": 404.6, + "index_ms": 4045.0 + }, + { + "repo": "sinatra", + "language": "ruby", + "ndcg10": 0.9262, + "p50_ms": 362.1, + "index_ms": 3060.0 + }, + { + "repo": "snapkit", + "language": "swift", + "ndcg10": 0.662, + "p50_ms": 362.9, + "index_ms": 3047.0 + }, + { + "repo": "starlette", + "language": "python", + "ndcg10": 0.6899, + "p50_ms": 366.1, + "index_ms": 3245.0 + }, + { + "repo": "telescope.nvim", + "language": "lua", + "ndcg10": 0.7227, + "p50_ms": 369.6, + "index_ms": 3241.0 + }, + { + "repo": "tokio", + "language": "rust", + "ndcg10": 0.7328, + "p50_ms": 406.4, + "index_ms": 5662.0 + }, + { + "repo": "trpc", + "language": "typescript", + "ndcg10": 0.6276, + "p50_ms": 374.8, + "index_ms": 3060.0 + }, + { + "repo": "vapor", + "language": "swift", + "ndcg10": 0.6035, + "p50_ms": 394.2, + "index_ms": 3433.0 + }, + { + "repo": "vitest", + "language": "typescript", + "ndcg10": 0.4447, + "p50_ms": 400.3, + "index_ms": 3881.0 + }, + { + "repo": "xmonad", + "language": "haskell", + "ndcg10": 0.7069, + "p50_ms": 362.4, + "index_ms": 2927.0 + }, + { + "repo": "zig", + "language": "zig", + "ndcg10": 0.7601, + "p50_ms": 431.1, + "index_ms": 10665.0 + }, + { + "repo": "zig-clap", + "language": "zig", + "ndcg10": 0.6755, + "p50_ms": 365.8, + "index_ms": 3024.0 + }, + { + "repo": "zls", + "language": "zig", + "ndcg10": 0.5574, + "p50_ms": 368.9, + "index_ms": 3620.0 + }, + { + "repo": "zod", + "language": "typescript", + "ndcg10": 0.3548, + "p50_ms": 388.6, + "index_ms": 4017.0 + } + ], + "avg_ndcg10": 0.6701, + "avg_p50_ms": 385.6, + "avg_index_ms": 3953.0 +} diff --git a/benchmarks/speed_benchmark.py b/benchmarks/speed_benchmark.py index 196af935..4343926b 100644 --- a/benchmarks/speed_benchmark.py +++ b/benchmarks/speed_benchmark.py @@ -1,8 +1,9 @@ import argparse +import shutil import subprocess import sys import time -from collections.abc import Sequence +from collections.abc import Callable, Sequence from dataclasses import dataclass import numpy as np @@ -47,6 +48,8 @@ _TOP_K = 10 _COLGREP = "colgrep" +_ZG = "zg" +_ZG_EMBEDDING = "local/potion-code-16m-v2" @dataclass(frozen=True) @@ -95,8 +98,12 @@ def __init__(self, model: SentenceTransformer, max_seq_length: int = 512) -> Non self._model = model self._model.max_seq_length = max_seq_length - def encode(self, texts: Sequence[str], use_multiprocessing: object = _UNSET) -> np.ndarray: - """Encode with the query prompt only when use_multiprocessing wasn't passed.""" + def encode(self, texts: Sequence[str], use_multiprocessing: object = _UNSET, **_kwargs: object) -> np.ndarray: + """Encode with the query prompt only when use_multiprocessing wasn't passed. + + Extra keyword arguments are swallowed: semble passes max_length, which does not apply here because + max_seq_length is pinned on the wrapped model in __init__. + """ text_list = list(texts) if use_multiprocessing is _UNSET: return self._model.encode(text_list, prompt_name="query", batch_size=1) # type: ignore[return-value] @@ -203,6 +210,37 @@ def _bench_colgrep(spec: RepoSpec, tasks: list[Task]) -> tuple[float, tuple[floa return index_ms, tuple(latencies) +def _bench_zvecgrep(spec: RepoSpec, tasks: list[Task]) -> tuple[float, tuple[float, ...]] | None: + """Index a repo with zvec-grep and measure query latency; return (index_ms, latencies_ms) or None on failure.""" + shutil.rmtree(spec.benchmark_dir / ".zvec-grep", ignore_errors=True) + started = time.perf_counter() + proc = subprocess.run( + [_ZG, "index", str(spec.benchmark_dir), "--embedding", _ZG_EMBEDDING, "--mode", "direct"], + capture_output=True, + text=True, + timeout=1800, + ) + index_ms = (time.perf_counter() - started) * 1000 + if proc.returncode != 0: + print(f" WARNING: zg index failed: {proc.stderr.strip()[:200]}", file=sys.stderr) + shutil.rmtree(spec.benchmark_dir / ".zvec-grep", ignore_errors=True) + return None + latencies: list[float] = [] + for task in tasks: + for _ in range(5): + started = time.perf_counter() + subprocess.run( + [_ZG, "query", task.query, "--limit", str(_TOP_K), "--mode", "direct", "--refresh", "off"], + cwd=spec.benchmark_dir, + capture_output=True, + text=True, + timeout=60, + ) + latencies.append((time.perf_counter() - started) * 1000) + shutil.rmtree(spec.benchmark_dir / ".zvec-grep", ignore_errors=True) + return index_ms, tuple(latencies) + + def _bench_ripgrep(spec: RepoSpec, tasks: list[Task]) -> tuple[float, tuple[float, ...]]: """Measure ripgrep query latency (no index step); return (0.0, latencies_ms).""" latencies: list[float] = [] @@ -214,6 +252,39 @@ def _bench_ripgrep(spec: RepoSpec, tasks: list[Task]) -> tuple[float, tuple[floa return 0.0, tuple(latencies) +# Tools benched by dispatching on the name; semble, bm25, coderankembed and ripgrep are special-cased in _run_tool +# because they need an extra argument (a model, semble's index) or return no index time. +_TOOL_BENCHES: dict[str, Callable[[RepoSpec, list[Task]], tuple[float, tuple[float, ...]] | None]] = { + "colgrep": _bench_colgrep, + "zvec-grep": _bench_zvecgrep, +} + +# Benchmark order: semble first (bm25 reuses its chunks), then the indexed baselines, then ripgrep. +_ALL_TOOLS: tuple[str, ...] = ("semble", "bm25", "coderankembed", "colgrep", "zvec-grep", "ripgrep") + + +def _run_tool( + tool: str, + spec: RepoSpec, + tasks: list[Task], + semble_index: "SembleIndex | None", + cre_model: "_AsymmetricWrapper | None", +) -> tuple[tuple[float | None, tuple[float, ...]] | None, "SembleIndex | None"]: + """Benchmark one tool on one repo; return ((index_ms, latencies_ms) or None, semble's index for bm25 to reuse).""" + if tool == "semble": + index_ms, semble_index, latencies_ms = _bench_semble(spec, tasks) + return (index_ms, latencies_ms), semble_index + if tool == "bm25": + assert semble_index is not None + return _bench_bm25(semble_index, tasks), semble_index + if tool == "coderankembed": + assert cre_model is not None + return _bench_coderankembed(spec, tasks, cre_model), semble_index + if tool == "ripgrep": + return (None, _bench_ripgrep(spec, tasks)[1]), semble_index + return _TOOL_BENCHES[tool](spec, tasks), semble_index + + def _fmt_stats(result: ToolResult) -> str: """Format p50/p90/p95/p99 columns for a single result row.""" return f"{result.p50_ms:>7.2f}ms {result.p90_ms:>7.2f}ms {result.p95_ms:>7.2f}ms {result.p99_ms:>7.2f}ms" @@ -240,29 +311,40 @@ def main() -> None: """Run cold-start index + query latency benchmark over a curated 1-per-language subset.""" parser = argparse.ArgumentParser(description="Benchmark cold-start index time and query latency per language.") parser.add_argument( - "--semble-only", action="store_true", help="Only benchmark semble (skip BM25, CodeRankEmbed, ColGREP, ripgrep)." + "--tools", + nargs="+", + default=list(_ALL_TOOLS), + choices=list(_ALL_TOOLS), + metavar="TOOL", + help=f"Tools to benchmark (default: all). Choices: {', '.join(_ALL_TOOLS)}. bm25 reuses semble's chunks, " + "so selecting it also runs semble.", ) + parser.add_argument("--repo", action="append", help="Limit to these repos (repeatable); default: all 19.") args = parser.parse_args() + tools = [tool for tool in _ALL_TOOLS if tool in args.tools] + if "bm25" in tools and "semble" not in tools: + tools.insert(0, "semble") + repos = [r for r in _REPOS if r in args.repo] if args.repo else list(_REPOS) + specs = available_repo_specs() all_tasks = load_tasks(repo_specs=specs) - repo_tasks: dict[str, list[Task]] = {repo: [t for t in all_tasks if t.repo == repo] for repo in _REPOS} + repo_tasks: dict[str, list[Task]] = {repo: [t for t in all_tasks if t.repo == repo] for repo in repos} - print("Loading semble model...", file=sys.stderr) - started = time.perf_counter() - load_model(DEFAULT_MODEL_NAME) # warms semble's internal model cache so repo #1 isn't penalized - print(f" loaded in {(time.perf_counter() - started) * 1000:.0f}ms", file=sys.stderr) + if {"semble", "bm25"} & set(tools): + print("Loading semble model...", file=sys.stderr) + started = time.perf_counter() + load_model(DEFAULT_MODEL_NAME) # warms semble's internal model cache so repo #1 isn't penalized + print(f" loaded in {(time.perf_counter() - started) * 1000:.0f}ms", file=sys.stderr) cre_model = None - if not args.semble_only: + if "coderankembed" in tools: print("Loading CodeRankEmbed...", file=sys.stderr) started = time.perf_counter() cre_model = _AsymmetricWrapper(SentenceTransformer(_CRE_MODEL_NAME, trust_remote_code=True, device="cpu")) print(f" loaded in {(time.perf_counter() - started) * 1000:.0f}ms", file=sys.stderr) print(file=sys.stderr) - tools = ["semble"] if args.semble_only else ["semble", "bm25", "coderankembed", "colgrep", "ripgrep"] - print( f"{'Repo':<22} {'Language':<14} {'Tool':<16} {'Index':>10} {'p50':>8} {'p90':>8} {'p95':>8} {'p99':>8}", file=sys.stderr, @@ -271,53 +353,28 @@ def main() -> None: all_results: list[ToolResult] = [] - for repo in _REPOS: + for repo in repos: spec = specs[repo] tasks = repo_tasks[repo] + semble_index: SembleIndex | None = None + repo_label = repo - index_ms, semble_index, latencies_ms = _bench_semble(spec, tasks) - result = ToolResult( - repo=repo, language=spec.language, tool="semble", index_ms=index_ms, latencies_ms=latencies_ms - ) - all_results.append(result) - print(f"{repo:<22} {spec.language:<14} {'semble':<16} {index_ms:>8.0f}ms {_fmt_stats(result)}", file=sys.stderr) - if cre_model is None: - continue - - bm25_index_ms, latencies_ms = _bench_bm25(semble_index, tasks) - result = ToolResult( - repo=repo, language=spec.language, tool="bm25", index_ms=bm25_index_ms, latencies_ms=latencies_ms - ) - all_results.append(result) - print(f"{'':22} {spec.language:<14} {'bm25':<16} {bm25_index_ms:>8.0f}ms {_fmt_stats(result)}", file=sys.stderr) + for tool in tools: + outcome, semble_index = _run_tool(tool, spec, tasks, semble_index, cre_model) - cre_index_ms, latencies_ms = _bench_coderankembed(spec, tasks, cre_model) - result = ToolResult( - repo=repo, language=spec.language, tool="coderankembed", index_ms=cre_index_ms, latencies_ms=latencies_ms - ) - all_results.append(result) - print( - f"{'':22} {spec.language:<14} {'coderankembed':<16} {cre_index_ms:>8.0f}ms {_fmt_stats(result)}", - file=sys.stderr, - ) + if outcome is None: + print(f"{repo_label:<22} {spec.language:<14} {tool:<16} {'N/A (failed)':>18}", file=sys.stderr) + repo_label = "" + continue - colgrep_result = _bench_colgrep(spec, tasks) - if colgrep_result is not None: - index_ms, latencies_ms = colgrep_result + index_ms_opt, latencies_ms = outcome result = ToolResult( - repo=repo, language=spec.language, tool="colgrep", index_ms=index_ms, latencies_ms=latencies_ms + repo=repo, language=spec.language, tool=tool, index_ms=index_ms_opt, latencies_ms=latencies_ms ) all_results.append(result) - print( - f"{'':22} {spec.language:<14} {'colgrep':<16} {index_ms:>8.0f}ms {_fmt_stats(result)}", file=sys.stderr - ) - else: - print(f"{'':22} {spec.language:<14} {'colgrep':<16} {'N/A (unsupported)':>18}", file=sys.stderr) - - _, latencies_ms = _bench_ripgrep(spec, tasks) - result = ToolResult(repo=repo, language=spec.language, tool="ripgrep", index_ms=None, latencies_ms=latencies_ms) - all_results.append(result) - print(f"{'':22} {spec.language:<14} {'ripgrep':<16} {'N/A':>10} {_fmt_stats(result)}", file=sys.stderr) + idx_col = f"{index_ms_opt:>8.0f}ms" if index_ms_opt is not None else f"{'N/A':>10}" + print(f"{repo_label:<22} {spec.language:<14} {tool:<16} {idx_col} {_fmt_stats(result)}", file=sys.stderr) + repo_label = "" summary = _build_summary(all_results, tools)