diff --git a/README.md b/README.md index ed72821c..e9d12bc8 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ -Semble is a code search library built for agents. It returns the exact code snippets they need instantly, using ~99% fewer tokens than grep+read. Indexing and searching a full codebase end-to-end takes under a second for most repos, matching the retrieval quality of a code-specialized transformer while indexing ~340x faster and querying ~17x faster (see [benchmarks](#benchmarks)). Everything runs on CPU with no API keys, GPU, or external services. Use it as an MCP server, a CLI tool via AGENTS.md, or a dedicated sub-agent, and any coding agent (Claude Code, Cursor, Codex, OpenCode, etc.) gets instant access to any repo. +Semble is a code search library built for agents. It returns the exact code snippets they need instantly, using ~99% fewer tokens than grep+read. Indexing and searching a full codebase end-to-end takes under a second for most repos, matching the retrieval quality of a code-specialized transformer while indexing ~380x faster and querying ~17x faster (see [benchmarks](#benchmarks)). Everything runs on CPU with no API keys, GPU, or external services. Use it as an MCP server, a CLI tool via AGENTS.md, or a dedicated sub-agent, and any coding agent (Claude Code, Cursor, Codex, OpenCode, etc.) gets instant access to any repo. @@ -249,7 +249,7 @@ We benchmark quality and speed across ~1,250 queries over 63 repositories in 19 -The quality benchmark (left) scores retrieval quality (NDCG@10) against total latency; semble matches the quality of the 137M-parameter [CodeRankEmbed](https://huggingface.co/nomic-ai/CodeRankEmbed) while indexing 340x faster. The token efficiency benchmark (right) measures how many tokens each method needs to reach a given recall level; semble uses 99% fewer tokens on average and hits 97% recall at only 2k tokens, while grep+read needs a full 100k context window to reach 85%. See [benchmarks](benchmarks/README.md) for per-language results, ablations, and full methodology. +The quality benchmark (left) scores retrieval quality (NDCG@10) against total latency; semble matches the quality of the 137M-parameter [CodeRankEmbed](https://huggingface.co/nomic-ai/CodeRankEmbed) while indexing 380x faster. The token efficiency benchmark (right) measures how many tokens each method needs to reach a given recall level; semble uses 99% fewer tokens on average and hits 97% recall at only 2k tokens, while grep+read needs a full 100k context window to reach 85%. See [benchmarks](benchmarks/README.md) for per-language results, ablations, and full methodology. ## How it works diff --git a/assets/images/speed_vs_ndcg_cold.png b/assets/images/speed_vs_ndcg_cold.png index 323a1a99..2d2fa5f1 100644 Binary files a/assets/images/speed_vs_ndcg_cold.png and b/assets/images/speed_vs_ndcg_cold.png differ diff --git a/benchmarks/README.md b/benchmarks/README.md index ec0d8d59..7501600e 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -17,7 +17,7 @@ Quality and speed across all methods. | Method | NDCG@10 | Index | Query p50 | | -------------------- | --------: | ---------: | ----------: | -| **semble** | **0.854** | **344 ms** | **0.91 ms** | +| **semble** | **0.854** | **306 ms** | **0.91 ms** | | CodeRankEmbed | 0.839 | 116 s | 16 ms | | ColGREP | 0.693 | 5.4 s | 122 ms | | BM25 | 0.673 | 47 ms | 0.17 ms | @@ -32,7 +32,7 @@ Quality and speed across all methods. | :-----------------------------------------------------------------: | :-----------------------------------------------------------------: | | _Time to first result (index + query) vs NDCG@10_ | _Query latency on a warm index vs NDCG@10_ | -semble matches the NDCG@10 of the 137M-param CodeRankEmbed while winning index time by ~340x and query latency by ~17x. +semble matches the NDCG@10 of the 137M-param CodeRankEmbed while winning index time by ~380x and query latency by ~17x. NDCG@10 is averaged across all queries. Speed numbers use one repo per language, CPU only: cold-start index time and warm query p50 (median across 5 consecutive runs). @@ -188,6 +188,7 @@ Full runs write to `benchmarks/results/semble-hybrid-.json`. ```bash uv run python -m benchmarks.speed_benchmark +uv run python -m benchmarks.speed_benchmark --semble-only # skip the baselines ``` Writes to `benchmarks/results/speed-.json`. diff --git a/benchmarks/plot.py b/benchmarks/plot.py index 9ef2c07d..d177684d 100644 --- a/benchmarks/plot.py +++ b/benchmarks/plot.py @@ -96,7 +96,7 @@ class _Method(TypedDict): { "name": "semble", "ndcg10": 0.8544, - "index_ms": 343.5, + "index_ms": 305.6, "query_p50_ms": 0.91, "color": "#1a5fa8", "params_m": 16, diff --git a/benchmarks/results/speed-eb6676ad14ec.json b/benchmarks/results/speed-eb6676ad14ec.json new file mode 100644 index 00000000..58f6448c --- /dev/null +++ b/benchmarks/results/speed-eb6676ad14ec.json @@ -0,0 +1,2152 @@ +{ + "repos": [ + "nvm", + "libuv", + "nlohmann-json", + "messagepack-csharp", + "phoenix", + "gin", + "aeson", + "gson", + "axios", + "ktor", + "telescope.nvim", + "monolog", + "flask", + "rack", + "axum", + "http4s", + "alamofire", + "trpc", + "zls" + ], + "summary": { + "semble": { + "avg_index_ms": 305.6, + "avg_p50_ms": 0.95, + "avg_p90_ms": 5.38, + "avg_p95_ms": 6.8, + "avg_p99_ms": 10.07 + } + }, + "results": [ + { + "repo": "nvm", + "language": "bash", + "tool": "semble", + "index_ms": 145.0, + "p50_ms": 0.61, + "p90_ms": 0.71, + "p95_ms": 0.89, + "p99_ms": 1.27, + "latencies_ms": [ + 1.266, + 0.68, + 0.612, + 0.631, + 0.602, + 0.624, + 0.773, + 0.704, + 0.615, + 0.572, + 0.666, + 0.63, + 0.621, + 0.636, + 0.624, + 0.638, + 0.625, + 0.611, + 0.625, + 1.787, + 0.905, + 0.707, + 0.63, + 0.613, + 0.599, + 0.539, + 0.518, + 0.51, + 0.55, + 0.508, + 0.595, + 0.561, + 0.558, + 0.552, + 0.551, + 0.618, + 0.599, + 0.612, + 0.595, + 0.6, + 1.005, + 0.713, + 0.625, + 0.607, + 0.604, + 0.527, + 0.5, + 0.496, + 0.504, + 0.507, + 0.698, + 0.661, + 0.648, + 0.632, + 0.628, + 0.654, + 0.678, + 0.616, + 0.598, + 0.599, + 0.703, + 0.642, + 0.604, + 0.6, + 0.605, + 0.643, + 0.604, + 0.695, + 0.693, + 0.72, + 0.662, + 0.604, + 0.572, + 0.565, + 0.555, + 0.628, + 0.613, + 0.611, + 0.635, + 0.936, + 0.884, + 0.621, + 0.55, + 0.517, + 0.497, + 0.62, + 0.599, + 0.587, + 0.712, + 0.766, + 0.586, + 0.548, + 0.6, + 0.553, + 0.532, + 0.644, + 0.639, + 0.579, + 0.548, + 0.539 + ] + }, + { + "repo": "libuv", + "language": "c", + "tool": "semble", + "index_ms": 563.9, + "p50_ms": 0.82, + "p90_ms": 1.25, + "p95_ms": 1.49, + "p99_ms": 1.8, + "latencies_ms": [ + 1.814, + 1.411, + 1.318, + 1.255, + 1.36, + 0.792, + 0.725, + 0.713, + 0.699, + 0.705, + 1.002, + 0.895, + 0.897, + 0.923, + 0.924, + 1.799, + 1.609, + 1.552, + 1.499, + 1.485, + 0.913, + 0.956, + 0.974, + 0.931, + 0.866, + 0.812, + 0.739, + 0.72, + 0.764, + 0.761, + 0.766, + 0.681, + 0.666, + 0.663, + 0.659, + 0.955, + 0.866, + 0.846, + 0.853, + 0.882, + 0.704, + 0.625, + 0.62, + 0.608, + 0.599, + 0.753, + 0.681, + 0.68, + 0.675, + 0.733, + 1.02, + 0.972, + 0.934, + 0.896, + 0.887, + 1.019, + 0.945, + 0.947, + 0.931, + 0.942, + 1.155, + 1.05, + 1.041, + 1.012, + 1.022, + 0.82, + 0.747, + 0.751, + 0.747, + 0.718, + 0.883, + 0.815, + 0.821, + 0.825, + 0.795, + 0.59, + 0.542, + 0.524, + 0.517, + 0.512, + 1.247, + 1.124, + 1.095, + 1.13, + 1.146, + 0.856, + 0.754, + 0.732, + 0.733, + 0.759, + 0.823, + 0.73, + 0.685, + 0.671, + 0.667, + 0.892, + 0.854, + 0.789, + 0.757, + 0.775 + ] + }, + { + "repo": "nlohmann-json", + "language": "cpp", + "tool": "semble", + "index_ms": 368.5, + "p50_ms": 1.03, + "p90_ms": 5.08, + "p95_ms": 5.34, + "p99_ms": 5.86, + "latencies_ms": [ + 1.458, + 1.202, + 1.139, + 1.092, + 1.11, + 0.843, + 0.901, + 0.909, + 0.832, + 0.749, + 0.806, + 0.762, + 0.762, + 0.768, + 0.768, + 1.086, + 1.024, + 0.988, + 1.104, + 1.061, + 1.227, + 1.173, + 1.2, + 1.289, + 1.179, + 1.008, + 0.95, + 1.028, + 1.003, + 0.985, + 1.109, + 1.067, + 1.049, + 1.08, + 1.118, + 1.356, + 1.371, + 1.287, + 1.221, + 1.205, + 0.997, + 0.936, + 0.926, + 0.932, + 0.918, + 1.011, + 0.985, + 0.947, + 0.94, + 0.92, + 0.973, + 0.93, + 0.934, + 0.927, + 0.942, + 0.908, + 0.933, + 0.891, + 0.837, + 0.822, + 1.672, + 1.616, + 1.599, + 1.62, + 1.688, + 5.852, + 5.307, + 5.306, + 5.484, + 5.401, + 1.158, + 1.004, + 0.947, + 0.926, + 0.966, + 0.841, + 0.782, + 0.766, + 0.767, + 0.78, + 1.016, + 1.015, + 1.012, + 0.972, + 0.952, + 4.964, + 4.566, + 4.561, + 4.538, + 4.812, + 5.811, + 5.339, + 6.205, + 5.169, + 5.169, + 5.071, + 4.372, + 4.464, + 4.422, + 4.421 + ] + }, + { + "repo": "messagepack-csharp", + "language": "csharp", + "tool": "semble", + "index_ms": 515.0, + "p50_ms": 4.95, + "p90_ms": 7.09, + "p95_ms": 7.21, + "p99_ms": 7.46, + "latencies_ms": [ + 7.197, + 6.706, + 6.663, + 6.592, + 6.58, + 1.265, + 1.186, + 1.265, + 1.232, + 1.318, + 1.268, + 1.159, + 1.156, + 1.13, + 1.133, + 1.404, + 1.326, + 1.335, + 1.296, + 1.291, + 0.881, + 0.82, + 0.831, + 0.811, + 0.814, + 1.135, + 1.063, + 1.06, + 1.052, + 1.043, + 1.238, + 1.27, + 1.222, + 1.22, + 1.198, + 1.024, + 0.956, + 0.963, + 1.014, + 1.239, + 6.777, + 7.04, + 7.057, + 7.457, + 6.984, + 1.26, + 1.012, + 0.947, + 0.949, + 0.989, + 6.908, + 7.089, + 6.813, + 6.642, + 6.632, + 7.252, + 6.697, + 6.695, + 6.764, + 6.836, + 7.573, + 7.272, + 7.159, + 7.073, + 7.232, + 5.923, + 5.329, + 5.345, + 5.178, + 5.209, + 5.336, + 4.657, + 4.587, + 4.695, + 4.694, + 6.742, + 6.189, + 6.146, + 6.315, + 6.444, + 5.486, + 4.954, + 5.076, + 4.947, + 5.002, + 7.069, + 7.099, + 7.093, + 7.207, + 7.178, + 1.361, + 1.215, + 1.135, + 1.057, + 1.089, + 6.457, + 5.773, + 5.741, + 5.813, + 5.908 + ] + }, + { + "repo": "phoenix", + "language": "elixir", + "tool": "semble", + "index_ms": 299.1, + "p50_ms": 0.81, + "p90_ms": 5.0, + "p95_ms": 6.05, + "p99_ms": 6.7, + "latencies_ms": [ + 1.389, + 1.116, + 1.057, + 1.058, + 1.052, + 0.788, + 0.749, + 0.846, + 0.815, + 0.778, + 0.754, + 0.687, + 0.678, + 0.676, + 0.681, + 0.713, + 0.682, + 0.68, + 0.678, + 0.681, + 0.857, + 0.861, + 0.794, + 0.752, + 0.734, + 5.862, + 5.181, + 5.071, + 5.133, + 5.134, + 0.704, + 0.59, + 0.574, + 0.566, + 0.572, + 0.75, + 0.7, + 0.679, + 0.668, + 0.675, + 1.04, + 0.979, + 0.986, + 1.025, + 1.148, + 0.774, + 0.72, + 0.705, + 0.705, + 0.698, + 0.928, + 0.892, + 0.883, + 0.956, + 1.021, + 0.772, + 0.742, + 0.686, + 0.66, + 0.64, + 0.742, + 0.701, + 0.679, + 0.685, + 0.677, + 0.779, + 0.727, + 0.714, + 0.705, + 0.734, + 0.876, + 0.819, + 0.81, + 0.833, + 0.838, + 1.147, + 1.083, + 1.072, + 1.059, + 1.053, + 4.865, + 4.331, + 4.31, + 4.324, + 4.411, + 6.827, + 6.481, + 6.541, + 6.689, + 6.593, + 4.889, + 4.408, + 4.497, + 4.471, + 4.484 + ] + }, + { + "repo": "gin", + "language": "go", + "tool": "semble", + "index_ms": 285.4, + "p50_ms": 0.78, + "p90_ms": 3.64, + "p95_ms": 4.73, + "p99_ms": 5.0, + "latencies_ms": [ + 0.873, + 0.658, + 0.605, + 0.596, + 0.59, + 0.52, + 0.461, + 0.45, + 0.454, + 0.455, + 0.826, + 0.768, + 0.806, + 0.729, + 0.705, + 0.818, + 0.784, + 0.793, + 0.775, + 0.768, + 0.786, + 0.721, + 0.71, + 0.724, + 0.711, + 4.823, + 4.234, + 4.235, + 4.303, + 4.363, + 0.76, + 0.649, + 0.633, + 0.618, + 0.611, + 0.591, + 0.573, + 0.549, + 0.571, + 0.552, + 0.491, + 0.462, + 0.457, + 0.455, + 0.463, + 0.813, + 0.701, + 0.703, + 0.716, + 0.7, + 0.631, + 0.582, + 0.58, + 0.653, + 0.617, + 0.928, + 0.859, + 0.959, + 0.964, + 0.889, + 0.864, + 0.82, + 0.814, + 0.829, + 0.818, + 0.883, + 0.849, + 0.837, + 0.807, + 0.799, + 0.786, + 0.758, + 0.762, + 0.746, + 0.73, + 0.947, + 0.909, + 1.055, + 0.99, + 0.942, + 3.57, + 3.083, + 3.105, + 3.1, + 3.098, + 5.151, + 4.729, + 4.801, + 4.822, + 4.995, + 3.47, + 3.475, + 3.311, + 3.347, + 3.386, + 0.861, + 0.738, + 0.723, + 0.717, + 0.708 + ] + }, + { + "repo": "aeson", + "language": "haskell", + "tool": "semble", + "index_ms": 245.3, + "p50_ms": 4.0, + "p90_ms": 8.05, + "p95_ms": 12.74, + "p99_ms": 21.61, + "latencies_ms": [ + 1.151, + 0.945, + 0.871, + 0.878, + 0.881, + 4.996, + 4.513, + 4.432, + 4.351, + 5.325, + 4.501, + 3.923, + 3.964, + 3.85, + 3.886, + 4.536, + 3.979, + 3.933, + 4.007, + 3.994, + 4.406, + 3.998, + 4.018, + 4.013, + 4.173, + 5.01, + 4.479, + 4.427, + 4.435, + 4.539, + 4.76, + 4.037, + 3.948, + 3.939, + 3.943, + 22.004, + 21.532, + 21.484, + 21.499, + 21.607, + 5.215, + 4.678, + 4.555, + 4.583, + 4.52, + 5.135, + 4.64, + 4.708, + 4.753, + 4.654, + 4.459, + 3.916, + 3.949, + 3.918, + 3.88, + 0.758, + 0.713, + 0.759, + 0.766, + 0.724, + 0.836, + 0.8, + 0.799, + 0.816, + 0.826, + 0.791, + 0.777, + 0.752, + 0.751, + 0.758, + 4.545, + 3.92, + 3.94, + 3.868, + 3.822, + 0.862, + 0.761, + 0.747, + 0.747, + 0.738, + 12.276, + 11.468, + 11.46, + 11.741, + 11.508, + 7.67, + 6.74, + 6.703, + 6.49, + 6.606, + 0.834, + 0.727, + 0.692, + 0.71, + 0.693, + 4.51, + 4.104, + 4.154, + 4.13, + 3.966 + ] + }, + { + "repo": "gson", + "language": "java", + "tool": "semble", + "index_ms": 635.1, + "p50_ms": 1.69, + "p90_ms": 9.63, + "p95_ms": 10.72, + "p99_ms": 13.51, + "latencies_ms": [ + 14.719, + 13.503, + 13.478, + 13.297, + 13.155, + 2.041, + 1.763, + 1.699, + 1.808, + 1.896, + 1.53, + 1.424, + 1.452, + 1.488, + 1.483, + 9.511, + 8.812, + 8.756, + 8.808, + 8.923, + 7.791, + 7.471, + 7.16, + 7.146, + 7.1, + 1.691, + 1.538, + 1.529, + 1.509, + 1.555, + 1.607, + 1.494, + 1.474, + 1.451, + 1.478, + 1.362, + 1.34, + 1.297, + 1.272, + 1.257, + 1.282, + 1.382, + 1.301, + 1.205, + 1.235, + 0.979, + 0.925, + 0.905, + 0.901, + 0.888, + 1.395, + 1.327, + 1.315, + 1.355, + 1.337, + 1.393, + 1.337, + 1.26, + 1.246, + 1.244, + 1.509, + 1.496, + 1.684, + 1.481, + 1.447, + 1.618, + 1.544, + 1.535, + 1.695, + 1.617, + 1.694, + 1.687, + 1.729, + 1.696, + 1.712, + 2.032, + 2.01, + 2.005, + 1.835, + 1.895, + 9.577, + 9.082, + 8.944, + 9.118, + 8.799, + 7.719, + 7.303, + 7.72, + 7.471, + 7.271, + 10.501, + 10.413, + 10.592, + 10.061, + 10.18, + 8.617, + 7.686, + 7.697, + 7.64, + 7.529 + ] + }, + { + "repo": "axios", + "language": "javascript", + "tool": "semble", + "index_ms": 71.7, + "p50_ms": 0.67, + "p90_ms": 2.34, + "p95_ms": 2.54, + "p99_ms": 2.78, + "latencies_ms": [ + 0.892, + 0.676, + 0.652, + 0.632, + 0.617, + 0.538, + 0.499, + 0.497, + 0.49, + 0.488, + 0.688, + 0.614, + 0.593, + 0.589, + 0.578, + 0.667, + 0.62, + 0.617, + 0.627, + 0.63, + 2.849, + 2.257, + 2.208, + 2.173, + 2.258, + 2.207, + 1.732, + 1.688, + 1.641, + 1.672, + 2.781, + 2.36, + 2.347, + 2.341, + 2.373, + 0.628, + 0.563, + 0.538, + 0.532, + 0.531, + 0.603, + 0.565, + 0.558, + 0.552, + 0.56, + 0.657, + 0.642, + 0.633, + 0.641, + 0.64, + 0.63, + 0.605, + 0.572, + 0.568, + 0.565, + 0.618, + 0.592, + 0.588, + 0.586, + 0.582, + 0.706, + 0.676, + 0.68, + 0.69, + 0.673, + 0.645, + 0.686, + 0.683, + 0.658, + 0.631, + 0.593, + 0.574, + 0.56, + 0.563, + 0.57, + 0.709, + 0.701, + 0.692, + 0.693, + 0.665, + 0.746, + 0.806, + 0.779, + 0.737, + 0.726, + 2.637, + 2.178, + 2.205, + 2.253, + 2.14, + 2.535, + 2.159, + 2.184, + 2.115, + 2.141, + 2.738, + 2.323, + 2.558, + 2.417, + 2.254 + ] + }, + { + "repo": "ktor", + "language": "kotlin", + "tool": "semble", + "index_ms": 211.3, + "p50_ms": 1.15, + "p90_ms": 5.69, + "p95_ms": 6.2, + "p99_ms": 6.46, + "latencies_ms": [ + 7.132, + 6.242, + 6.303, + 6.449, + 6.332, + 1.154, + 1.047, + 1.029, + 1.043, + 1.028, + 0.889, + 0.857, + 0.836, + 0.84, + 0.838, + 1.122, + 1.058, + 1.053, + 1.065, + 1.068, + 0.992, + 0.964, + 0.959, + 0.938, + 0.94, + 0.93, + 0.935, + 0.91, + 0.974, + 1.023, + 0.85, + 0.795, + 0.788, + 0.778, + 0.78, + 0.7, + 0.68, + 0.664, + 0.687, + 0.665, + 3.462, + 3.369, + 3.411, + 3.288, + 3.279, + 0.983, + 0.899, + 0.888, + 0.892, + 0.885, + 1.116, + 1.066, + 1.048, + 1.044, + 1.041, + 5.255, + 5.273, + 5.648, + 5.379, + 5.526, + 5.563, + 5.015, + 4.97, + 5.08, + 5.132, + 4.076, + 3.631, + 3.644, + 3.662, + 3.502, + 5.347, + 5.653, + 4.91, + 4.695, + 4.828, + 4.421, + 3.833, + 3.786, + 3.575, + 3.578, + 6.197, + 6.145, + 6.095, + 6.034, + 6.019, + 1.261, + 1.194, + 1.214, + 1.204, + 1.151, + 0.964, + 0.917, + 0.904, + 0.908, + 0.908, + 4.082, + 3.566, + 3.571, + 3.644, + 3.722 + ] + }, + { + "repo": "telescope.nvim", + "language": "lua", + "tool": "semble", + "index_ms": 219.3, + "p50_ms": 0.81, + "p90_ms": 1.0, + "p95_ms": 1.32, + "p99_ms": 3.91, + "latencies_ms": [ + 1.191, + 0.962, + 0.881, + 0.885, + 0.876, + 0.898, + 0.812, + 0.774, + 0.763, + 0.758, + 0.891, + 0.857, + 1.004, + 0.909, + 0.847, + 0.917, + 0.862, + 0.847, + 0.825, + 0.84, + 1.019, + 1.062, + 0.998, + 0.95, + 0.927, + 0.933, + 0.865, + 0.867, + 0.832, + 0.856, + 0.756, + 0.688, + 0.714, + 0.801, + 0.719, + 0.861, + 0.805, + 0.793, + 0.783, + 0.786, + 0.819, + 0.77, + 0.741, + 0.743, + 0.739, + 4.498, + 3.909, + 3.906, + 3.862, + 3.823, + 0.909, + 0.869, + 0.827, + 0.807, + 0.785, + 0.831, + 0.783, + 0.764, + 0.765, + 0.764, + 0.852, + 0.788, + 0.784, + 0.911, + 0.9, + 1.022, + 0.936, + 0.92, + 0.909, + 0.911, + 0.839, + 0.797, + 0.8, + 0.798, + 0.889, + 0.81, + 0.755, + 0.736, + 0.736, + 0.736, + 0.795, + 0.771, + 0.778, + 0.767, + 0.796, + 0.81, + 0.723, + 0.779, + 0.752, + 0.711, + 0.738, + 0.688, + 0.68, + 0.683, + 0.681, + 0.843, + 0.796, + 0.757, + 0.751, + 0.75 + ] + }, + { + "repo": "monolog", + "language": "php", + "tool": "semble", + "index_ms": 195.1, + "p50_ms": 1.02, + "p90_ms": 3.59, + "p95_ms": 3.7, + "p99_ms": 4.09, + "latencies_ms": [ + 1.115, + 1.007, + 0.949, + 0.85, + 0.826, + 1.06, + 1.087, + 1.02, + 1.015, + 0.962, + 0.912, + 0.816, + 0.789, + 0.78, + 0.775, + 0.769, + 0.718, + 0.725, + 0.718, + 0.708, + 4.315, + 3.589, + 3.675, + 3.7, + 3.642, + 3.82, + 3.373, + 3.334, + 3.229, + 3.358, + 4.083, + 3.387, + 3.498, + 3.754, + 3.487, + 1.002, + 0.908, + 0.974, + 0.947, + 0.917, + 1.137, + 1.075, + 1.058, + 1.053, + 1.301, + 1.114, + 1.056, + 1.006, + 0.962, + 0.95, + 0.983, + 0.94, + 0.963, + 1.032, + 1.007, + 1.002, + 1.07, + 1.015, + 0.919, + 0.893, + 0.824, + 0.8, + 0.783, + 0.774, + 0.764, + 0.858, + 0.834, + 0.817, + 0.822, + 0.813, + 1.026, + 1.078, + 1.063, + 0.987, + 0.989, + 1.023, + 0.98, + 0.969, + 0.976, + 1.11, + 1.22, + 1.188, + 1.144, + 1.097, + 1.094, + 1.032, + 0.995, + 1.035, + 1.042, + 1.023, + 4.005, + 3.594, + 3.496, + 3.385, + 3.352, + 3.654, + 3.26, + 3.186, + 3.24, + 3.249 + ] + }, + { + "repo": "flask", + "language": "python", + "tool": "semble", + "index_ms": 139.3, + "p50_ms": 0.49, + "p90_ms": 2.47, + "p95_ms": 3.32, + "p99_ms": 3.78, + "latencies_ms": [ + 0.774, + 0.535, + 0.488, + 0.478, + 0.475, + 0.501, + 0.447, + 0.434, + 0.442, + 0.455, + 0.434, + 0.403, + 0.397, + 0.391, + 0.386, + 0.42, + 0.382, + 0.381, + 0.367, + 0.374, + 0.465, + 0.41, + 0.41, + 0.405, + 0.404, + 0.4, + 0.361, + 0.357, + 0.357, + 0.356, + 0.499, + 0.45, + 0.44, + 0.437, + 0.427, + 0.433, + 0.396, + 0.391, + 0.394, + 0.39, + 0.415, + 0.447, + 0.509, + 0.432, + 0.423, + 0.563, + 0.498, + 0.48, + 0.487, + 0.486, + 0.87, + 0.687, + 0.659, + 0.644, + 0.633, + 0.742, + 0.697, + 0.683, + 0.691, + 0.682, + 0.76, + 0.758, + 0.726, + 0.722, + 0.75, + 0.669, + 0.682, + 0.593, + 0.555, + 0.546, + 0.776, + 0.729, + 0.717, + 0.715, + 0.711, + 3.794, + 3.334, + 3.334, + 3.257, + 3.181, + 2.637, + 2.226, + 2.163, + 2.137, + 2.17, + 3.797, + 3.38, + 3.329, + 3.27, + 3.278, + 0.409, + 0.371, + 0.361, + 0.359, + 0.36, + 0.324, + 0.31, + 0.308, + 0.316, + 0.315, + 0.742, + 0.744, + 0.69, + 0.672, + 0.664 + ] + }, + { + "repo": "rack", + "language": "ruby", + "tool": "semble", + "index_ms": 125.5, + "p50_ms": 0.6, + "p90_ms": 3.71, + "p95_ms": 5.08, + "p99_ms": 5.68, + "latencies_ms": [ + 0.769, + 0.575, + 0.559, + 0.551, + 0.547, + 0.706, + 0.646, + 0.631, + 0.634, + 0.646, + 0.495, + 0.449, + 0.437, + 0.435, + 0.441, + 0.602, + 0.587, + 0.532, + 0.509, + 0.513, + 3.705, + 3.178, + 3.093, + 3.167, + 3.084, + 6.426, + 5.593, + 5.516, + 5.443, + 5.55, + 4.638, + 3.755, + 3.801, + 3.681, + 3.741, + 0.679, + 0.584, + 0.571, + 0.558, + 0.566, + 0.58, + 0.553, + 0.542, + 0.539, + 0.537, + 0.618, + 0.578, + 0.6, + 0.643, + 0.595, + 0.549, + 0.523, + 0.515, + 0.527, + 0.514, + 0.618, + 0.589, + 0.589, + 0.578, + 0.587, + 0.577, + 0.656, + 0.732, + 0.607, + 0.595, + 0.595, + 0.533, + 0.514, + 0.503, + 0.511, + 0.604, + 0.58, + 0.583, + 0.594, + 0.561, + 0.702, + 0.66, + 0.649, + 0.646, + 0.648, + 3.341, + 2.91, + 2.842, + 2.889, + 2.873, + 3.262, + 2.887, + 2.854, + 2.84, + 2.781 + ] + }, + { + "repo": "axum", + "language": "rust", + "tool": "semble", + "index_ms": 225.2, + "p50_ms": 0.92, + "p90_ms": 4.48, + "p95_ms": 4.94, + "p99_ms": 6.27, + "latencies_ms": [ + 1.193, + 0.87, + 0.821, + 0.801, + 0.805, + 0.905, + 0.853, + 0.839, + 0.962, + 0.929, + 7.176, + 6.264, + 6.248, + 6.149, + 6.041, + 0.882, + 0.768, + 0.72, + 0.692, + 0.694, + 0.671, + 0.628, + 0.642, + 0.623, + 0.632, + 0.741, + 0.696, + 0.673, + 0.673, + 0.68, + 4.861, + 4.385, + 4.397, + 4.451, + 4.475, + 1.183, + 1.057, + 1.019, + 1.071, + 1.083, + 0.841, + 0.793, + 0.774, + 0.774, + 0.78, + 0.874, + 0.827, + 0.798, + 0.803, + 0.796, + 0.882, + 0.821, + 0.785, + 0.784, + 0.789, + 3.516, + 3.452, + 3.531, + 3.414, + 3.397, + 4.88, + 4.602, + 4.58, + 4.55, + 4.432, + 4.383, + 3.909, + 3.938, + 3.808, + 3.882, + 0.904, + 0.799, + 0.771, + 0.752, + 0.759, + 0.927, + 0.899, + 0.862, + 0.952, + 0.906, + 0.697, + 0.662, + 0.651, + 0.648, + 0.641, + 3.411, + 3.407, + 3.376, + 3.384, + 3.37, + 3.561, + 3.523, + 3.619, + 3.572, + 3.46, + 3.81, + 3.761, + 3.696, + 3.664, + 3.995 + ] + }, + { + "repo": "http4s", + "language": "scala", + "tool": "semble", + "index_ms": 597.1, + "p50_ms": 5.16, + "p90_ms": 7.46, + "p95_ms": 7.87, + "p99_ms": 8.09, + "latencies_ms": [ + 7.001, + 6.007, + 6.014, + 6.008, + 5.89, + 1.139, + 1.035, + 1.012, + 1.095, + 1.122, + 5.946, + 5.565, + 5.797, + 5.67, + 5.618, + 5.67, + 5.184, + 5.214, + 5.145, + 5.295, + 1.182, + 1.073, + 0.949, + 0.917, + 0.889, + 0.886, + 0.826, + 0.822, + 0.911, + 0.852, + 8.445, + 7.914, + 7.848, + 7.956, + 7.868, + 1.063, + 0.973, + 0.963, + 0.967, + 0.968, + 0.991, + 0.965, + 0.964, + 0.977, + 1.043, + 1.088, + 1.01, + 0.987, + 0.986, + 0.995, + 6.238, + 5.878, + 5.756, + 5.778, + 5.834, + 1.166, + 1.054, + 0.981, + 0.983, + 0.97, + 7.892, + 7.774, + 7.604, + 7.457, + 7.501, + 1.168, + 1.075, + 1.069, + 1.018, + 1.116, + 8.083, + 7.368, + 7.435, + 7.37, + 7.423, + 0.889, + 0.775, + 0.751, + 0.748, + 0.748, + 6.208, + 5.686, + 5.808, + 5.686, + 5.866, + 5.322, + 5.303, + 5.367, + 5.334, + 5.262, + 5.04, + 4.983, + 5.064, + 4.935, + 5.555, + 6.14, + 5.631, + 5.544, + 5.62, + 5.787 + ] + }, + { + "repo": "alamofire", + "language": "swift", + "tool": "semble", + "index_ms": 249.5, + "p50_ms": 0.96, + "p90_ms": 4.99, + "p95_ms": 6.02, + "p99_ms": 7.17, + "latencies_ms": [ + 1.211, + 0.969, + 0.931, + 0.935, + 0.907, + 1.017, + 0.957, + 1.084, + 1.044, + 0.979, + 0.802, + 0.742, + 0.755, + 0.738, + 0.74, + 0.818, + 0.772, + 0.753, + 0.743, + 0.861, + 0.863, + 0.779, + 0.764, + 0.768, + 0.744, + 0.561, + 0.539, + 0.52, + 0.514, + 0.514, + 0.633, + 0.57, + 0.563, + 0.553, + 0.552, + 0.882, + 0.802, + 0.783, + 0.794, + 0.773, + 0.835, + 0.807, + 0.794, + 0.779, + 0.762, + 0.834, + 0.791, + 0.9, + 0.87, + 0.846, + 5.967, + 5.558, + 5.525, + 5.395, + 5.388, + 1.089, + 0.942, + 0.916, + 0.919, + 0.914, + 7.477, + 7.052, + 7.052, + 7.167, + 7.147, + 4.943, + 4.474, + 4.417, + 4.354, + 4.409, + 3.973, + 3.578, + 3.583, + 3.449, + 3.536, + 3.761, + 3.259, + 3.362, + 3.209, + 3.229, + 4.649, + 4.267, + 4.144, + 4.179, + 4.131, + 1.2, + 1.033, + 0.989, + 0.951, + 0.95, + 1.043, + 1.003, + 0.995, + 0.998, + 0.996, + 4.622, + 4.19, + 4.31, + 4.139, + 4.289 + ] + }, + { + "repo": "trpc", + "language": "typescript", + "tool": "semble", + "index_ms": 171.9, + "p50_ms": 3.03, + "p90_ms": 4.06, + "p95_ms": 4.85, + "p99_ms": 5.58, + "latencies_ms": [ + 4.816, + 4.093, + 4.007, + 3.975, + 3.986, + 0.877, + 0.894, + 0.861, + 0.881, + 0.905, + 1.008, + 0.978, + 0.953, + 0.937, + 0.948, + 3.686, + 3.58, + 3.587, + 3.702, + 3.739, + 0.989, + 0.951, + 0.921, + 0.904, + 0.896, + 0.941, + 0.919, + 0.975, + 1.005, + 0.924, + 0.797, + 0.747, + 0.738, + 0.746, + 0.726, + 1.04, + 1.015, + 1.013, + 1.048, + 1.096, + 3.501, + 3.537, + 3.55, + 3.949, + 3.858, + 3.719, + 3.631, + 3.63, + 3.762, + 3.676, + 3.835, + 3.746, + 3.777, + 3.796, + 3.696, + 1.026, + 0.964, + 0.892, + 0.858, + 0.844, + 3.537, + 3.154, + 3.098, + 3.101, + 3.207, + 3.422, + 3.026, + 2.923, + 2.986, + 2.887, + 3.486, + 3.036, + 2.99, + 2.886, + 2.928, + 3.385, + 2.823, + 2.825, + 2.677, + 2.712, + 3.275, + 2.731, + 2.749, + 2.966, + 2.76, + 5.592, + 5.584, + 5.569, + 5.425, + 5.42, + 4.139, + 4.06, + 4.096, + 4.026, + 4.148, + 3.691, + 3.771, + 3.627, + 3.659, + 3.631 + ] + }, + { + "repo": "zls", + "language": "zig", + "tool": "semble", + "index_ms": 543.1, + "p50_ms": 0.9, + "p90_ms": 1.36, + "p95_ms": 2.16, + "p99_ms": 9.18, + "latencies_ms": [ + 1.808, + 1.642, + 1.498, + 1.511, + 1.536, + 9.635, + 8.912, + 9.171, + 9.139, + 8.805, + 1.013, + 0.876, + 0.819, + 0.736, + 0.714, + 1.091, + 1.138, + 1.244, + 1.081, + 1.01, + 1.087, + 1.239, + 1.097, + 1.015, + 0.917, + 1.193, + 1.083, + 1.049, + 1.074, + 1.07, + 0.837, + 0.763, + 0.87, + 0.805, + 0.827, + 0.836, + 0.841, + 0.761, + 0.757, + 0.743, + 1.16, + 1.027, + 0.901, + 0.933, + 0.909, + 1.35, + 1.211, + 1.213, + 1.258, + 1.245, + 0.865, + 0.789, + 0.8, + 0.789, + 0.791, + 0.851, + 0.814, + 0.867, + 0.902, + 0.877, + 1.006, + 0.92, + 0.908, + 0.902, + 1.011, + 0.776, + 0.695, + 0.677, + 0.662, + 0.661, + 0.803, + 0.792, + 0.793, + 0.758, + 0.737, + 0.837, + 0.739, + 0.777, + 0.785, + 0.76, + 0.693, + 0.648, + 0.655, + 0.65, + 0.645, + 1.145, + 1.036, + 1.036, + 1.016, + 1.112, + 0.985, + 0.884, + 0.844, + 0.83, + 0.836, + 0.865, + 0.799, + 0.957, + 0.936, + 0.993 + ] + } + ] +} diff --git a/benchmarks/speed_benchmark.py b/benchmarks/speed_benchmark.py index 0e56be67..196af935 100644 --- a/benchmarks/speed_benchmark.py +++ b/benchmarks/speed_benchmark.py @@ -1,3 +1,4 @@ +import argparse import subprocess import sys import time @@ -237,6 +238,12 @@ def _build_summary(results: list[ToolResult], tools: list[str]) -> dict[str, obj def main() -> None: """Run cold-start index + query latency benchmark over a curated 1-per-language subset.""" + parser = argparse.ArgumentParser(description="Benchmark cold-start index time and query latency per language.") + parser.add_argument( + "--semble-only", action="store_true", help="Only benchmark semble (skip BM25, CodeRankEmbed, ColGREP, ripgrep)." + ) + args = parser.parse_args() + specs = available_repo_specs() all_tasks = load_tasks(repo_specs=specs) repo_tasks: dict[str, list[Task]] = {repo: [t for t in all_tasks if t.repo == repo] for repo in _REPOS} @@ -246,13 +253,15 @@ def main() -> None: load_model(DEFAULT_MODEL_NAME) # warms semble's internal model cache so repo #1 isn't penalized print(f" loaded in {(time.perf_counter() - started) * 1000:.0f}ms", file=sys.stderr) - print("Loading CodeRankEmbed...", file=sys.stderr) - started = time.perf_counter() - cre_model = _AsymmetricWrapper(SentenceTransformer(_CRE_MODEL_NAME, trust_remote_code=True, device="cpu")) - print(f" loaded in {(time.perf_counter() - started) * 1000:.0f}ms", file=sys.stderr) + cre_model = None + if not args.semble_only: + print("Loading CodeRankEmbed...", file=sys.stderr) + started = time.perf_counter() + cre_model = _AsymmetricWrapper(SentenceTransformer(_CRE_MODEL_NAME, trust_remote_code=True, device="cpu")) + print(f" loaded in {(time.perf_counter() - started) * 1000:.0f}ms", file=sys.stderr) print(file=sys.stderr) - tools = ["semble", "bm25", "coderankembed", "colgrep", "ripgrep"] + tools = ["semble"] if args.semble_only else ["semble", "bm25", "coderankembed", "colgrep", "ripgrep"] print( f"{'Repo':<22} {'Language':<14} {'Tool':<16} {'Index':>10} {'p50':>8} {'p90':>8} {'p95':>8} {'p99':>8}", @@ -272,6 +281,8 @@ def main() -> None: ) all_results.append(result) print(f"{repo:<22} {spec.language:<14} {'semble':<16} {index_ms:>8.0f}ms {_fmt_stats(result)}", file=sys.stderr) + if cre_model is None: + continue bm25_index_ms, latencies_ms = _bench_bm25(semble_index, tasks) result = ToolResult( diff --git a/src/semble/chunking/chunking.py b/src/semble/chunking/chunking.py index e9ddf71c..af1b706e 100644 --- a/src/semble/chunking/chunking.py +++ b/src/semble/chunking/chunking.py @@ -1,4 +1,6 @@ import logging +import re +from bisect import bisect_left from semble.chunking.core import chunk, chunk_lines from semble.types import Chunk @@ -22,6 +24,8 @@ def chunk_source(source: str, file_path: str, language: str | None) -> list[Chun if chunk_boundaries is None: chunk_boundaries = chunk_lines(source, _DESIRED_CHUNK_LENGTH_CHARS) + # Line numbers come from a binary search over newline offsets, so large files aren't rescanned per chunk. + newline_offsets = [match.start() for match in re.finditer("\n", source)] chunks: list[Chunk] = [] for boundary in chunk_boundaries: # Clamp to start_index so zero-length chunks don't produce an off-by-one. @@ -31,8 +35,8 @@ def chunk_source(source: str, file_path: str, language: str | None) -> list[Chun Chunk( content=text, file_path=file_path, - start_line=source[: boundary.start].count("\n") + 1, - end_line=source[:end_index].count("\n") + 1, + start_line=bisect_left(newline_offsets, boundary.start) + 1, + end_line=bisect_left(newline_offsets, end_index) + 1, language=language, ) ) diff --git a/src/semble/version.py b/src/semble/version.py index 1ecfdcf3..4dcdc853 100644 --- a/src/semble/version.py +++ b/src/semble/version.py @@ -1,2 +1,2 @@ -__version_triple__ = (0, 5, 6) +__version_triple__ = (0, 6, 0) __version__ = ".".join(map(str, __version_triple__)) diff --git a/tests/test_chunker.py b/tests/test_chunker.py index 135b3f1d..59965065 100644 --- a/tests/test_chunker.py +++ b/tests/test_chunker.py @@ -45,6 +45,17 @@ def test_chunk_source_language() -> None: chunk_line_spy.assert_called_once() +@pytest.mark.parametrize("language", [None, "python"]) +def test_chunk_source_line_numbers_match_content(language: str | None) -> None: + """chunk_source line numbers point at the lines each chunk's content comes from.""" + source = "".join(f"def f{i}():\n return {i}\n\n" for i in range(200)) + lines = source.splitlines(keepends=True) + chunks = chunk_source(source, "foo.py", language) + assert len(chunks) > 1 + for c in chunks: + assert c.content in "".join(lines[c.start_line - 1 : c.end_line]) + + def test_core_chunk_empty_input() -> None: """core.chunk returns [] for whitespace-only input.""" assert chunk(" \n", "python", 100) == []