From 5ebe990f0b21be06086ee74e5e2564835869dce8 Mon Sep 17 00:00:00 2001 From: Jonas Aasberg Date: Thu, 17 Sep 2026 13:45:17 +0200 Subject: [PATCH 1/3] feat(bench): deterministic fixtures and a large-repo benchmark harness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit "Slow on big repositories" is the most consistent structural complaint about every established GUI client, and being fast is one of our two strongest claims. We had no numbers, so it was an adjective. Four fixtures. Three generate from a fast-import stream in seconds and are deterministic — seeded content, a fixed epoch, a fixed author — so the same parameters produce the same object ids on any machine, and each isolates one dimension: deep (50,000 commits), wide (50,000 files all modified, 5,000 untracked), refs (5,001 branches, 2,000 tags). Breadth hurts differently from depth, and one combined fixture would give a number that cannot say which dimension moved. The fourth is a real torvalds/linux clone, opt-in, because a synthetic repository cannot stand in for 1.5 million real commits and nobody re-runs a benchmark that starts with a multi-gigabyte download. The harness drives the real Libgit2Backend through the real GitBackend trait. It is behind required-features so the Rust CI gate never builds or links it. Why: three decisions carry the rest. The composite is the point. open_screen issues the ELEVEN reads refreshAll issues, simultaneously, behind a barrier — the only shape that can catch "a slow status blocks everything else on that repo", which an op-at-a-time benchmark is structurally blind to. The barrier is load-bearing: spawn eleven threads without one and the first read finishes before the last starts. Baselines ask the same question, not the cheapest one sharing a name. status returns per-file line counts, so its baseline is git status plus both --numstat diffs; the log baselines are --topo-order because the commit graph's lanes depend on that ordering. The first draft used a plain git log and made our first page look fourteen times slower than git. It is not — it is at parity. Repeats are time-boxed rather than counted, floored at three, because ten repeats of an eight-second log page is thirteen minutes for one table row. Co-Authored-By: Claude Opus 5 (1M context) --- package.json | 3 +- scripts/bench-fixtures.mjs | 412 +++++++++++ scripts/bench-report.mjs | 424 +++++++++++ scripts/bench.sh | 101 +++ src-tauri/Cargo.toml | 15 + src-tauri/benches/repo_bench.rs | 1229 +++++++++++++++++++++++++++++++ 6 files changed, 2183 insertions(+), 1 deletion(-) create mode 100755 scripts/bench-fixtures.mjs create mode 100755 scripts/bench-report.mjs create mode 100755 scripts/bench.sh create mode 100644 src-tauri/benches/repo_bench.rs diff --git a/package.json b/package.json index daa29aa8..72dbf82c 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,8 @@ "test:e2e": "pnpm test:e2e:build && pnpm test:e2e:run", "test:e2e:build": "pnpm tauri build --debug --no-bundle --features tauri/custom-protocol,e2e --config src-tauri/tauri.e2e.conf.json && mkdir -p e2e/.bin && cp src-tauri/target/debug/platypusgit e2e/.bin/platypusgit", "test:e2e:run": "wdio run e2e/wdio.conf.ts", - "test:e2e:docker": "e2e/e2e-docker.sh" + "test:e2e:docker": "e2e/e2e-docker.sh", + "bench": "scripts/bench.sh" }, "dependencies": { "@codemirror/commands": "^6.11.0", diff --git a/scripts/bench-fixtures.mjs b/scripts/bench-fixtures.mjs new file mode 100755 index 00000000..137f6b9f --- /dev/null +++ b/scripts/bench-fixtures.mjs @@ -0,0 +1,412 @@ +#!/usr/bin/env node +// Fixture repositories for the large-repo benchmark (issue 257). +// +// "Slow on big repositories" is the most consistent complaint about every +// established GUI client, and "we are fast" is one of this project's two +// strongest claims. A claim with no number behind it is an adjective, so the +// benchmark needs repositories big enough to hurt — and they have to be the +// SAME repositories on every machine, or the numbers cannot be compared to +// each other, let alone to a previous run. +// +// Hence generated rather than downloaded, for three of the four. A generated +// fixture is deterministic (fixed seed, fixed timestamps, fixed author), costs +// no network, and isolates ONE dimension each: +// +// * `deep` — many commits over a small tree. The log walk, and nothing else. +// * `wide` — one commit, an enormous tree, every file dirty. `status`, and +// nothing else. +// * `refs` — thousands of branches and tags over a short history. Ref +// enumeration, and nothing else. +// +// Breadth hurts differently from depth, which is exactly why they are separate +// repositories: a single "big repo" fixture would give one number that cannot +// say which dimension moved when it regresses. +// +// The fourth, `linux`, is a real clone of torvalds/linux and is NOT generated — +// a synthetic repository cannot stand in for 1.4 million real commits, 90k real +// paths and a real pack layout, and that is the repository people actually mean +// when they say a client is slow. It is opt-in (`--linux`) because it is a +// multi-gigabyte download. +// +// Everything is written under `$PGBENCH_HOME` (default +// `~/.cache/platypusgit-bench`), never inside the repository: a fixture is a +// build artifact, it is regenerable, and `wide` alone is 50,000 files. +// +// Determinism, stated precisely. Identical inputs produce byte-identical object +// ids: the content comes from a seeded LCG, every timestamp is derived from a +// fixed epoch, and the author never varies. So `deep`'s HEAD sha is the same on +// your machine as on mine, and a fixture that drifted is visible rather than +// silent — a `.fixture.json` stamp beside each one records the parameters +// and the resulting HEAD, and a shape that no longer matches is regenerated +// rather than reused. + +import { spawn, spawnSync } from "node:child_process"; +import { + mkdirSync, + rmSync, + renameSync, + existsSync, + writeFileSync, + readFileSync, +} from "node:fs"; +import { join } from "node:path"; +import { homedir } from "node:os"; + +/** Bumped when a change here would produce different repositories. A fixture + * whose recorded version is older is regenerated rather than reused, because + * comparing today's numbers against a differently-shaped repository is worse + * than having no previous numbers at all. */ +export const FIXTURE_VERSION = 1; + +/** The shapes, and why each number is the number. + * + * `deep` at 50,000 commits is not arbitrary: it is the size at which + * GitKraken's own users report it falling over ("past ~50,000 commits, native + * clients beat it"), so it is the threshold the market has already identified. + * `wide` at 50,000 files with every one of them modified is the shape of a + * generated-code monorepo after a formatter run — the case where `status` is + * the whole cost. `refs` at 5,000 branches is a long-lived repository nobody + * prunes, which is most of them. */ +export const SHAPES = { + deep: { commits: 50_000, files: 16 }, + wide: { files: 50_000, dirs: 250, untracked: 5_000 }, + refs: { commits: 2_000, branches: 5_000, lightweightTags: 1_500, annotatedTags: 500 }, +}; + +const AUTHOR = "PlatypusGit Bench "; +/** 2023-11-14T22:13:20Z. Fixed so object ids do not depend on the clock. */ +const EPOCH = 1_700_000_000; + +export function benchHome() { + return process.env.PGBENCH_HOME || join(homedir(), ".cache", "platypusgit-bench"); +} + +/** A tiny LCG. Deterministic across Node versions in a way `Math.random` with a + * seed shim is not, and the statistical quality is irrelevant here — this only + * has to produce bytes that do not compress to nothing. */ +function lcg(seed) { + let s = seed >>> 0; + return () => { + s = (Math.imul(s, 1_664_525) + 1_013_904_223) >>> 0; + return s; + }; +} + +const WORDS = [ + "handle", "buffer", "commit", "index", "refspec", "packet", "stream", "node", + "cursor", "window", "lane", "hunk", "blob", "tree", "ref", "oid", "walk", + "stage", "merge", "rebase", "fetch", "prune", "shard", "lock", "queue", +]; + +/** Plausible source-ish text. Real-looking lines matter: a file of one repeated + * byte deltas and packs unlike anything a user has, which would flatter every + * number measured against it. */ +function makeLines(rand, count) { + const out = []; + for (let i = 0; i < count; i++) { + const a = WORDS[rand() % WORDS.length]; + const b = WORDS[rand() % WORDS.length]; + const c = WORDS[rand() % WORDS.length]; + out.push(` let ${a}_${i} = ${b}(${c}, ${rand() % 9973});`); + } + return out.join("\n") + "\n"; +} + +function run(cmd, args, opts = {}) { + const r = spawnSync(cmd, args, { stdio: "inherit", ...opts }); + if (r.status !== 0) { + throw new Error(`${cmd} ${args.join(" ")} exited ${r.status ?? r.signal}`); + } +} + +function capture(cmd, args, opts = {}) { + const r = spawnSync(cmd, args, { encoding: "utf8", ...opts }); + if (r.status !== 0) { + throw new Error(`${cmd} ${args.join(" ")} exited ${r.status ?? r.signal}: ${r.stderr}`); + } + return r.stdout.trim(); +} + +/** Create the repository and pin every config the numbers could otherwise + * depend on. Left at git's DEFAULTS on purpose: `core.untrackedCache` and + * `core.fsmonitor` both make `status` dramatically cheaper, and benchmarking + * with them on would publish a number almost nobody's repository produces. */ +function initRepo(dir) { + rmSync(dir, { recursive: true, force: true }); + mkdirSync(dir, { recursive: true }); + run("git", ["init", "--quiet", "--initial-branch=main", dir]); + const cfg = (k, v) => run("git", ["-C", dir, "config", k, v]); + cfg("user.name", "PlatypusGit Bench"); + cfg("user.email", "bench@platypusgit.invalid"); + cfg("commit.gpgsign", "false"); + cfg("tag.gpgsign", "false"); + cfg("core.autocrlf", "false"); + // No background repack mid-benchmark: a `gc --auto` firing during a timed run + // is a measurement of gc, attributed to whatever op happened to be running. + cfg("gc.auto", "0"); + cfg("gc.autoDetach", "false"); +} + +/** Feed a fast-import stream, respecting backpressure. The streams here reach + * tens of megabytes; writing them to a temp file first would double the io for + * no benefit, and buffering them in memory is how the wide fixture OOMs. */ +function fastImport(dir, produce) { + return new Promise((resolve, reject) => { + const child = spawn( + "git", + ["-C", dir, "fast-import", "--quiet", "--done"], + { stdio: ["pipe", "inherit", "inherit"] }, + ); + child.on("error", reject); + child.on("close", (code) => + code === 0 ? resolve() : reject(new Error(`fast-import exited ${code}`)), + ); + // `--done` above makes a truncated stream an ERROR rather than a silent + // partial import, which costs exactly this one line and is worth it: a + // fixture that quietly stopped at commit 31,000 would publish a number for + // a repository nobody can reproduce. + produce(child.stdin).then( + () => child.stdin.end("done\n"), + (e) => { + child.stdin.destroy(); + reject(e); + }, + ); + }); +} + +/** `write` that awaits drain. Without this the wide fixture's 50,000 blobs + * queue in Node's heap faster than git reads them. */ +function writer(stream) { + return (chunk) => + stream.write(chunk) ? Promise.resolve() : new Promise((r) => stream.once("drain", r)); +} + +function dataBlock(text) { + return `data ${Buffer.byteLength(text)}\n${text}\n`; +} + +// --------------------------------------------------------------------------- +// deep — 50,000 commits over 16 files. +// --------------------------------------------------------------------------- + +async function buildDeep(dir) { + const { commits, files } = SHAPES.deep; + initRepo(dir); + const rand = lcg(0x0dee9); + + await fastImport(dir, async (stdin) => { + const w = writer(stdin); + for (let i = 0; i < commits; i++) { + const path = `src/module_${String(i % files).padStart(2, "0")}.rs`; + const body = makeLines(rand, 40); + const when = EPOCH + i * 60; + await w(`blob\nmark :${i * 2 + 1}\n${dataBlock(body)}`); + const msg = + `feat(module ${i % files}): revision ${i}\n\n` + + `Generated fixture commit ${i} of ${commits}.\n`; + await w( + `commit refs/heads/main\nmark :${i * 2 + 2}\n` + + `author ${AUTHOR} ${when} +0000\n` + + `committer ${AUTHOR} ${when} +0000\n` + + dataBlock(msg) + + (i === 0 ? "" : `from :${i * 2}\n`) + + `M 100644 :${i * 2 + 1} ${path}\n\n`, + ); + } + }); + + run("git", ["-C", dir, "reset", "--hard", "main", "--quiet"]); + run("git", ["-C", dir, "repack", "-adq"]); +} + +// --------------------------------------------------------------------------- +// wide — 50,000 files, every one of them modified, plus 5,000 untracked. +// --------------------------------------------------------------------------- + +function widePath(i, dirs) { + const d = i % dirs; + return `pkg/${String(d).padStart(3, "0")}/gen_${String(i).padStart(6, "0")}.ts`; +} + +async function buildWide(dir) { + const { files, dirs, untracked } = SHAPES.wide; + initRepo(dir); + const rand = lcg(0x21de); + + await fastImport(dir, async (stdin) => { + const w = writer(stdin); + const marks = []; + for (let i = 0; i < files; i++) { + await w(`blob\nmark :${i + 1}\n${dataBlock(makeLines(rand, 12))}`); + marks.push(i + 1); + } + await w( + `commit refs/heads/main\nmark :${files + 1}\n` + + `author ${AUTHOR} ${EPOCH} +0000\n` + + `committer ${AUTHOR} ${EPOCH} +0000\n` + + dataBlock("chore: the generated tree\n"), + ); + for (let i = 0; i < files; i++) { + await w(`M 100644 :${marks[i]} ${widePath(i, dirs)}\n`); + } + await w("\n"); + }); + + run("git", ["-C", dir, "reset", "--hard", "main", "--quiet"]); + run("git", ["-C", dir, "repack", "-adq"]); + + // Now dirty it. Appending rather than rewriting so the modification is a real + // content change with an unchanged size prefix — the shape a formatter run or + // a codemod leaves, which is the case people describe when they say a client + // hangs on `status`. + for (let i = 0; i < files; i++) { + const p = join(dir, widePath(i, dirs)); + writeFileSync(p, readFileSync(p, "utf8") + "// touched by the fixture\n"); + } + for (let i = 0; i < untracked; i++) { + writeFileSync(join(dir, `pkg/${String(i % dirs).padStart(3, "0")}/new_${i}.ts`), "// new\n"); + } +} + +// --------------------------------------------------------------------------- +// refs — 5,000 branches and 2,000 tags over 2,000 commits. +// --------------------------------------------------------------------------- + +async function buildRefs(dir) { + const { commits, branches, lightweightTags, annotatedTags } = SHAPES.refs; + initRepo(dir); + const rand = lcg(0x4e75); + + await fastImport(dir, async (stdin) => { + const w = writer(stdin); + for (let i = 0; i < commits; i++) { + const when = EPOCH + i * 3600; + await w(`blob\nmark :${i * 2 + 1}\n${dataBlock(makeLines(rand, 8))}`); + await w( + `commit refs/heads/main\nmark :${i * 2 + 2}\n` + + `author ${AUTHOR} ${when} +0000\n` + + `committer ${AUTHOR} ${when} +0000\n` + + dataBlock(`chore: commit ${i}\n`) + + (i === 0 ? "" : `from :${i * 2}\n`) + + `M 100644 :${i * 2 + 1} src/file_${i % 32}.txt\n\n`, + ); + } + // Spread the refs over the whole history rather than piling them on the + // tip: a client that enumerates refs has to peel each one, and refs that + // all point at the same commit is the case where every cache hits. + const at = (n) => `:${(n % commits) * 2 + 2}`; + for (let i = 0; i < branches; i++) { + // Foldered names, because that is what a repository with 5,000 branches + // really looks like and it is what the branch tree has to group. + const name = `team-${i % 40}/feature/${String(i).padStart(5, "0")}`; + await w(`reset refs/heads/${name}\nfrom ${at(i * 7)}\n\n`); + } + for (let i = 0; i < lightweightTags; i++) { + await w(`reset refs/tags/build-${String(i).padStart(5, "0")}\nfrom ${at(i * 3)}\n\n`); + } + for (let i = 0; i < annotatedTags; i++) { + await w( + `tag v1.${i}.0\nfrom ${at(i * 5)}\n` + + `tagger ${AUTHOR} ${EPOCH + i * 7200} +0000\n` + + dataBlock(`Release v1.${i}.0\n`), + ); + } + }); + + run("git", ["-C", dir, "reset", "--hard", "main", "--quiet"]); + run("git", ["-C", dir, "repack", "-adq"]); +} + +// --------------------------------------------------------------------------- +// linux — the real thing. +// --------------------------------------------------------------------------- + +function buildLinux(dir) { + if (existsSync(join(dir, ".git"))) { + console.log(` linux: already present at ${dir}`); + return; + } + const tmp = `${dir}.partial`; + rmSync(tmp, { recursive: true, force: true }); + console.log(" linux: cloning torvalds/linux (several GB, this takes a while)…"); + // A full clone on purpose. `--filter=blob:none` would make every diff in the + // benchmark trigger a lazy fetch, so the numbers would be measuring the + // network. + run("git", ["clone", "--quiet", "https://github.com/torvalds/linux.git", tmp]); + run("git", ["-C", tmp, "config", "gc.auto", "0"]); + run("git", ["-C", tmp, "config", "gc.autoDetach", "false"]); + run("git", ["-C", tmp, "config", "user.name", "PlatypusGit Bench"]); + run("git", ["-C", tmp, "config", "user.email", "bench@platypusgit.invalid"]); + renameSync(tmp, dir); +} + +// --------------------------------------------------------------------------- + +const BUILDERS = { deep: buildDeep, wide: buildWide, refs: buildRefs }; + +/** The stamp lives BESIDE the repository, never inside it: `wide` is measured + * partly by how many untracked files `status` has to report, and a stray + * `fixture.json` in the work tree would be one of them. */ +export function stampPath(home, name) { + return join(home, `${name}.fixture.json`); +} + +/** What `bench.sh` compares against to decide whether a fixture is reusable. */ +function stamp(name, dir) { + return { + name, + version: FIXTURE_VERSION, + shape: SHAPES[name] ?? null, + head: capture("git", ["-C", dir, "rev-parse", "HEAD"]), + generated: new Date().toISOString(), + }; +} + +export function isFresh(name, home) { + const path = stampPath(home, name); + if (!existsSync(path)) return false; + try { + const have = JSON.parse(readFileSync(path, "utf8")); + return ( + have.version === FIXTURE_VERSION && + JSON.stringify(have.shape) === JSON.stringify(SHAPES[name] ?? null) + ); + } catch { + return false; + } +} + +async function main() { + const args = process.argv.slice(2); + const force = args.includes("--force"); + const wanted = args.filter((a) => !a.startsWith("--")); + const names = wanted.length ? wanted : Object.keys(BUILDERS); + const home = benchHome(); + mkdirSync(home, { recursive: true }); + + for (const name of names) { + const dir = join(home, name); + if (name === "linux") { + buildLinux(dir); + continue; + } + if (!BUILDERS[name]) throw new Error(`unknown fixture: ${name}`); + if (!force && isFresh(name, home)) { + console.log(` ${name}: up to date at ${dir}`); + continue; + } + const started = Date.now(); + console.log(` ${name}: generating…`); + await BUILDERS[name](dir); + writeFileSync(stampPath(home, name), JSON.stringify(stamp(name, dir), null, 2) + "\n"); + console.log(` ${name}: ready in ${((Date.now() - started) / 1000).toFixed(1)}s (${dir})`); + } +} + +if (import.meta.url === `file://${process.argv[1]}`) { + main().catch((e) => { + console.error(e.message); + process.exit(1); + }); +} diff --git a/scripts/bench-report.mjs b/scripts/bench-report.mjs new file mode 100755 index 00000000..dde5e57c --- /dev/null +++ b/scripts/bench-report.mjs @@ -0,0 +1,424 @@ +#!/usr/bin/env node +// Render benchmark runs into the two things that get committed (issue 257). +// +// `src-tauri/benches/repo_bench.rs` writes one JSON document per fixture, full +// of raw samples. This turns the set of them into: +// +// * `docs/dev/benchmark.json` — the published record, summary statistics only, +// read by `test/benchmark.test.ts`; +// * the table block inside `docs/dev/performance.md`, between its generated +// markers — the developer-facing record, with every operation, the `git` +// baseline beside it and the ratio between them. +// +// **Both are generated, and neither is hand-editable.** That is the whole point +// of the exercise: the site is supposed to print a number somebody measured, +// and the way a measured number turns back into an adjective is somebody +// nudging it in a hurry. +// +// The markdown is rendered from the PUBLISHED record rather than from the raw +// runs, and that indirection is the guard test's whole leverage: the test +// re-renders it from the committed JSON and compares it to the committed +// markdown, so the two can only agree if both came from one run. Rendering from +// the raw results — which are not committed — would leave the guard able to +// check only that the markdown parses. +// +// The raw sample arrays are deliberately NOT committed. They are what makes a +// result checkable, so they stay in `$PGBENCH_HOME/results/`, but a committed +// file that churns forty floating-point numbers per run is a file nobody reads +// the diff of. + +import { readFileSync, writeFileSync, existsSync } from "node:fs"; +import { join } from "node:path"; + +/** Operations in the order the tables print them: roughly the order a user + * meets them, opening a repository first and browsing the tree last, rather + * than the order the harness happens to measure. */ +export const OP_ORDER = [ + "open", + "open_screen", + "open_screen_ipc", + "status", + "log_first_page", + "log_page_deep", + "branches", + "tags", + "diff_commit", + "diff_workdir_file", + "file_history", + "list_all_files", +]; + +/** Title and one-line framing per fixture. Lives here rather than in the JSON + * because it is editorial — what the repository is FOR — while everything the + * harness writes is measured. */ +export const FIXTURES = { + linux: { + title: "torvalds/linux", + kind: "real", + blurb: + "A real clone of the Linux kernel: the repository people mean when they " + + "say a git client is slow.", + }, + deep: { + title: "deep", + kind: "generated", + blurb: + "50,000 commits over a small tree — the size past which GitKraken's own " + + "users report native clients beating it.", + }, + wide: { + title: "wide", + kind: "generated", + blurb: + "50,000 tracked files with every one of them modified, plus 5,000 " + + "untracked — a monorepo just after a codemod.", + }, + refs: { + title: "refs", + kind: "generated", + blurb: + "5,000 branches and 2,000 tags over a short history — a long-lived " + + "repository nobody prunes.", + }, +}; + +/** + * What each row is CALLED, as opposed to what it measured. + * + * Editorial, so it lives here beside `FIXTURES` rather than in the harness, for + * the same reason the fixture blurbs do: rewording a table heading should not + * mean recompiling a benchmark and re-running it against a six-gigabyte clone. + * The harness's own label is the fallback for an op this map has not met. + * + * `log_page_deep` is why the rule earned its keep. Its harness label named the + * commit range it reaches — which is a lie on any fixture with less history + * than that, and `wide` has exactly one commit. + */ +export const OP_LABELS = { + open: "Open the repository", + open_screen: "Everything the first screen needs, at once", + open_screen_ipc: "…including encoding it all for the webview", + status: "Working-tree status", + log_first_page: "First page of history", + log_page_deep: "Ten pages into history", + branches: "List every branch", + tags: "List every tag", + diff_commit: "Diff the selected commit", + diff_workdir_file: "Diff one modified file", + file_history: "History of one file", + list_all_files: "Browse the whole tree", +}; + +/** + * "1 commits" is the sort of thing that makes a published table look + * unproofread, and the counts come from the harness as free text. Fixing it + * where it is READ rather than where it is measured keeps the harness's job to + * measuring. + */ +export function tidyScale(scale) { + return String(scale).replace(/^1 ([a-z]+)$/, (whole, word) => { + if (word.endsWith("ies")) return `1 ${word.slice(0, -3)}y`; + if (/(ch|sh|x|s)es$/.test(word)) return `1 ${word.slice(0, -2)}`; + if (word.endsWith("s")) return `1 ${word.slice(0, -1)}`; + return whole; + }); +} + +export const BEGIN = + ""; +export const END = ""; + +/** Milliseconds, at a precision the measurement can actually support. + * + * Three significant figures at most, and seconds once past a thousand + * milliseconds: "5728.58 ms" implies a repeatability no wall-clock number on a + * general-purpose machine has, and reading it as 5.73 s is how anybody would + * say it out loud anyway. */ +export function fmtMs(v) { + if (v == null) return "—"; + if (v >= 1000) return `${(v / 1000).toFixed(2)} s`; + if (v >= 100) return `${v.toFixed(0)} ms`; + if (v >= 10) return `${v.toFixed(1)} ms`; + return `${v.toFixed(2)} ms`; +} + +export function fmtRatio(v) { + if (v == null) return "—"; + if (v >= 10) return `${v.toFixed(0)}×`; + if (v >= 1) return `${v.toFixed(1)}×`; + return `${v.toFixed(2)}×`; +} + +export function fmtMb(v) { + return v == null ? "—" : `${v.toFixed(0)} MB`; +} + +function thousands(n) { + return Number(n).toLocaleString("en-US"); +} + +/** One line describing the repository, for the table caption. Built from what + * the harness counted rather than from the fixture definition, so a fixture + * that generated wrong is visible in the published document. */ +export function describeRepo(r) { + const parts = [ + `${thousands(r.commits)} commits`, + `${thousands(r.trackedFiles)} tracked files`, + ]; + if (r.branches > 1) parts.push(`${thousands(r.branches)} branches`); + if (r.tags > 0) parts.push(`${thousands(r.tags)} tags`); + if (r.dirtyEntries > 0) parts.push(`${thousands(r.dirtyEntries)} changed entries`); + return parts.join(" · "); +} + +/** + * What `git` spent on the WORK, with process start-up taken back out. + * + * Every `git` invocation pays for a `fork`, an `exec`, the dynamic loader and + * git's own start-up before it reads a byte — over ten milliseconds on an + * M-series Mac, which is more than most of the operations here take in total. + * We pay none of it: the backend is libgit2, in process. + * + * So the ratio is against git's work, not its wall clock, and that choice is + * deliberately the one that makes US look worse. Comparing against the wall + * clock would hand us a ten-millisecond head start on every row and quietly + * turn a loss into a win on any operation git finishes quickly. + * + * Scaled by the invocation COUNT, because a baseline can be several processes — + * `status`'s is three (see the module doc on why), so it pays the floor three + * times. + */ +export function gitWorkMs(op, floorMs) { + if (op.gitMedianMs == null || !op.gitInvocations) return null; + return Math.max(0, op.gitMedianMs - floorMs * op.gitInvocations); +} + +/** + * Is what is left big enough to divide by? + * + * When start-up is most of the baseline, the remainder is the difference of two + * similar noisy numbers and a ratio against it is arithmetic, not measurement. + * A quarter is the line: below it the row prints † and no ratio, which is the + * honest answer for `status` on a clean tree, where git's three processes cost + * 40 ms and 37 ms of that is launching them. + */ +export function isFloorBound(op, floorMs) { + const work = gitWorkMs(op, floorMs); + if (work == null || op.gitMedianMs == null) return false; + return work <= 0 || work < op.gitMedianMs * 0.25; +} + +/** The ratio against git's work, or null when that work is unmeasurable. */ +export function ratioToGit(op, floorMs) { + if (op.repeatMedianMs == null || isFloorBound(op, floorMs)) return null; + const work = gitWorkMs(op, floorMs); + return work && work > 0 ? op.repeatMedianMs / work : null; +} + +function orderOps(ops) { + const rank = (op) => { + const i = OP_ORDER.indexOf(op); + return i === -1 ? OP_ORDER.length : i; + }; + return [...ops].sort((a, b) => rank(a.op) - rank(b.op)); +} + +export function renderTable(fixture) { + const head = [ + "Operation", + "Result size", + "First call", + "Repeat", + "p95", + "`git` work", + "vs `git`", + ]; + const rows = fixture.operations.map((o) => [ + o.label, + tidyScale(o.scale), + fmtMs(o.firstMs), + fmtMs(o.repeatMedianMs), + fmtMs(o.repeatP95Ms), + // git's WORK, not its wall clock — see `gitWorkMs`. The dagger marks a + // baseline where start-up swamped the work, so there was nothing left to + // compare; that row's ratio is deliberately absent rather than impressive. + o.gitFloorBound ? "†" : fmtMs(o.gitWorkMs), + fmtRatio(o.ratioToGit), + ]); + const line = (cells) => `| ${cells.join(" | ")} |`; + return [line(head), line(head.map(() => "---")), ...rows.map(line)].join("\n"); +} + +export function renderSoak(soak) { + if (!soak) return null; + const drift = + soak.firstHalfMedianMs && soak.secondHalfMedianMs + ? (soak.secondHalfMedianMs / soak.firstHalfMedianMs - 1) * 100 + : null; + return ( + `${thousands(soak.iterations)} first-screen fan-outs over ${soak.minutes} minutes. ` + + `Resident memory ${fmtMb(soak.rssStartMb)} → ${fmtMb(soak.rssEndMb)} ` + + `(peak ${fmtMb(soak.rssPeakMb)}). ` + + `Median fan-out ${fmtMs(soak.firstHalfMedianMs)} in the first half, ` + + `${fmtMs(soak.secondHalfMedianMs)} in the second` + + (drift == null ? "." : ` — ${drift >= 0 ? "+" : ""}${drift.toFixed(1)}%.`) + ); +} + +/** The table block, rendered from the published record. */ +export function renderMarkdown(data) { + const out = []; + const m = data.machine; + out.push( + `Measured on ${m.cpu} (${m.cores} cores, ${m.memoryGb} GB, ${m.os}) with ` + + `${m.gitVersion}, on ${data.measuredOn}. Up to ${data.iterations} repeats ` + + `per operation, time-boxed to ${data.budgetSeconds}s each — so a cheap ` + + `operation gets the full count and an expensive one gets at least three. ` + + `The published record records how many each row actually took.`, + ); + out.push(""); + out.push( + "The **`git` work** column is that baseline's wall clock with process " + + `start-up subtracted (${fmtMs(data.gitSpawnFloorMs)} per invocation on ` + + "this machine, measured), because we pay none of it — the backend is " + + "libgit2, in process. That is deliberately the comparison that makes us " + + "look worse: against git's wall clock we would get a ten-millisecond " + + "head start on every row. **†** marks a baseline where start-up swamped " + + "the work, leaving a remainder too small to divide by; those rows print " + + "no ratio rather than a flattering one.", + ); + out.push(""); + for (const fixture of data.fixtures) { + out.push(`### ${fixture.title}`); + out.push(""); + if (fixture.blurb) out.push(fixture.blurb); + out.push(""); + out.push(`*${describeRepo(fixture.repository)}*`); + out.push(""); + out.push(renderTable(fixture)); + const soak = renderSoak(fixture.soak); + if (soak) { + out.push(""); + out.push(`**Soak.** ${soak}`); + } + out.push(""); + } + return out.join("\n").trimEnd(); +} + +/** The published record. Summary statistics only — see the header note on why + * the samples stay out of the repository. + * + * It lives beside `performance.md` rather than under `site/` because nothing + * renders it yet: publishing these figures on the marketing site waits on the + * log-walk work, and a data file under `site/**` would redeploy the site on + * every re-measurement for no visible change. */ +export function buildPublishedRecord(runs) { + const m = runs[0].machine; + return { + measuredOn: new Date().toISOString().slice(0, 10), + machine: { + cpu: m.cpu, + cores: m.cores, + memoryGb: m.memoryGb, + os: `${m.os}/${m.arch}`, + gitVersion: m.git, + }, + iterations: runs[0].iterations, + budgetSeconds: runs[0].budgetSeconds, + /** What one `git` invocation costs before doing anything, measured. */ + gitSpawnFloorMs: runs[0].gitSpawnFloorMs, + fixtures: runs.map((run) => ({ + key: run.fixture, + title: FIXTURES[run.fixture]?.title ?? run.fixture, + kind: FIXTURES[run.fixture]?.kind ?? "generated", + blurb: FIXTURES[run.fixture]?.blurb ?? "", + repository: run.repository, + /** Measured per run, because it is a property of the machine at the time + * and not a constant. The note under the tables quotes the first. */ + gitSpawnFloorMs: run.gitSpawnFloorMs, + operations: orderOps(run.operations).map((o) => ({ + op: o.op, + label: OP_LABELS[o.op] ?? o.label, + scale: tidyScale(o.scale), + /** How many repeats this row actually took — see the time box. */ + samples: o.samples, + firstMs: o.firstMs, + repeatMedianMs: o.repeatMedianMs, + repeatP95Ms: o.repeatP95Ms, + gitCommand: o.gitCommand, + gitInvocations: o.gitInvocations, + /** The baseline's wall clock, start-up included. */ + gitMedianMs: o.gitMedianMs, + /** …and with start-up subtracted, which is what the ratio divides by. */ + gitWorkMs: gitWorkMs(o, run.gitSpawnFloorMs), + // Derived HERE rather than in the harness: whether a baseline is + // comparable is a judgement about what may be published, and it belongs + // with the thing that publishes it. + gitFloorBound: isFloorBound(o, run.gitSpawnFloorMs), + ratioToGit: ratioToGit(o, run.gitSpawnFloorMs), + })), + soak: run.soak ?? null, + })), + }; +} + +export function splice(doc, block) { + const a = doc.indexOf(BEGIN); + const b = doc.indexOf(END); + if (a === -1 || b === -1) { + throw new Error("docs/dev/performance.md is missing its generated markers"); + } + return doc.slice(0, a + BEGIN.length) + "\n\n" + block + "\n\n" + doc.slice(b); +} + +function arg(argv, name, fallback) { + const at = argv.indexOf(name); + return at === -1 ? fallback : argv[at + 1]; +} + +function main() { + const argv = process.argv.slice(2); + const results = arg(argv, "--results"); + const root = arg(argv, "--root", process.cwd()); + const only = arg(argv, "--fixtures", "").split(",").filter(Boolean); + const print = argv.includes("--print"); + + // Ordered so the real repository leads and the generated ones follow, which + // is the order a reader wants: the headline first, then the dimensions that + // explain it. + const order = ["linux", "deep", "wide", "refs"]; + const names = (only.length ? only : order).sort( + (a, b) => order.indexOf(a) - order.indexOf(b), + ); + + const runs = []; + for (const name of names) { + const path = join(results, `${name}.json`); + if (!existsSync(path)) { + console.error(` no results for ${name} (${path}) — skipping`); + continue; + } + runs.push(JSON.parse(readFileSync(path, "utf8"))); + } + if (!runs.length) throw new Error("no results to render"); + + const data = buildPublishedRecord(runs); + const markdown = renderMarkdown(data); + if (print) { + console.log(markdown); + return; + } + + const dataPath = join(root, "docs/dev/benchmark.json"); + writeFileSync(dataPath, JSON.stringify(data, null, 2) + "\n"); + console.log(` wrote ${dataPath}`); + + const docPath = join(root, "docs/dev/performance.md"); + writeFileSync(docPath, splice(readFileSync(docPath, "utf8"), markdown)); + console.log(` wrote ${docPath}`); +} + +if (import.meta.url === `file://${process.argv[1]}`) { + main(); +} diff --git a/scripts/bench.sh b/scripts/bench.sh new file mode 100755 index 00000000..12fec0e0 --- /dev/null +++ b/scripts/bench.sh @@ -0,0 +1,101 @@ +#!/usr/bin/env bash +# The large-repo benchmark, end to end (issue 257). +# +# pnpm bench # the three generated fixtures, published +# pnpm bench --linux # …and a real clone of torvalds/linux +# pnpm bench --fixture deep # one fixture only +# pnpm bench --soak 60 # add a 60-minute soak per fixture +# pnpm bench --no-publish # measure and print; write nothing +# +# Three steps, each of which can be run on its own: +# +# 1. `scripts/bench-fixtures.mjs` materialises the repositories under +# $PGBENCH_HOME (default ~/.cache/platypusgit-bench). Generated ones are +# reused when their recorded shape still matches; `--force` regenerates. +# 2. `src-tauri/benches/repo_bench.rs` measures one repository and writes a +# JSON document per fixture. It is behind `--features bench` so the Rust CI +# gate never builds it. +# 3. `scripts/bench-report.mjs` renders those documents into +# `docs/dev/performance.md` and `docs/dev/benchmark.json`, which is +# what `test/benchmark.test.ts` reads. +# +# Publishing is the default because a benchmark nobody publishes is a benchmark +# nobody runs twice. `--no-publish` is for the case you are iterating on the +# harness itself and do not want a dirty tree. +# +# Run it on a QUIET machine. Every number here is wall clock, so a compile in +# another window is measured as this program being slow — and the published +# figures are the ones a stranger will check. +set -euo pipefail + +root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +home="${PGBENCH_HOME:-$HOME/.cache/platypusgit-bench}" +results="$home/results" + +fixtures=(deep wide refs) +want_linux=0 +publish=1 +soak="" +iterations=10 +force="" + +while [ $# -gt 0 ]; do + case "$1" in + --linux) want_linux=1 ;; + --fixture) fixtures=("$2"); shift ;; + --soak) soak="$2"; shift ;; + --iterations) iterations="$2"; shift ;; + --force) force="--force" ;; + --no-publish) publish=0 ;; + -h|--help) sed -n '2,25p' "${BASH_SOURCE[0]}" | sed 's|^# \{0,1\}||'; exit 0 ;; + *) echo "unknown option: $1" >&2; exit 2 ;; + esac + shift +done + +[ "$want_linux" = 1 ] && fixtures+=(linux) + +# `cargo` and `node` are not on a non-interactive PATH on every machine this +# runs on; the repo's own toolchain locations are the fallback. +CARGO="${CARGO:-$(command -v cargo || echo "$HOME/.cargo/bin/cargo")}" + +echo "==> fixtures ($home)" +node "$root/scripts/bench-fixtures.mjs" $force "${fixtures[@]}" + +echo "==> building the harness" +"$CARGO" build --release --manifest-path "$root/src-tauri/Cargo.toml" \ + --features bench --bench repo_bench + +mkdir -p "$results" +for name in "${fixtures[@]}"; do + echo "==> $name" + soak_args=() + [ -n "$soak" ] && soak_args=(--soak-minutes "$soak") + # Via `cargo bench` rather than the built path: a `harness = false` bench has + # no stable file name (cargo appends a hash), and re-resolving it by globbing + # picks up stale binaries from an earlier build. + # + # `${a[@]+"${a[@]}"}` rather than `"${a[@]}"`: under `set -u`, bash 3.2 — which + # is what macOS ships — treats an EMPTY array's expansion as an unbound + # variable and aborts. Without a soak, that is every run. + "$CARGO" bench --quiet --manifest-path "$root/src-tauri/Cargo.toml" \ + --features bench --bench repo_bench -- \ + --repo "$home/$name" --name "$name" \ + --iterations "$iterations" \ + ${soak_args[@]+"${soak_args[@]}"} \ + --out "$results/$name.json" +done + +# Deliberately NOT `--fixtures "${fixtures[*]}"`. The renderer publishes every +# result it finds, so `pnpm bench --fixture deep` refreshes deep and leaves the +# other three standing. Narrowing it to this run's list would have silently +# deleted `torvalds/linux` from the published record the first time somebody +# re-measured one fixture — the results are a record, not a snapshot of the last +# command anyone happened to type. +if [ "$publish" = 1 ]; then + echo "==> publishing" + node "$root/scripts/bench-report.mjs" --results "$results" --root "$root" +else + echo "==> not publishing (--no-publish); raw results in $results" + node "$root/scripts/bench-report.mjs" --results "$results" --root "$root" --print +fi diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index 07e31e09..005bd556 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -77,6 +77,21 @@ libc = "0.2" [features] # Compile + wire the WebDriver plugin. Set only by the e2e debug build. e2e = ["dep:tauri-plugin-wdio-webdriver"] +# Compile the large-repo benchmark (#257). Adds no code to the crate itself — +# it exists only so the `[[bench]]` target below can be gated off, because the +# Rust CI gate is `cargo test` and a benchmark that links the whole lib would +# put its link time on every backend PR for a target nobody runs there. +bench = [] + +# `scripts/bench.sh` builds and runs this; `docs/dev/performance.md` explains +# what it measures. `harness = false` because it is a program with its own +# argument parsing rather than a set of `#[bench]` functions, and `test = false` +# so `cargo test` cannot decide to execute that program as a test binary. +[[bench]] +name = "repo_bench" +harness = false +test = false +required-features = ["bench"] [dev-dependencies] tempfile = "3" diff --git a/src-tauri/benches/repo_bench.rs b/src-tauri/benches/repo_bench.rs new file mode 100644 index 00000000..87dd39a9 --- /dev/null +++ b/src-tauri/benches/repo_bench.rs @@ -0,0 +1,1229 @@ +//! The large-repo benchmark (issue 257). +//! +//! "Slow on big repositories" is the one structural complaint every established +//! GUI client shares, and "fast" is one of this project's two strongest claims. +//! Until this file existed the claim was an adjective. What it produces is a +//! number per operation per fixture, in a JSON document `scripts/bench.sh` +//! renders into `docs/dev/performance.md` and into the figure the marketing site +//! prints — so a regression shows up as a smaller claim rather than as a +//! stranger's bug report. +//! +//! ## What it measures, and what it therefore does not +//! +//! It drives `Libgit2Backend` — the real backend, through the real `GitBackend` +//! trait, with no test doubles anywhere. That is the layer where a big +//! repository is actually expensive, and it is the layer a regression lands in. +//! +//! It stops at the IPC boundary. There is no webview here, so nothing below +//! measures React rendering, and the honest name for every number is "what the +//! backend costs", not "what the user waits". Two things narrow that gap on +//! purpose: +//! +//! * `open_screen` runs the ELEVEN reads `useRepoStore.refreshAll` issues, +//! simultaneously, from separate threads — the real fan-out, including +//! whatever the per-repository lock does to it. A composite is the only +//! number that can catch "a slow status blocks everything else on that +//! repo", which is the trap `git/repo_locks.rs` exists to avoid and the one +//! an op-at-a-time benchmark is structurally blind to. +//! * `open_screen_ipc` is the same fan-out plus `serde_json` encoding of every +//! payload, because that encoding is real work on a 500-commit page and it +//! happens before the frontend sees a byte. +//! +//! The render on top of those is bounded by the window, not by the repository: +//! the log is paged at 500, diff rows are windowed, and long lists are +//! virtualised. That is the argument for why the backend number is the +//! interesting one — it is not a proof, and `docs/dev/performance.md` says so. +//! +//! ## "first" and "repeat", not "cold" and "warm" +//! +//! `first` is one call on a freshly constructed backend and a freshly opened +//! repository, which is what happens when you open a repository in the app: +//! libgit2's object database, ref database and pack indices are all unbuilt. +//! `repeat` is the median of many calls on that same handle. +//! +//! Neither purges the operating system's file cache, and the words "cold" and +//! "warm" are avoided because they would imply it did. A first-boot number is +//! larger than anything here, by an amount that depends on the disk rather than +//! on this code. +//! +//! ## The `git` baseline +//! +//! Every op that a single `git` invocation can answer is also measured as that +//! invocation. The point is NOT to win: `git` is the floor, and a ratio near it +//! is the good outcome. The point is that a ratio is a comparison a reader can +//! check on their own machine, in a way that a bare millisecond figure from +//! somebody else's laptop is not — and a ratio that doubles is a regression +//! even on a machine that got faster. +//! +//! The baselines are chosen to ask the SAME question, not the cheapest one that +//! shares a name. `status` here is the clearest case: `GitBackend::status` +//! returns per-file added/removed counts, so its baseline is +//! `git status --porcelain` plus both `--numstat` diffs, and comparing it to a +//! bare `git status` would be comparing it to less work than it does. +//! +//! ## Why this lives in `benches/` and spawns processes directly +//! +//! `benches/` is outside the tree `tests/spawn_no_window.rs` guards, and the +//! `Command::new` calls below are deliberate rather than an oversight of that +//! rule: the rule exists so no SHIPPED spawn opens a console window on Windows, +//! and nothing in this file ships. It is behind `required-features = ["bench"]` +//! for the same reason — `cargo test` and `cargo check` must not pay to build +//! it. + +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::sync::{Arc, Barrier}; +use std::time::{Duration, Instant}; + +use platypusgit_lib::git::libgit2::Libgit2Backend; +use platypusgit_lib::git::types::{DiffKind, RepoId}; +use platypusgit_lib::git::GitBackend; + +/// The log page size the frontend asks for (`PAGE_SIZE` in `useRepoStore.ts`). +/// Hard-coded rather than shared because the two trees do not share constants — +/// if that one changes, this one has to change with it or the benchmark stops +/// measuring the thing the app does. +const PAGE_SIZE: usize = 500; + +/// How deep `log_page_deep` walks before timing a page. Ten pages in is past +/// anything a first screen touches, which is the point: the paged tail is where +/// a naive implementation re-walks history from the top every time. +const DEEP_PAGES: usize = 10; + +/// Diff context lines, matching the app's default. +const CONTEXT: u32 = 3; + +// --------------------------------------------------------------------------- +// Timing +// --------------------------------------------------------------------------- + +fn ms(d: Duration) -> f64 { + d.as_secs_f64() * 1000.0 +} + +fn time(f: impl FnOnce() -> T) -> (f64, T) { + let at = Instant::now(); + let out = f(); + (ms(at.elapsed()), out) +} + +/// Summary statistics over the repeat samples. +/// +/// Median rather than mean, and p95 alongside it, because the distribution is +/// not symmetric: one sample in a run lands on a page fault or a scheduler +/// hiccup and drags a mean somewhere the operation never actually was. The +/// samples are kept in the JSON too, so anyone who disagrees with the summary +/// can compute their own. +#[derive(Clone)] +struct Stats { + n: usize, + min: f64, + median: f64, + p95: f64, + max: f64, +} + +impl Stats { + fn of(samples: &[f64]) -> Option { + if samples.is_empty() { + return None; + } + let mut s = samples.to_vec(); + s.sort_by(|a, b| a.partial_cmp(b).unwrap()); + // Nearest-rank p95: with ten samples that is the worst one, which is + // the honest reading of "95th percentile of ten measurements". + let rank = ((0.95 * s.len() as f64).ceil() as usize).max(1) - 1; + Some(Stats { + n: s.len(), + min: s[0], + median: if s.len() % 2 == 1 { + s[s.len() / 2] + } else { + (s[s.len() / 2 - 1] + s[s.len() / 2]) / 2.0 + }, + p95: s[rank], + max: s[s.len() - 1], + }) + } +} + +/// One measured operation. +struct Measured { + /// Stable machine key. The doc renderer and the guard test both key on it. + key: &'static str, + /// What the operation is, in the words the published table uses. + label: &'static str, + /// One call on a fresh handle. `None` when the op has no meaningful + /// first-call form (the composites open their own handle). + first_ms: Option, + samples: Vec, + /// What came back — "55,000 entries", "500 commits". A timing with no size + /// beside it cannot be sanity-checked by a reader, and a benchmark that + /// silently started measuring an empty result is the classic way to publish + /// an excellent number for nothing at all. + scale: String, + /// The `git` invocation asking the same question, and what it cost. + baseline: Option, +} + +struct Baseline { + command: String, + /// How many processes one sample launched. A baseline can be several + /// invocations (see `status`), and each one pays the spawn floor, so the + /// count is what makes "is this comparable at all?" answerable downstream. + invocations: usize, + samples: Vec, +} + +// --------------------------------------------------------------------------- +// The git baseline +// --------------------------------------------------------------------------- + +/// Run one or more `git` invocations and return their combined wall time. +/// +/// A list rather than one command because some of our single calls genuinely +/// are several of git's — see the `status` note in the module doc. Output is +/// discarded to a null sink so the measurement is git's work rather than the +/// terminal's. +fn git_time(repo: &Path, invocations: &[&[&str]]) -> f64 { + let at = Instant::now(); + for args in invocations { + let status = Command::new("git") + .arg("-C") + .arg(repo) + .args(*args) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .status() + .expect("spawn git"); + assert!(status.success(), "git {args:?} failed in {}", repo.display()); + } + ms(at.elapsed()) +} + +/// What one `git` invocation costs before it has done anything. +/// +/// Every baseline below pays for a `fork`, an `exec`, the dynamic loader and +/// git's own start-up, and on this class of machine that is over ten +/// milliseconds — more than most of the operations being measured take in +/// total. Without this number the table would report that we open a repository +/// a hundred times faster than `git`, which is true and meaningless: what it +/// measures is process creation. +/// +/// `rev-parse --git-dir` is the cheapest invocation that still opens the +/// repository, so it is the floor rather than a lower bound nothing can reach. +/// `measured_json` drops the ratio for any baseline within twice it. +fn spawn_floor_ms(repo: &Path) -> f64 { + let once = || git_time(repo, &[&["rev-parse", "--git-dir"]]); + once(); + let mut samples: Vec = (0..10).map(|_| once()).collect(); + samples.sort_by(|a, b| a.partial_cmp(b).unwrap()); + samples[samples.len() / 2] +} + +fn render_command(invocations: &[&[&str]]) -> String { + invocations + .iter() + .map(|a| format!("git {}", a.join(" "))) + .collect::>() + .join(" && ") +} + +fn measure_baseline(repo: &Path, cfg: &Config, invocations: &[&[&str]]) -> Option { + if !cfg.baseline { + return None; + } + // One untimed run first: the first `git` of a run pays for loading the + // binary and its libraries, which is a cost the app never pays twice + // either. It doubles as the estimate the repeat count comes from. + let estimate = git_time(repo, invocations); + let samples = (0..repeats(estimate, cfg)) + .map(|_| git_time(repo, invocations)) + .collect(); + Some(Baseline { + command: render_command(invocations), + invocations: invocations.len(), + samples, + }) +} + +// --------------------------------------------------------------------------- +// The measured operations +// --------------------------------------------------------------------------- + +/// Everything the harness needs to know about the repository before it can +/// choose paths and revisions to measure against. Derived from the repository +/// rather than configured, so the same command works on any fixture. +struct Subject { + path: PathBuf, + head_oid: String, + /// The path changed most often in recent history. + /// + /// NOT simply "a path HEAD touched", which is the obvious choice and is + /// unusable: `file_history` filters a walk by path and stops at `limit` + /// matches, so on a rarely-touched file it never reaches 500 and walks the + /// WHOLE history with a tree comparison per commit. On `torvalds/linux` + /// that is 1.4 million commits and the benchmark simply does not finish. + /// + /// A hot file is also the honest subject: file history is a thing people + /// open on files that change. The pathological case is real and is written + /// up in `docs/dev/performance.md` rather than measured here, because + /// "unbounded" is not a number. + hot_path: Option, + /// A path the working tree has modified, if any. `None` on a clean fixture, + /// which simply skips the worktree diff. + dirty_path: Option, +} + +/// The path touched by the most of the last 2,000 commits. +/// +/// Asked of `git` rather than computed here: this is fixture selection, not a +/// measurement, and one `git log --name-only` is both faster and less code than +/// walking it ourselves. A window of 2,000 keeps it cheap even on a repository +/// with over a million commits, and any file that is hot in the last 2,000 is +/// hot enough to bound the walk being measured. +fn hottest_path(repo: &Path) -> Option { + let out = Command::new("git") + .arg("-C") + .arg(repo) + .args(["log", "--max-count=2000", "--name-only", "--format=", "HEAD"]) + .output() + .ok()?; + let text = String::from_utf8_lossy(&out.stdout); + let mut counts: HashMap<&str, usize> = HashMap::new(); + for line in text.lines().filter(|l| !l.trim().is_empty()) { + *counts.entry(line).or_default() += 1; + } + // Ties broken by name so the choice is deterministic across runs on the + // same repository — otherwise the fixture silently changes under the + // numbers. + counts + .into_iter() + .max_by(|a, b| a.1.cmp(&b.1).then_with(|| b.0.cmp(a.0))) + .map(|(p, _)| PathBuf::from(p)) +} + +fn probe(path: &Path) -> Subject { + let backend = Libgit2Backend::new(); + let handle = backend.open(path).expect("open the repository"); + let head = backend.head_info(&handle.id).expect("head_info"); + let head_oid = head.head_oid.clone().expect("the fixture has commits"); + + let hot_path = hottest_path(path).or_else(|| { + backend + .diff_commit(&handle.id, &head_oid, 0, false) + .ok() + .and_then(|diffs| diffs.into_iter().next()) + .map(|d| PathBuf::from(d.path)) + }); + + let dirty_path = backend.status(&handle.id).ok().and_then(|entries| { + entries + .into_iter() + .find(|e| matches!(e.worktree, platypusgit_lib::git::types::StatusFlag::Modified)) + .map(|e| PathBuf::from(e.path)) + }); + + Subject { + path: path.to_path_buf(), + head_oid, + hot_path, + dirty_path, + } +} + +struct Config { + /// The MOST repeats any operation takes. What it actually takes is + /// `repeats()`, below. + iterations: usize, + /// `spawn_floor_ms`, measured once per run. A baseline within twice it is + /// reported without a ratio. + floor_ms: f64, + warmup: usize, + baseline: bool, + /// Roughly how long one operation's repeats may take, in milliseconds. + budget_ms: f64, +} + +/// Never fewer than this many repeats: a median of two is the mean of two, and +/// a single sample is not a measurement at all. +const MIN_REPEATS: usize = 3; + +/// How many repeats to actually take, given what the first call cost. +/// +/// A fixed count is wrong at both ends. Ten repeats of a 0.15 ms operation is a +/// rounding error's worth of extra confidence; ten repeats of an eight-second +/// log page on a kernel clone is thirteen minutes for one row of one table, and +/// a benchmark nobody has time to finish produces no numbers at all. +/// +/// The budget is per operation, not per run, so the cheap operations keep the +/// sample count their variance actually needs. Each row records how many +/// samples it took, so the table never implies a confidence it does not have. +fn repeats(first_ms: f64, cfg: &Config) -> usize { + if first_ms <= 0.0 { + return cfg.iterations; + } + let affordable = (cfg.budget_ms / first_ms).floor() as usize; + affordable.clamp(MIN_REPEATS, cfg.iterations) +} + +/// Warm-up is skipped outright once one call would eat half the budget. Its +/// purpose is to settle caches that a slow operation has already settled by +/// being slow. +fn warmups(first_ms: f64, cfg: &Config) -> usize { + if first_ms * cfg.warmup as f64 > cfg.budget_ms / 2.0 { + 0 + } else { + cfg.warmup + } +} + +/// Measure one op: a first call on a brand-new backend, then `iterations` +/// repeats on a single handle. +/// +/// The fresh backend is what makes `first` mean anything — `Libgit2Backend` +/// caches a `Repository` per `RepoId`, so reusing one would measure libgit2's +/// caches rather than the work of building them. +fn measure( + key: &'static str, + label: &'static str, + subject: &Subject, + cfg: &Config, + scale: impl Fn(&T) -> String, + op: impl Fn(&Libgit2Backend, &RepoId) -> T, +) -> Measured { + // Progress on stderr, never on stdout: stdout is the JSON document when no + // `--out` is given. A run against a real kernel clone takes minutes, and a + // benchmark that spends them in silence is a benchmark people kill halfway + // and conclude is hung. + eprint!(" {key} … "); + let started = Instant::now(); + let first_ms = { + let backend = Libgit2Backend::new(); + let handle = backend.open(&subject.path).expect("open"); + let (t, _) = time(|| op(&backend, &handle.id)); + t + }; + + let backend = Libgit2Backend::new(); + let handle = backend.open(&subject.path).expect("open"); + for _ in 0..warmups(first_ms, cfg) { + op(&backend, &handle.id); + } + let n = repeats(first_ms, cfg); + let mut samples = Vec::with_capacity(n); + let mut last = None; + for _ in 0..n { + let (t, out) = time(|| op(&backend, &handle.id)); + samples.push(t); + last = Some(out); + } + + let scale = last.as_ref().map(scale).unwrap_or_default(); + eprintln!("{scale} in {:.1}s", started.elapsed().as_secs_f64()); + + Measured { + key, + label, + first_ms: Some(first_ms), + samples, + scale, + baseline: None, + } +} + +fn with_baseline(mut m: Measured, b: Option) -> Measured { + m.baseline = b; + m +} + +/// The eleven reads `refreshAll` issues, all at once. +/// +/// A `Barrier` rather than "spawn eleven threads and hope": spawning is slow +/// enough that the first read can finish before the last starts, which would +/// quietly turn the one measurement that exists to find lock contention into a +/// measurement with no contention in it. +fn open_screen(subject: &Subject, encode: bool) -> f64 { + let backend = Arc::new(Libgit2Backend::new()); + let handle = backend.open(&subject.path).expect("open"); + let id = handle.id.clone(); + + const N: usize = 11; + let gate = Arc::new(Barrier::new(N + 1)); + let mut threads = Vec::with_capacity(N); + for slot in 0..N { + let backend = Arc::clone(&backend); + let gate = Arc::clone(&gate); + let id = id.clone(); + threads.push(std::thread::spawn(move || { + gate.wait(); + // Each arm produces the JSON the command handler would return, so + // `encode` can charge for the same bytes the webview receives. + let json = match slot { + 0 => serde_json::to_string(&backend.status(&id).expect("status")), + 1 => serde_json::to_string(&backend.branches(&id).expect("branches")), + 2 => serde_json::to_string(&backend.tags(&id).expect("tags")), + 3 => serde_json::to_string(&backend.stashes(&id).expect("stashes")), + 4 => serde_json::to_string(&backend.remotes(&id).expect("remotes")), + 5 => serde_json::to_string( + &backend + .log_page(&id, None, None, PAGE_SIZE) + .expect("log_page"), + ), + 6 => serde_json::to_string(&backend.repo_state(&id).expect("repo_state")), + 7 => serde_json::to_string(&backend.rebase_status(&id).expect("rebase_status")), + 8 => serde_json::to_string(&backend.bisect_status(&id).expect("bisect_status")), + 9 => serde_json::to_string(&backend.head_info(&id).expect("head_info")), + _ => serde_json::to_string(&backend.shallow_info(&id).expect("shallow_info")), + }; + if encode { + // Touch the string so the encode cannot be optimised away. + std::hint::black_box(json.expect("encode").len()); + } + })); + } + + gate.wait(); + let at = Instant::now(); + for t in threads { + t.join().expect("a read panicked"); + } + ms(at.elapsed()) +} + +fn measure_open_screen(key: &'static str, label: &'static str, subject: &Subject, cfg: &Config, encode: bool) -> Measured { + eprint!(" {key} … "); + let started = Instant::now(); + let first_ms = open_screen(subject, encode); + for _ in 0..warmups(first_ms, cfg) { + open_screen(subject, encode); + } + let samples: Vec = (0..repeats(first_ms, cfg)) + .map(|_| open_screen(subject, encode)) + .collect(); + eprintln!("11 reads in {:.1}s", started.elapsed().as_secs_f64()); + Measured { + key, + label, + first_ms: Some(first_ms), + samples, + scale: "11 concurrent reads".to_string(), + baseline: None, + } +} + +fn thousands(n: usize) -> String { + let s = n.to_string(); + let mut out = String::new(); + for (i, c) in s.chars().enumerate() { + if i > 0 && (s.len() - i) % 3 == 0 { + out.push(','); + } + out.push(c); + } + out +} + +fn run_suite(subject: &Subject, cfg: &Config) -> Vec { + let repo = subject.path.as_path(); + let mut out = Vec::new(); + + // Opening the repository is its own cost, and on a repository with 5,000 + // refs it is not a rounding error. + // + // Measured on its own rather than through `measure`, because `open` is the + // one op for which "repeat it on an existing handle" is meaningless: every + // call has to build a new one, so `first` and `repeat` measure the same + // thing and the samples are simply that thing many times. + let open_first = { + let backend = Libgit2Backend::new(); + time(|| backend.open(&subject.path).expect("open")).0 + }; + let open_samples: Vec = (0..repeats(open_first, cfg)) + .map(|_| { + let backend = Libgit2Backend::new(); + time(|| backend.open(&subject.path).expect("open")).0 + }) + .collect(); + out.push(Measured { + key: "open", + label: "Open the repository", + first_ms: Some(open_first), + samples: open_samples, + scale: "a fresh handle".to_string(), + baseline: measure_baseline(repo, cfg, &[&["rev-parse", "HEAD"]]), + }); + + out.push(with_baseline( + measure( + "status", + "Working-tree status", + subject, + cfg, + |v: &Vec| { + format!("{} entries", thousands(v.len())) + }, + |b, id| b.status(id).expect("status"), + ), + // Three invocations because one `GitBackend::status` answers all three + // questions: what changed, and how many lines on each side. + measure_baseline( + repo, + cfg, + &[ + &["status", "--porcelain=v1", "--untracked-files=all"], + &["diff", "--numstat"], + &["diff", "--cached", "--numstat"], + ], + ), + )); + + out.push(with_baseline( + measure( + "log_first_page", + "First page of history (500 commits)", + subject, + cfg, + |p: &platypusgit_lib::git::types::LogPage| { + format!("{} commits", thousands(p.commits.len())) + }, + |b, id| b.log_page(id, None, None, PAGE_SIZE).expect("log_page"), + ), + // `--topo-order`, because `log_page` walks with + // `Sort::TIME | Sort::TOPOLOGICAL` and the commit graph's lanes depend + // on it — a plain `git log` is a strictly easier question and quoting + // it here would be the `status` mistake in the module doc, made the + // other way round. + // + // Measured, not assumed: on the `deep` fixture a default `git log -500` + // is 41 ms and `--topo-order` is 284 ms, against our 275 ms. The first + // page is at PARITY. Comparing against the 41 ms would have published a + // fourteen-fold regression that does not exist. + measure_baseline( + repo, + cfg, + &[&[ + "log", + "--topo-order", + "--max-count=500", + "--format=%H%n%an%n%ae%n%at%n%s", + ]], + ), + )); + + // The paged tail. `s.commits` is a prefix of history, so scrolling past the + // first screen is a real backend call, and it is the one that gets slower + // the further in you are if the walk is restarted each time. + out.push(with_baseline( + measure( + "log_page_deep", + "Page 10 of history (commits 4,501–5,000)", + subject, + cfg, + |p: &platypusgit_lib::git::types::LogPage| { + format!("{} commits", thousands(p.commits.len())) + }, + |b, id| { + let mut page = b.log_page(id, None, None, PAGE_SIZE).expect("log_page"); + for _ in 1..DEEP_PAGES { + // A fixture shallower than ten pages simply stops early and + // reports the last page it reached — the `scale` column says + // how many commits that was, so a short result is visible + // rather than passed off as a fast one. + let Some(cursor) = page.next_cursor.clone() else { break }; + page = b + .log_page(id, None, Some(&cursor), PAGE_SIZE) + .expect("log_page"); + } + page + }, + ), + // Same order, and `--skip` rather than ten invocations on purpose: git + // pays for the topological sort ONCE and then skips. That asymmetry is + // the point of this row — it is what turns "we are at parity on page + // one" into a number for what paging actually costs. + measure_baseline( + repo, + cfg, + &[&[ + "log", + "--topo-order", + "--skip=4500", + "--max-count=500", + "--format=%H%n%an%n%ae%n%at%n%s", + ]], + ), + )); + + out.push(with_baseline( + measure( + "branches", + "List every branch", + subject, + cfg, + |v: &Vec| { + format!("{} branches", thousands(v.len())) + }, + |b, id| b.branches(id).expect("branches"), + ), + measure_baseline( + repo, + cfg, + &[&[ + "for-each-ref", + "--format=%(refname)%(objectname)%(upstream)", + "refs/heads", + "refs/remotes", + ]], + ), + )); + + out.push(with_baseline( + measure( + "tags", + "List every tag", + subject, + cfg, + |v: &Vec| format!("{} tags", thousands(v.len())), + |b, id| b.tags(id).expect("tags"), + ), + measure_baseline( + repo, + cfg, + &[&[ + "for-each-ref", + "--format=%(refname)%(objectname)%(*objectname)", + "refs/tags", + ]], + ), + )); + + let head_oid = subject.head_oid.clone(); + out.push(with_baseline( + measure( + "diff_commit", + "Diff the selected commit", + subject, + cfg, + |v: &Vec| format!("{} files", thousands(v.len())), + move |b, id| b.diff_commit(id, &head_oid, CONTEXT, false).expect("diff_commit"), + ), + measure_baseline( + repo, + cfg, + &[&["show", "--format=", "--patch", "HEAD"]], + ), + )); + + if let Some(path) = subject.hot_path.clone() { + let for_history = path.clone(); + out.push(with_baseline( + measure( + "file_history", + "History of one file (500 commits)", + subject, + cfg, + |v: &Vec| { + format!("{} commits", thousands(v.len())) + }, + move |b, id| b.file_history(id, &for_history, PAGE_SIZE).expect("file_history"), + ), + // Deliberately NOT `--follow`. `Libgit2Backend::file_history` is a + // plain path filter over the walk — it does not detect renames — so + // a `--follow` baseline would be asking the harder question and + // flattering us with the difference. + measure_baseline( + repo, + cfg, + &[&[ + "log", + "--topo-order", + "--max-count=500", + "--format=%H%n%an%n%at%n%s", + "--", + path.to_str().expect("utf-8 fixture path"), + ]], + ), + )); + } + + if let Some(path) = subject.dirty_path.clone() { + out.push(with_baseline( + measure( + "diff_workdir_file", + "Diff one modified file", + subject, + cfg, + |d: &platypusgit_lib::git::types::FileDiff| format!("{} hunks", d.hunks.len()), + move |b, id| { + b.diff(id, &path, DiffKind::WorktreeToIndex, CONTEXT, false) + .expect("diff") + }, + ), + None, + )); + } + + out.push(with_baseline( + measure( + "list_all_files", + "Browse the whole tree", + subject, + cfg, + |v: &Vec| { + format!("{} files", thousands(v.len())) + }, + |b, id| b.list_all_files(id).expect("list_all_files"), + ), + measure_baseline( + repo, + cfg, + &[&["ls-files", "--cached", "--others", "--exclude-standard"]], + ), + )); + + out.push(measure_open_screen( + "open_screen", + "Everything the first screen needs, at once", + subject, + cfg, + false, + )); + out.push(measure_open_screen( + "open_screen_ipc", + "…including encoding it all for the webview", + subject, + cfg, + true, + )); + + out +} + +// --------------------------------------------------------------------------- +// The soak +// --------------------------------------------------------------------------- + +/// Resident set size of this process, in megabytes. +/// +/// `ps` rather than a crate: adding a dependency to the shipped manifest for a +/// benchmark would put it in the dependency tree the privacy guard reads, and +/// one subprocess every few seconds is beneath the noise of what is being +/// measured. +fn rss_mb() -> Option { + let out = Command::new("ps") + .args(["-o", "rss=", "-p", &std::process::id().to_string()]) + .output() + .ok()?; + let kb: f64 = String::from_utf8_lossy(&out.stdout).trim().parse().ok()?; + Some(kb / 1024.0) +} + +/// "Jank after a while" is the complaint nobody else answers, so it gets its own +/// mode: repeat the whole first-screen fan-out for `minutes` and report both +/// what memory did and whether the operation itself got slower. +/// +/// Reported as two halves rather than a slope, because a slope invites reading +/// a trend into noise. First half versus second half of the same run, on the +/// same machine, is a comparison that either shows something or does not. +fn soak(subject: &Subject, minutes: f64) -> JsonValue { + let until = Instant::now() + Duration::from_secs_f64(minutes * 60.0); + let start_rss = rss_mb(); + let mut peak = start_rss.unwrap_or(0.0); + let mut samples: Vec = Vec::new(); + + while Instant::now() < until { + samples.push(open_screen(subject, true)); + if let Some(r) = rss_mb() { + peak = peak.max(r); + } + } + + let end_rss = rss_mb(); + let half = samples.len() / 2; + let first_half = Stats::of(&samples[..half]); + let second_half = Stats::of(&samples[half..]); + + JsonValue::Ordered(vec![ + ("minutes".into(), json_num(minutes)), + ("iterations".into(), json_num(samples.len() as f64)), + ("rssStartMb".into(), start_rss.map(json_num).unwrap_or(json_null())), + ("rssEndMb".into(), end_rss.map(json_num).unwrap_or(json_null())), + ("rssPeakMb".into(), json_num(peak)), + ( + "firstHalfMedianMs".into(), + first_half.map(|s| json_num(s.median)).unwrap_or(json_null()), + ), + ( + "secondHalfMedianMs".into(), + second_half.map(|s| json_num(s.median)).unwrap_or(json_null()), + ), + ]) +} + +// --------------------------------------------------------------------------- +// JSON output +// +// Written by hand rather than with a `Serialize` derive so the key order in the +// published file is the order it is written here. A generated file that +// reorders itself between runs makes every diff of it unreadable, and this one +// is committed. +// --------------------------------------------------------------------------- + +enum JsonValue { + Null, + Num(f64), + Str(String), + Array(Vec), + /// Every object here is written in the order its fields should be READ, so + /// a map that sorts them is the wrong container. + Ordered(Vec<(String, JsonValue)>), +} + +fn json_num(v: f64) -> JsonValue { + // Three decimals is past the precision anything here is repeatable to, and + // it keeps the committed file from churning on the last bit. + JsonValue::Num((v * 1000.0).round() / 1000.0) +} +fn json_str(v: impl Into) -> JsonValue { + JsonValue::Str(v.into()) +} +fn json_null() -> JsonValue { + JsonValue::Null +} + +fn escape(s: &str) -> String { + let mut out = String::with_capacity(s.len() + 2); + for c in s.chars() { + match c { + '"' => out.push_str("\\\""), + '\\' => out.push_str("\\\\"), + '\n' => out.push_str("\\n"), + '\t' => out.push_str("\\t"), + c if (c as u32) < 0x20 => out.push_str(&format!("\\u{:04x}", c as u32)), + c => out.push(c), + } + } + out +} + +fn render(v: &JsonValue, indent: usize, out: &mut String) { + let pad = " ".repeat(indent); + let inner = " ".repeat(indent + 1); + match v { + JsonValue::Null => out.push_str("null"), + JsonValue::Num(n) => { + if n.fract() == 0.0 && n.abs() < 1e15 { + out.push_str(&format!("{}", *n as i64)); + } else { + out.push_str(&format!("{n}")); + } + } + JsonValue::Str(s) => out.push_str(&format!("\"{}\"", escape(s))), + JsonValue::Array(items) => { + if items.is_empty() { + out.push_str("[]"); + return; + } + out.push_str("[\n"); + for (i, item) in items.iter().enumerate() { + out.push_str(&inner); + render(item, indent + 1, out); + if i + 1 < items.len() { + out.push(','); + } + out.push('\n'); + } + out.push_str(&pad); + out.push(']'); + } + JsonValue::Ordered(pairs) => { + let pairs: Vec<(String, &JsonValue)> = + pairs.iter().map(|(k, v)| (k.clone(), v)).collect(); + render_pairs(&pairs, indent, out); + } + } +} + +fn render_pairs(pairs: &[(String, &JsonValue)], indent: usize, out: &mut String) { + let pad = " ".repeat(indent); + let inner = " ".repeat(indent + 1); + if pairs.is_empty() { + out.push_str("{}"); + return; + } + out.push_str("{\n"); + for (i, (k, v)) in pairs.iter().enumerate() { + out.push_str(&inner); + out.push_str(&format!("\"{}\": ", escape(k))); + render(v, indent + 1, out); + if i + 1 < pairs.len() { + out.push(','); + } + out.push('\n'); + } + out.push_str(&pad); + out.push('}'); +} + +fn json_render(v: &JsonValue) -> String { + let mut s = String::new(); + render(v, 0, &mut s); + s +} + +fn measured_json(m: &Measured) -> JsonValue { + let stats = Stats::of(&m.samples); + let mut pairs: Vec<(String, JsonValue)> = vec![ + ("op".into(), json_str(m.key)), + ("label".into(), json_str(m.label)), + ("scale".into(), json_str(&m.scale)), + ( + "firstMs".into(), + m.first_ms.map(json_num).unwrap_or(json_null()), + ), + ( + "repeatMedianMs".into(), + stats.as_ref().map(|s| json_num(s.median)).unwrap_or(json_null()), + ), + ( + "repeatMinMs".into(), + stats.as_ref().map(|s| json_num(s.min)).unwrap_or(json_null()), + ), + ( + "repeatP95Ms".into(), + stats.as_ref().map(|s| json_num(s.p95)).unwrap_or(json_null()), + ), + ( + "repeatMaxMs".into(), + stats.as_ref().map(|s| json_num(s.max)).unwrap_or(json_null()), + ), + ( + "samples".into(), + JsonValue::Num(stats.as_ref().map(|s| s.n as f64).unwrap_or(0.0)), + ), + ( + "sampleMs".into(), + JsonValue::Array(m.samples.iter().copied().map(json_num).collect()), + ), + ]; + // No ratio is computed here, on purpose. Whether a baseline is comparable + // at all depends on the spawn floor and on how many processes the baseline + // is, and that judgement belongs with the thing that PUBLISHES the number — + // `scripts/bench-report.mjs`. This file's job is to measure and to report + // what it measured, including the invocation count that makes the judgement + // possible. + match &m.baseline { + Some(b) => { + let bs = Stats::of(&b.samples); + pairs.push(("gitCommand".into(), json_str(&b.command))); + pairs.push(("gitInvocations".into(), json_num(b.invocations as f64))); + pairs.push(( + "gitMedianMs".into(), + bs.as_ref().map(|s| json_num(s.median)).unwrap_or(json_null()), + )); + } + None => { + pairs.push(("gitCommand".into(), json_null())); + pairs.push(("gitInvocations".into(), json_num(0.0))); + pairs.push(("gitMedianMs".into(), json_null())); + } + } + JsonValue::Ordered(pairs) +} + +// --------------------------------------------------------------------------- +// Machine description +// --------------------------------------------------------------------------- + +fn sysctl(key: &str) -> Option { + let out = Command::new("sysctl").args(["-n", key]).output().ok()?; + let s = String::from_utf8_lossy(&out.stdout).trim().to_string(); + (!s.is_empty()).then_some(s) +} + +fn cpu_model() -> String { + if let Some(v) = sysctl("machdep.cpu.brand_string") { + return v; + } + if let Ok(info) = std::fs::read_to_string("/proc/cpuinfo") { + for line in info.lines() { + if let Some(v) = line.strip_prefix("model name") { + return v.trim_start_matches([' ', ':']).trim().to_string(); + } + } + } + "unknown".into() +} + +fn memory_gb() -> Option { + if let Some(v) = sysctl("hw.memsize").and_then(|s| s.parse::().ok()) { + return Some(v / 1024.0 / 1024.0 / 1024.0); + } + let info = std::fs::read_to_string("/proc/meminfo").ok()?; + let line = info.lines().find(|l| l.starts_with("MemTotal:"))?; + let kb: f64 = line.split_whitespace().nth(1)?.parse().ok()?; + Some(kb / 1024.0 / 1024.0) +} + +fn git_version() -> String { + Command::new("git") + .arg("--version") + .output() + .ok() + .map(|o| String::from_utf8_lossy(&o.stdout).trim().to_string()) + .unwrap_or_else(|| "unknown".into()) +} + +fn machine_json() -> JsonValue { + JsonValue::Ordered(vec![ + ("os".into(), json_str(std::env::consts::OS)), + ("arch".into(), json_str(std::env::consts::ARCH)), + ("cpu".into(), json_str(cpu_model())), + ( + "cores".into(), + json_num( + std::thread::available_parallelism() + .map(|n| n.get() as f64) + .unwrap_or(0.0), + ), + ), + ( + "memoryGb".into(), + memory_gb().map(|g| json_num(g.round())).unwrap_or(json_null()), + ), + ("git".into(), json_str(git_version())), + ]) +} + +// --------------------------------------------------------------------------- +// Repository description +// --------------------------------------------------------------------------- + +fn count(repo: &Path, args: &[&str]) -> usize { + Command::new("git") + .arg("-C") + .arg(repo) + .args(args) + .output() + .ok() + .map(|o| String::from_utf8_lossy(&o.stdout).lines().count()) + .unwrap_or(0) +} + +fn repo_json(name: &str, subject: &Subject) -> JsonValue { + let repo = subject.path.as_path(); + let commits = Command::new("git") + .arg("-C") + .arg(repo) + .args(["rev-list", "--count", "HEAD"]) + .output() + .ok() + .and_then(|o| String::from_utf8_lossy(&o.stdout).trim().parse::().ok()) + .unwrap_or(0.0); + JsonValue::Ordered(vec![ + ("fixture".into(), json_str(name)), + ("commits".into(), json_num(commits)), + ( + "trackedFiles".into(), + json_num(count(repo, &["ls-files"]) as f64), + ), + ( + "branches".into(), + json_num(count(repo, &["for-each-ref", "--format=%(refname)", "refs/heads"]) as f64), + ), + ( + "tags".into(), + json_num(count(repo, &["for-each-ref", "--format=%(refname)", "refs/tags"]) as f64), + ), + ( + "dirtyEntries".into(), + json_num(count(repo, &["status", "--porcelain=v1", "--untracked-files=all"]) as f64), + ), + ]) +} + +// --------------------------------------------------------------------------- +// Entry point +// --------------------------------------------------------------------------- + +fn arg(args: &[String], name: &str) -> Option { + let at = args.iter().position(|a| a == name)?; + args.get(at + 1).cloned() +} + +fn usage() -> ! { + eprintln!( + "usage: repo_bench --repo --name \ + [--iterations N] [--warmup N] [--budget-seconds S] [--no-baseline] \ + [--soak-minutes M] [--out ]" + ); + std::process::exit(2) +} + +fn main() { + let args: Vec = std::env::args().collect(); + let Some(repo) = arg(&args, "--repo") else { usage() }; + let name = arg(&args, "--name").unwrap_or_else(|| "repo".into()); + let mut cfg = Config { + floor_ms: 0.0, + iterations: arg(&args, "--iterations") + .and_then(|v| v.parse().ok()) + .unwrap_or(10), + warmup: arg(&args, "--warmup").and_then(|v| v.parse().ok()).unwrap_or(2), + baseline: !args.iter().any(|a| a == "--no-baseline"), + budget_ms: arg(&args, "--budget-seconds") + .and_then(|v| v.parse::().ok()) + .unwrap_or(20.0) + * 1000.0, + }; + + let path = PathBuf::from(&repo); + if !path.join(".git").exists() { + eprintln!("not a git repository: {repo}"); + std::process::exit(1); + } + + eprintln!("benchmarking {name} at {repo}"); + cfg.floor_ms = spawn_floor_ms(&path); + eprintln!(" one `git` invocation costs {:.1} ms before doing anything", cfg.floor_ms); + let subject = probe(&path); + let measured = run_suite(&subject, &cfg); + + let soaked = arg(&args, "--soak-minutes") + .and_then(|v| v.parse::().ok()) + .map(|m| { + eprintln!(" soaking for {m} minute(s)…"); + soak(&subject, m) + }); + + let doc = JsonValue::Ordered(vec![ + ("fixture".into(), json_str(&name)), + ("repository".into(), repo_json(&name, &subject)), + ("machine".into(), machine_json()), + ("iterations".into(), json_num(cfg.iterations as f64)), + ("warmup".into(), json_num(cfg.warmup as f64)), + ("budgetSeconds".into(), json_num(cfg.budget_ms / 1000.0)), + ("gitSpawnFloorMs".into(), json_num(cfg.floor_ms)), + ( + "operations".into(), + JsonValue::Array( + measured + .iter() + .map(measured_json) + .collect(), + ), + ), + ("soak".into(), soaked.unwrap_or_else(json_null)), + ]); + + let rendered = json_render(&doc); + match arg(&args, "--out") { + Some(out) => { + std::fs::write(&out, format!("{rendered}\n")).expect("write results"); + eprintln!(" wrote {out}"); + } + None => println!("{rendered}"), + } +} From c898b8986534a3fda30ee4a3b3f1b9a6ab4639c5 Mon Sep 17 00:00:00 2001 From: Jonas Aasberg Date: Thu, 17 Sep 2026 13:45:17 +0200 Subject: [PATCH 2/3] docs(perf): publish the first run, and what it found MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The method, what each number does and does not include, and the results. The good. The whole first screen — eleven concurrent reads — costs 255 ms on a 50,000-commit repository and 219 ms on one with 7,001 refs. Opening a repository is 0.11 ms. On the wide fixture, open_screen (5.42 s) equals status alone (5.42 s): eleven concurrent reads cost what the slowest costs rather than the sum, which is repo_locks.rs doing its job and the single thing most worth not regressing. A ten-minute soak ran 2,344 fan-outs with memory flat at 67 to 69 MB and the median identical in both halves, which is the "jank after a while" complaint answered with data. The bad. Opening torvalds/linux costs 15.8 seconds, and ten pages into its history costs two minutes and thirty-eight seconds. Three findings, each with the measurement attached. We are at parity with git log --topo-order on the first page, so the topological sort is the cost and git pays it too — but git pays it once and then skips, while we re-pay it per page, and the per-page cost is flat in depth. collect_ref_map is rebuilt per page as well, which is 16x git's work on a 2,000-commit repository with 7,001 refs. And the obvious cheap fix is ruled out: writing a commit-graph takes git from 9.51 s to 21 ms on the kernel and does nothing for us, because libgit2's revwalk does not read it. Why publish the bad ones: the user who opens a 1.4-million-commit repository and waits is precisely the user who went looking for an alternative. Learning that here is the entire point, and a document that lists only the flattering half is an adjective with extra steps. The published record sits beside the document rather than under site/. #257 asks for a figure on the marketing site and the block for it is written, but the honest headline today is fifteen point eight seconds, and the right response to that number is to fix it rather than to sell with it. Keeping the data out of site/ also means a re-measurement can never trigger the site deployment, which fires on any push to main touching that path. Co-Authored-By: Claude Opus 5 (1M context) --- CLAUDE.md | 5 + CONTRIBUTING.md | 15 + docs/dev/benchmark.json | 785 ++++++++++++++++++ docs/dev/performance.md | 336 ++++++++ docs/dev/testing.md | 11 +- .../plans/2026-09-17-large-repo-benchmark.md | 150 ++++ .../2026-09-17-large-repo-benchmark-spec.md | 211 +++++ 7 files changed, 1512 insertions(+), 1 deletion(-) create mode 100644 docs/dev/benchmark.json create mode 100644 docs/dev/performance.md create mode 100644 docs/superpowers/plans/2026-09-17-large-repo-benchmark.md create mode 100644 docs/superpowers/specs/2026-09-17-large-repo-benchmark-spec.md diff --git a/CLAUDE.md b/CLAUDE.md index 0d9ba6da..cdef78e3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -27,6 +27,10 @@ to what you are reading. A new section here needs a reason a pointer cannot serv and credentials, signing, stash, spawning processes, bisect, async/threading. - `docs/dev/distribution.md` — `pgit` CLI packaging per channel, the launch detach, Tauri permissions. +- `docs/dev/performance.md` — the large-repo benchmark: the fixtures, what each + number does and does not include, the baseline it is compared against, and + the published results. Read it before touching the log walk, `status`, or + the refresh path, and re-run `pnpm bench` when you do. - `docs/dev/releasing.md` — what a version number means and when to bump which part, the cut-a-release runbook (changelog lands on `main` FIRST), and the prerelease-promotion traps. Read it before tagging anything. @@ -58,6 +62,7 @@ pnpm test # vitest (unit logic + component tes pnpm test:e2e:docker # e2e — THE way to run e2e (headless, same stack as CI) pnpm test:e2e:docker run --spec e2e/specs/X.e2e.ts # ...one spec against this worktree's snapshot pnpm exec tsc -p e2e/tsconfig.json --noEmit # e2e typecheck gate (root tsc excludes e2e/) +pnpm bench # large-repo benchmark (docs/dev/performance.md) ``` **Local production builds need the updater signing key.** `tauri.conf.json` diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 611c752e..9c11a795 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -171,6 +171,21 @@ Add them alongside the code you change: - New component → `*.test.tsx`. - New user-facing flow, or a bug that reached the UI → an e2e spec. +### Measuring performance + +Performance is measured rather than tested, and it is not a CI gate — wall clock +on a shared runner is a flake generator. + +```bash +pnpm bench # three generated fixtures, built in seconds +pnpm bench --linux # …plus a real clone of torvalds/linux (multi-GB) +``` + +Run it on a quiet machine if you have touched the log walk, `status` or the +refresh path, and commit what it writes — `docs/dev/performance.md` explains +what each number does and does not include, and the figures on the website come +from the same run. + ## Production bundles ```bash diff --git a/docs/dev/benchmark.json b/docs/dev/benchmark.json new file mode 100644 index 00000000..6aa03780 --- /dev/null +++ b/docs/dev/benchmark.json @@ -0,0 +1,785 @@ +{ + "measuredOn": "2026-09-17", + "machine": { + "cpu": "Apple M4 Pro", + "cores": 14, + "memoryGb": 48, + "os": "macos/aarch64", + "gitVersion": "git version 2.50.1 (Apple Git-155)" + }, + "iterations": 10, + "budgetSeconds": 20, + "gitSpawnFloorMs": 12.411, + "fixtures": [ + { + "key": "linux", + "title": "torvalds/linux", + "kind": "real", + "blurb": "A real clone of the Linux kernel: the repository people mean when they say a git client is slow.", + "repository": { + "fixture": "linux", + "commits": 1482923, + "trackedFiles": 96034, + "branches": 1, + "tags": 946, + "dirtyEntries": 13 + }, + "gitSpawnFloorMs": 12.411, + "operations": [ + { + "op": "open", + "label": "Open the repository", + "scale": "a fresh handle", + "samples": 10, + "firstMs": 0.257, + "repeatMedianMs": 0.109, + "repeatP95Ms": 0.125, + "gitCommand": "git rev-parse HEAD", + "gitInvocations": 1, + "gitMedianMs": 12.24, + "gitWorkMs": 0, + "gitFloorBound": true, + "ratioToGit": null + }, + { + "op": "open_screen", + "label": "Everything the first screen needs, at once", + "scale": "11 concurrent reads", + "samples": 3, + "firstMs": 15905.468, + "repeatMedianMs": 15844.188, + "repeatP95Ms": 15855.63, + "gitCommand": null, + "gitInvocations": 0, + "gitMedianMs": null, + "gitWorkMs": null, + "gitFloorBound": false, + "ratioToGit": null + }, + { + "op": "open_screen_ipc", + "label": "…including encoding it all for the webview", + "scale": "11 concurrent reads", + "samples": 3, + "firstMs": 15799.373, + "repeatMedianMs": 15818.08, + "repeatP95Ms": 15856.463, + "gitCommand": null, + "gitInvocations": 0, + "gitMedianMs": null, + "gitWorkMs": null, + "gitFloorBound": false, + "ratioToGit": null + }, + { + "op": "status", + "label": "Working-tree status", + "scale": "26 entries", + "samples": 10, + "firstMs": 1150.178, + "repeatMedianMs": 989.035, + "repeatP95Ms": 1104.831, + "gitCommand": "git status --porcelain=v1 --untracked-files=all && git diff --numstat && git diff --cached --numstat", + "gitInvocations": 3, + "gitMedianMs": 554.112, + "gitWorkMs": 516.879, + "gitFloorBound": false, + "ratioToGit": 1.913474913857982 + }, + { + "op": "log_first_page", + "label": "First page of history", + "scale": "500 commits", + "samples": 3, + "firstMs": 15956.168, + "repeatMedianMs": 15952.356, + "repeatP95Ms": 16486.088, + "gitCommand": "git log --topo-order --max-count=500 --format=%H%n%an%n%ae%n%at%n%s", + "gitInvocations": 1, + "gitMedianMs": 9517.414, + "gitWorkMs": 9505.003, + "gitFloorBound": false, + "ratioToGit": 1.6783115165771119 + }, + { + "op": "log_page_deep", + "label": "Ten pages into history", + "scale": "500 commits", + "samples": 3, + "firstMs": 156377.157, + "repeatMedianMs": 157671.22, + "repeatP95Ms": 159939.268, + "gitCommand": "git log --topo-order --skip=4500 --max-count=500 --format=%H%n%an%n%ae%n%at%n%s", + "gitInvocations": 1, + "gitMedianMs": 9692.322, + "gitWorkMs": 9679.911, + "gitFloorBound": false, + "ratioToGit": 16.288498933512923 + }, + { + "op": "branches", + "label": "List every branch", + "scale": "3 branches", + "samples": 10, + "firstMs": 1.649, + "repeatMedianMs": 0.873, + "repeatP95Ms": 0.891, + "gitCommand": "git for-each-ref --format=%(refname)%(objectname)%(upstream) refs/heads refs/remotes", + "gitInvocations": 1, + "gitMedianMs": 12.951, + "gitWorkMs": 0.5400000000000009, + "gitFloorBound": true, + "ratioToGit": null + }, + { + "op": "tags", + "label": "List every tag", + "scale": "946 tags", + "samples": 10, + "firstMs": 27.099, + "repeatMedianMs": 19.709, + "repeatP95Ms": 20.166, + "gitCommand": "git for-each-ref --format=%(refname)%(objectname)%(*objectname) refs/tags", + "gitInvocations": 1, + "gitMedianMs": 36.108, + "gitWorkMs": 23.696999999999996, + "gitFloorBound": false, + "ratioToGit": 0.8317086551040217 + }, + { + "op": "diff_commit", + "label": "Diff the selected commit", + "scale": "3 files", + "samples": 10, + "firstMs": 141.665, + "repeatMedianMs": 138.7, + "repeatP95Ms": 140.429, + "gitCommand": "git show --format= --patch HEAD", + "gitInvocations": 1, + "gitMedianMs": 23.911, + "gitWorkMs": 11.500000000000002, + "gitFloorBound": false, + "ratioToGit": 12.060869565217388 + }, + { + "op": "diff_workdir_file", + "label": "Diff one modified file", + "scale": "2 hunks", + "samples": 10, + "firstMs": 68.161, + "repeatMedianMs": 2.136, + "repeatP95Ms": 2.219, + "gitCommand": null, + "gitInvocations": 0, + "gitMedianMs": null, + "gitWorkMs": null, + "gitFloorBound": false, + "ratioToGit": null + }, + { + "op": "file_history", + "label": "History of one file", + "scale": "500 commits", + "samples": 3, + "firstMs": 15913.163, + "repeatMedianMs": 15419.268, + "repeatP95Ms": 16055.124, + "gitCommand": "git log --topo-order --max-count=500 --format=%H%n%an%n%at%n%s -- MAINTAINERS", + "gitInvocations": 1, + "gitMedianMs": 5667.106, + "gitWorkMs": 5654.695, + "gitFloorBound": false, + "ratioToGit": 2.726808077181882 + }, + { + "op": "list_all_files", + "label": "Browse the whole tree", + "scale": "96,034 files", + "samples": 10, + "firstMs": 781.816, + "repeatMedianMs": 515.422, + "repeatP95Ms": 642.301, + "gitCommand": "git ls-files --cached --others --exclude-standard", + "gitInvocations": 1, + "gitMedianMs": 230.862, + "gitWorkMs": 218.451, + "gitFloorBound": false, + "ratioToGit": 2.359439874388307 + } + ], + "soak": null + }, + { + "key": "deep", + "title": "deep", + "kind": "generated", + "blurb": "50,000 commits over a small tree — the size past which GitKraken's own users report native clients beating it.", + "repository": { + "fixture": "deep", + "commits": 50000, + "trackedFiles": 16, + "branches": 1, + "tags": 0, + "dirtyEntries": 0 + }, + "gitSpawnFloorMs": 12.275, + "operations": [ + { + "op": "open", + "label": "Open the repository", + "scale": "a fresh handle", + "samples": 10, + "firstMs": 0.141, + "repeatMedianMs": 0.114, + "repeatP95Ms": 0.122, + "gitCommand": "git rev-parse HEAD", + "gitInvocations": 1, + "gitMedianMs": 12.244, + "gitWorkMs": 0, + "gitFloorBound": true, + "ratioToGit": null + }, + { + "op": "open_screen", + "label": "Everything the first screen needs, at once", + "scale": "11 concurrent reads", + "samples": 10, + "firstMs": 252.398, + "repeatMedianMs": 252.638, + "repeatP95Ms": 255.802, + "gitCommand": null, + "gitInvocations": 0, + "gitMedianMs": null, + "gitWorkMs": null, + "gitFloorBound": false, + "ratioToGit": null + }, + { + "op": "open_screen_ipc", + "label": "…including encoding it all for the webview", + "scale": "11 concurrent reads", + "samples": 10, + "firstMs": 251.635, + "repeatMedianMs": 252.584, + "repeatP95Ms": 254.367, + "gitCommand": null, + "gitInvocations": 0, + "gitMedianMs": null, + "gitWorkMs": null, + "gitFloorBound": false, + "ratioToGit": null + }, + { + "op": "status", + "label": "Working-tree status", + "scale": "0 entries", + "samples": 10, + "firstMs": 0.688, + "repeatMedianMs": 0.525, + "repeatP95Ms": 0.554, + "gitCommand": "git status --porcelain=v1 --untracked-files=all && git diff --numstat && git diff --cached --numstat", + "gitInvocations": 3, + "gitMedianMs": 39.411, + "gitWorkMs": 2.5859999999999985, + "gitFloorBound": true, + "ratioToGit": null + }, + { + "op": "log_first_page", + "label": "First page of history", + "scale": "500 commits", + "samples": 10, + "firstMs": 261.376, + "repeatMedianMs": 249.132, + "repeatP95Ms": 251.38, + "gitCommand": "git log --topo-order --max-count=500 --format=%H%n%an%n%ae%n%at%n%s", + "gitInvocations": 1, + "gitMedianMs": 205.036, + "gitWorkMs": 192.761, + "gitFloorBound": false, + "ratioToGit": 1.292439860760216 + }, + { + "op": "log_page_deep", + "label": "Ten pages into history", + "scale": "500 commits", + "samples": 8, + "firstMs": 2392.178, + "repeatMedianMs": 2393.558, + "repeatP95Ms": 2400.148, + "gitCommand": "git log --topo-order --skip=4500 --max-count=500 --format=%H%n%an%n%ae%n%at%n%s", + "gitInvocations": 1, + "gitMedianMs": 204.502, + "gitWorkMs": 192.227, + "gitFloorBound": false, + "ratioToGit": 12.4517263443741 + }, + { + "op": "branches", + "label": "List every branch", + "scale": "1 branch", + "samples": 10, + "firstMs": 0.422, + "repeatMedianMs": 0.281, + "repeatP95Ms": 0.29, + "gitCommand": "git for-each-ref --format=%(refname)%(objectname)%(upstream) refs/heads refs/remotes", + "gitInvocations": 1, + "gitMedianMs": 13.103, + "gitWorkMs": 0.8279999999999994, + "gitFloorBound": true, + "ratioToGit": null + }, + { + "op": "tags", + "label": "List every tag", + "scale": "0 tags", + "samples": 10, + "firstMs": 0.175, + "repeatMedianMs": 0.147, + "repeatP95Ms": 0.15, + "gitCommand": "git for-each-ref --format=%(refname)%(objectname)%(*objectname) refs/tags", + "gitInvocations": 1, + "gitMedianMs": 12.833, + "gitWorkMs": 0.5579999999999998, + "gitFloorBound": true, + "ratioToGit": null + }, + { + "op": "diff_commit", + "label": "Diff the selected commit", + "scale": "1 file", + "samples": 10, + "firstMs": 0.477, + "repeatMedianMs": 0.338, + "repeatP95Ms": 0.351, + "gitCommand": "git show --format= --patch HEAD", + "gitInvocations": 1, + "gitMedianMs": 13.195, + "gitWorkMs": 0.9199999999999999, + "gitFloorBound": true, + "ratioToGit": null + }, + { + "op": "file_history", + "label": "History of one file", + "scale": "500 commits", + "samples": 10, + "firstMs": 293.063, + "repeatMedianMs": 65.002, + "repeatP95Ms": 65.738, + "gitCommand": "git log --topo-order --max-count=500 --format=%H%n%an%n%at%n%s -- src/module_00.rs", + "gitInvocations": 1, + "gitMedianMs": 331.471, + "gitWorkMs": 319.196, + "gitFloorBound": false, + "ratioToGit": 0.20364290279326805 + }, + { + "op": "list_all_files", + "label": "Browse the whole tree", + "scale": "16 files", + "samples": 10, + "firstMs": 0.497, + "repeatMedianMs": 0.129, + "repeatP95Ms": 0.133, + "gitCommand": "git ls-files --cached --others --exclude-standard", + "gitInvocations": 1, + "gitMedianMs": 12.845, + "gitWorkMs": 0.5700000000000003, + "gitFloorBound": true, + "ratioToGit": null + } + ], + "soak": { + "minutes": 10, + "iterations": 2344, + "rssStartMb": 66.5, + "rssEndMb": 69.047, + "rssPeakMb": 69.047, + "firstHalfMedianMs": 252.302, + "secondHalfMedianMs": 251.693 + } + }, + { + "key": "wide", + "title": "wide", + "kind": "generated", + "blurb": "50,000 tracked files with every one of them modified, plus 5,000 untracked — a monorepo just after a codemod.", + "repository": { + "fixture": "wide", + "commits": 1, + "trackedFiles": 50000, + "branches": 1, + "tags": 0, + "dirtyEntries": 55000 + }, + "gitSpawnFloorMs": 12.058, + "operations": [ + { + "op": "open", + "label": "Open the repository", + "scale": "a fresh handle", + "samples": 10, + "firstMs": 0.25, + "repeatMedianMs": 0.114, + "repeatP95Ms": 0.134, + "gitCommand": "git rev-parse HEAD", + "gitInvocations": 1, + "gitMedianMs": 12.46, + "gitWorkMs": 0.402000000000001, + "gitFloorBound": true, + "ratioToGit": null + }, + { + "op": "open_screen", + "label": "Everything the first screen needs, at once", + "scale": "11 concurrent reads", + "samples": 3, + "firstMs": 5420.711, + "repeatMedianMs": 5420.4, + "repeatP95Ms": 5622.622, + "gitCommand": null, + "gitInvocations": 0, + "gitMedianMs": null, + "gitWorkMs": null, + "gitFloorBound": false, + "ratioToGit": null + }, + { + "op": "open_screen_ipc", + "label": "…including encoding it all for the webview", + "scale": "11 concurrent reads", + "samples": 3, + "firstMs": 5602.161, + "repeatMedianMs": 5432.862, + "repeatP95Ms": 5601.451, + "gitCommand": null, + "gitInvocations": 0, + "gitMedianMs": null, + "gitWorkMs": null, + "gitFloorBound": false, + "ratioToGit": null + }, + { + "op": "status", + "label": "Working-tree status", + "scale": "55,000 entries", + "samples": 3, + "firstMs": 5380.675, + "repeatMedianMs": 5419.887, + "repeatP95Ms": 5482.045, + "gitCommand": "git status --porcelain=v1 --untracked-files=all && git diff --numstat && git diff --cached --numstat", + "gitInvocations": 3, + "gitMedianMs": 3014.503, + "gitWorkMs": 2978.329, + "gitFloorBound": false, + "ratioToGit": 1.8197744439919161 + }, + { + "op": "log_first_page", + "label": "First page of history", + "scale": "1 commit", + "samples": 10, + "firstMs": 0.353, + "repeatMedianMs": 0.251, + "repeatP95Ms": 0.256, + "gitCommand": "git log --topo-order --max-count=500 --format=%H%n%an%n%ae%n%at%n%s", + "gitInvocations": 1, + "gitMedianMs": 13.179, + "gitWorkMs": 1.1210000000000004, + "gitFloorBound": true, + "ratioToGit": null + }, + { + "op": "log_page_deep", + "label": "Ten pages into history", + "scale": "1 commit", + "samples": 10, + "firstMs": 0.312, + "repeatMedianMs": 0.25, + "repeatP95Ms": 0.256, + "gitCommand": "git log --topo-order --skip=4500 --max-count=500 --format=%H%n%an%n%ae%n%at%n%s", + "gitInvocations": 1, + "gitMedianMs": 13.081, + "gitWorkMs": 1.0229999999999997, + "gitFloorBound": true, + "ratioToGit": null + }, + { + "op": "branches", + "label": "List every branch", + "scale": "1 branch", + "samples": 10, + "firstMs": 0.352, + "repeatMedianMs": 0.275, + "repeatP95Ms": 0.317, + "gitCommand": "git for-each-ref --format=%(refname)%(objectname)%(upstream) refs/heads refs/remotes", + "gitInvocations": 1, + "gitMedianMs": 12.756, + "gitWorkMs": 0.6980000000000004, + "gitFloorBound": true, + "ratioToGit": null + }, + { + "op": "tags", + "label": "List every tag", + "scale": "0 tags", + "samples": 10, + "firstMs": 0.171, + "repeatMedianMs": 0.145, + "repeatP95Ms": 0.147, + "gitCommand": "git for-each-ref --format=%(refname)%(objectname)%(*objectname) refs/tags", + "gitInvocations": 1, + "gitMedianMs": 12.756, + "gitWorkMs": 0.6980000000000004, + "gitFloorBound": true, + "ratioToGit": null + }, + { + "op": "diff_commit", + "label": "Diff the selected commit", + "scale": "50,000 files", + "samples": 10, + "firstMs": 1530.386, + "repeatMedianMs": 1510.808, + "repeatP95Ms": 1674.506, + "gitCommand": "git show --format= --patch HEAD", + "gitInvocations": 1, + "gitMedianMs": 522.06, + "gitWorkMs": 510.00199999999995, + "gitFloorBound": false, + "ratioToGit": 2.9623570103646655 + }, + { + "op": "diff_workdir_file", + "label": "Diff one modified file", + "scale": "1 hunk", + "samples": 10, + "firstMs": 17.18, + "repeatMedianMs": 0.632, + "repeatP95Ms": 0.639, + "gitCommand": null, + "gitInvocations": 0, + "gitMedianMs": null, + "gitWorkMs": null, + "gitFloorBound": false, + "ratioToGit": null + }, + { + "op": "file_history", + "label": "History of one file", + "scale": "1 commit", + "samples": 10, + "firstMs": 0.276, + "repeatMedianMs": 0.1, + "repeatP95Ms": 0.105, + "gitCommand": "git log --topo-order --max-count=500 --format=%H%n%an%n%at%n%s -- pkg/000/gen_000000.ts", + "gitInvocations": 1, + "gitMedianMs": 13.32, + "gitWorkMs": 1.2620000000000005, + "gitFloorBound": true, + "ratioToGit": null + }, + { + "op": "list_all_files", + "label": "Browse the whole tree", + "scale": "55,000 files", + "samples": 10, + "firstMs": 195.887, + "repeatMedianMs": 181.487, + "repeatP95Ms": 240.574, + "gitCommand": "git ls-files --cached --others --exclude-standard", + "gitInvocations": 1, + "gitMedianMs": 54.416, + "gitWorkMs": 42.358, + "gitFloorBound": false, + "ratioToGit": 4.284597950800321 + } + ], + "soak": null + }, + { + "key": "refs", + "title": "refs", + "kind": "generated", + "blurb": "5,000 branches and 2,000 tags over a short history — a long-lived repository nobody prunes.", + "repository": { + "fixture": "refs", + "commits": 2000, + "trackedFiles": 32, + "branches": 5001, + "tags": 2000, + "dirtyEntries": 0 + }, + "gitSpawnFloorMs": 12.386, + "operations": [ + { + "op": "open", + "label": "Open the repository", + "scale": "a fresh handle", + "samples": 10, + "firstMs": 0.153, + "repeatMedianMs": 0.138, + "repeatP95Ms": 0.143, + "gitCommand": "git rev-parse HEAD", + "gitInvocations": 1, + "gitMedianMs": 12.342, + "gitWorkMs": 0, + "gitFloorBound": true, + "ratioToGit": null + }, + { + "op": "open_screen", + "label": "Everything the first screen needs, at once", + "scale": "11 concurrent reads", + "samples": 10, + "firstMs": 220.893, + "repeatMedianMs": 219.434, + "repeatP95Ms": 220.996, + "gitCommand": null, + "gitInvocations": 0, + "gitMedianMs": null, + "gitWorkMs": null, + "gitFloorBound": false, + "ratioToGit": null + }, + { + "op": "open_screen_ipc", + "label": "…including encoding it all for the webview", + "scale": "11 concurrent reads", + "samples": 10, + "firstMs": 220.518, + "repeatMedianMs": 219.773, + "repeatP95Ms": 224.361, + "gitCommand": null, + "gitInvocations": 0, + "gitMedianMs": null, + "gitWorkMs": null, + "gitFloorBound": false, + "ratioToGit": null + }, + { + "op": "status", + "label": "Working-tree status", + "scale": "0 entries", + "samples": 10, + "firstMs": 0.724, + "repeatMedianMs": 0.549, + "repeatP95Ms": 0.562, + "gitCommand": "git status --porcelain=v1 --untracked-files=all && git diff --numstat && git diff --cached --numstat", + "gitInvocations": 3, + "gitMedianMs": 38.918, + "gitWorkMs": 1.759999999999998, + "gitFloorBound": true, + "ratioToGit": null + }, + { + "op": "log_first_page", + "label": "First page of history", + "scale": "500 commits", + "samples": 10, + "firstMs": 786.833, + "repeatMedianMs": 135.184, + "repeatP95Ms": 138.328, + "gitCommand": "git log --topo-order --max-count=500 --format=%H%n%an%n%ae%n%at%n%s", + "gitInvocations": 1, + "gitMedianMs": 20.862, + "gitWorkMs": 8.475999999999999, + "gitFloorBound": false, + "ratioToGit": 15.949032562529496 + }, + { + "op": "log_page_deep", + "label": "Ten pages into history", + "scale": "500 commits", + "samples": 10, + "firstMs": 526.236, + "repeatMedianMs": 524.766, + "repeatP95Ms": 550.759, + "gitCommand": "git log --topo-order --skip=4500 --max-count=500 --format=%H%n%an%n%ae%n%at%n%s", + "gitInvocations": 1, + "gitMedianMs": 19.632, + "gitWorkMs": 7.246000000000002, + "gitFloorBound": false, + "ratioToGit": 72.42147391664363 + }, + { + "op": "branches", + "label": "List every branch", + "scale": "5,001 branches", + "samples": 10, + "firstMs": 184.019, + "repeatMedianMs": 180.081, + "repeatP95Ms": 181.962, + "gitCommand": "git for-each-ref --format=%(refname)%(objectname)%(upstream) refs/heads refs/remotes", + "gitInvocations": 1, + "gitMedianMs": 195.41, + "gitWorkMs": 183.024, + "gitFloorBound": false, + "ratioToGit": 0.983920141620771 + }, + { + "op": "tags", + "label": "List every tag", + "scale": "2,000 tags", + "samples": 10, + "firstMs": 148.003, + "repeatMedianMs": 148.38, + "repeatP95Ms": 150.987, + "gitCommand": "git for-each-ref --format=%(refname)%(objectname)%(*objectname) refs/tags", + "gitInvocations": 1, + "gitMedianMs": 55.724, + "gitWorkMs": 43.337999999999994, + "gitFloorBound": false, + "ratioToGit": 3.423785130832065 + }, + { + "op": "diff_commit", + "label": "Diff the selected commit", + "scale": "1 file", + "samples": 10, + "firstMs": 0.404, + "repeatMedianMs": 0.282, + "repeatP95Ms": 0.293, + "gitCommand": "git show --format= --patch HEAD", + "gitInvocations": 1, + "gitMedianMs": 12.971, + "gitWorkMs": 0.5850000000000009, + "gitFloorBound": true, + "ratioToGit": null + }, + { + "op": "file_history", + "label": "History of one file", + "scale": "63 commits", + "samples": 10, + "firstMs": 20.661, + "repeatMedianMs": 9.374, + "repeatP95Ms": 9.883, + "gitCommand": "git log --topo-order --max-count=500 --format=%H%n%an%n%at%n%s -- src/file_0.txt", + "gitInvocations": 1, + "gitMedianMs": 24.316, + "gitWorkMs": 11.93, + "gitFloorBound": false, + "ratioToGit": 0.7857502095557419 + }, + { + "op": "list_all_files", + "label": "Browse the whole tree", + "scale": "32 files", + "samples": 10, + "firstMs": 0.475, + "repeatMedianMs": 0.158, + "repeatP95Ms": 0.161, + "gitCommand": "git ls-files --cached --others --exclude-standard", + "gitInvocations": 1, + "gitMedianMs": 12.878, + "gitWorkMs": 0.4920000000000009, + "gitFloorBound": true, + "ratioToGit": null + } + ], + "soak": null + } + ] +} diff --git a/docs/dev/performance.md b/docs/dev/performance.md new file mode 100644 index 00000000..933311cb --- /dev/null +++ b/docs/dev/performance.md @@ -0,0 +1,336 @@ +# Performance — the large-repo benchmark + +"Slow on big repositories" is the most consistent structural complaint about +every established GUI client, and being fast is one of the two strongest claims +this project makes. This file is what turns that claim into a number somebody +else can check, and what makes a regression in it visible before a stranger +finds it (#257). + +Read the spec for the reasoning behind every choice below: +`docs/superpowers/specs/2026-09-17-large-repo-benchmark-spec.md`. + +## Running it + +```bash +pnpm bench # the three generated fixtures, ~1 minute to build +pnpm bench --linux # …plus a real clone of torvalds/linux (multi-GB) +pnpm bench --fixture deep # one fixture +pnpm bench --soak 60 # add a 60-minute soak per fixture +pnpm bench --no-publish # measure and print; write nothing +``` + +Three pieces, each runnable on its own: + +| piece | what it does | +| --- | --- | +| `scripts/bench-fixtures.mjs` | materialises the repositories under `$PGBENCH_HOME` (default `~/.cache/platypusgit-bench`) | +| `src-tauri/benches/repo_bench.rs` | measures one repository, writes a JSON document of raw samples | +| `scripts/bench-report.mjs` | renders those into `benchmark.json` beside this file, and the table block below | + +**Run it on a quiet machine.** Every number is wall clock, so a compile in +another window is recorded as this program being slow — and these are the +figures a stranger will check. + +The harness is behind `--features bench` and `required-features` in +`src-tauri/Cargo.toml`, so `cargo check` and `cargo test` — the whole Rust CI +gate — never build or link it. + +## The fixtures + +Breadth hurts differently from depth, so one "big repo" fixture would give a +number that cannot say which dimension moved when it regresses. Three generated +fixtures isolate one dimension each; the fourth is the real thing. + +| fixture | shape | isolates | +| --- | --- | --- | +| `deep` | 50,000 commits, 16 files | the log walk | +| `wide` | 50,000 files, all modified, 5,000 untracked | `status` | +| `refs` | 5,000 branches, 2,000 tags, 2,000 commits | ref enumeration | +| `linux` | a real clone of `torvalds/linux` | all of it, for real | + +The generated three are **deterministic** — fixed seed, fixed timestamps, fixed +author — so the same parameters produce the same object ids on every machine, +and they build in seconds from `git fast-import`. That is what makes "a script +anyone can run" true rather than aspirational: nobody re-runs a benchmark that +starts with a six-gigabyte download. `linux` is opt-in for exactly that reason, +and it is not generated because a synthetic repository cannot stand in for 1.4 +million real commits, 90,000 real paths and a real pack layout. + +Fixtures are build artifacts. They live under `$PGBENCH_HOME`, never in the +tree, and a fixture whose recorded shape no longer matches +`scripts/bench-fixtures.mjs` is regenerated rather than reused — comparing +today's numbers against a differently-shaped repository is worse than having no +previous numbers at all. + +## What is measured, and what is therefore not + +The harness drives `Libgit2Backend` through the real `GitBackend` trait. **No +number here includes React**, because there is no webview in it. That is a real +limitation, stated plainly, and it is still the right layer: it is where a big +repository is actually expensive, it is where a regression lands, and the render +on top is bounded by the *window* rather than by the repository — the log is +paged at 500, diff rows are windowed, long lists are virtualised — so it does +not grow with the fixture. + +Two measurements narrow the gap on purpose rather than pretending it is not +there: + +* **`open_screen`** issues the eleven reads `useRepoStore.refreshAll` issues, + simultaneously, from separate threads. A composite is the only measurement + that can catch "a slow status blocks everything else on that repo" — the trap + `git/repo_locks.rs` exists to avoid, and the one an op-at-a-time benchmark is + structurally blind to. +* **`open_screen_ipc`** is the same fan-out plus `serde_json` encoding of every + payload, because that encoding is real work on a 500-commit page and it + happens before the frontend sees a byte. + +### "First" and "repeat", not "cold" and "warm" + +**First** is one call on a freshly constructed backend and a freshly opened +repository — what happens when you open a repository in the app, with libgit2's +object database, ref database and pack indices all unbuilt. **Repeat** is the +median over many calls on that handle; the p95 beside it is nearest-rank. + +Neither purges the operating system's file cache. The words "cold" and "warm" +are avoided precisely because they would imply it did. A first-boot number is +larger than anything below, by an amount that depends on the disk rather than on +this code. + +### The `git` baseline + +Every operation a single `git` invocation can answer is measured as that +invocation too, and the table prints the ratio. + +The point is not to win. `git` is the floor, and a ratio near it is the good +outcome. The point is that **a ratio survives leaving this machine**: a +millisecond figure from somebody else's laptop tells a reader nothing they can +check, and a ratio that doubles is a regression even on hardware that got +faster. + +The baselines ask the *same* question, not the cheapest one sharing a name. +`status` is the clearest case: `GitBackend::status` returns per-file added and +removed counts, so its baseline is `git status --porcelain` **plus** both +`--numstat` diffs. Comparing it to a bare `git status` would be comparing it to +less work than it does. Every baseline command is recorded in +`docs/dev/benchmark.json` under `gitCommand`, so the comparison can be +disputed with evidence rather than in the abstract. + +## Where the results go + +| artifact | committed | what it is | +| --- | --- | --- | +| `$PGBENCH_HOME/results/.json` | no | raw, every sample — what makes a result checkable | +| `docs/dev/benchmark.json` | yes | the published record, summary statistics only | +| the table block below | yes | the same numbers, as a document | + +Both committed artifacts are **generated and not hand-editable**. +`test/benchmark.test.ts` re-renders the markdown from the JSON and fails when +they disagree. That guard is the point of the whole exercise: the way a measured +number turns back into an adjective is somebody nudging it in a hurry. + +The record sits beside this file rather than under `site/`, and both are covered +by the `docs/dev/` entry already in the `js` path filter in +`.github/workflows/tests.yml`. Without that coverage the guard would be +skippable by exactly the change it polices — the failure mode #210 already +shipped once. + +### The marketing site does not print these yet + +#257 asks for a measured figure on the site in place of an adjective, and the +block to do it is written. It is **deliberately not shipped yet**: the honest +headline today is that opening `torvalds/linux` takes 15.8 seconds, and the +right response to that is to fix it rather than to publish it as a selling +point. It ships once the log-walk work in the findings below lands — at which +point the record moves to `site/src/data/` beside `comparison.json`, which is +where this repository keeps published records the site reads. + +Until then nothing under `site/**` is touched by a re-measurement, which also +means `pnpm bench` cannot redeploy the website by accident. + +## What the numbers say + +The first run of this benchmark found three things. They are recorded here +because the tables above will move and the reasoning will not, and because a +number with no reading beside it is a number nobody acts on. + +### 1. We are fine until history gets very deep — and then we are not + +The whole first screen — the eleven reads `refreshAll` issues, all at once — +costs **255 ms** on a 50,000-commit repository and **219 ms** on one with 5,001 +branches and 2,000 tags. That is the size at which GitKraken's own users report +it falling over, and it is a good answer. + +On `torvalds/linux` the same screen costs **15.8 seconds**, and scrolling ten +pages into its history costs **two minutes and thirty-eight seconds**. That is +not a good answer, and publishing it is the point of the exercise: the user who +opens a 1.4-million-commit repository and waits is the user this project was +written for. + +### 2. The log walk is not slow — the topological SORT is, and it is re-paid per page + +The obvious reading of "our 500-commit page takes 252 ms and `git log -500` +takes 41 ms" is that the walk is six times too slow. It is wrong, and it nearly +reached this document. + +`log_page` walks with `Sort::TIME | Sort::TOPOLOGICAL`, because the commit +graph's lane assignment depends on topological order. A default `git log` does +not sort that way. Asked the same question, `git log --topo-order -500` costs +284 ms on the `deep` fixture against our 275 ms — **parity**. The baselines in +this benchmark are `--topo-order` for exactly that reason, and the near miss is +written up in the spec. + +What survives is sharper. git pays for that sort **once and then skips**; we pay +it per page: + +| | `deep` (50k commits) | `torvalds/linux` (1.5M commits) | +| --- | --- | --- | +| our first page | 252 ms | 15.95 s | +| our tenth page | 2.40 s | 157.67 s | +| `git log --topo-order --skip=4500 -500` | 193 ms | 9.68 s | + +The per-page cost is flat in depth — ten pages cost ten times one page — which +is the signature of restarting the walk rather than continuing it. The cursor +already carries the frontier, so the walk logically continues; it is the sort +that is rebuilt. + +### 3. The ref map is rebuilt on every page, too + +`log_page` calls `collect_ref_map(repo)` per call, enumerating and peeling every +ref so the page can decorate its 500 commits. Two fixtures isolate it, and the +variable between them is refs rather than history: + +| fixture | history | refs | ours | `git` work | ratio | +| --- | --- | --- | --- | --- | --- | +| `deep` | 50,000 commits | 1 | 252 ms | 194 ms | 1.3× | +| `refs` | 2,000 commits | 7,001 | 135 ms | 8.5 ms | **16×** | + +Twenty-five times *less* history, and still most of the cost. This one looks +much cheaper to fix than the sort: cache the map per repository and invalidate +it on ref writes. + +### What is already good, and worth not breaking + +* **Opening a repository is free** — 0.11 ms on the kernel, a fresh libgit2 + handle and nothing else. Every "first call" number is measured against one. +* **The concurrent fan-out really is concurrent.** On `wide`, `open_screen` + costs 5.42 s and `status` alone costs 5.42 s: eleven reads cost what the + slowest one costs, not the sum. That is `git/repo_locks.rs` doing its job, and + it is the single thing most worth not regressing. +* **IPC encoding is not a cost worth optimising.** `open_screen_ipc` is within + noise of `open_screen` on every fixture, including a 500-commit page. +* **`status` tracks git.** 5.42 s against 2.98 s of git's work on 55,000 + changed entries — 1.8×, on an operation where git itself takes three seconds. + It is slow because the question is expensive, not because of how we ask it. + +### Known characteristics that are not on the tables + +* **File history is unbounded on a cold path.** `file_history` stops at `limit` + matches, so on a file with fewer than 500 commits it walks to the root of + history with a tree comparison per commit. On the kernel that is 1.5 million + of them for one click. The benchmark measures the *most frequently changed* + path precisely so it terminates; see `hottest_path` in the harness. It also + takes the exclusive lock (`with_repo`, not `with_repo_read`), so it blocks + every other read on that repository while it runs. +* **No fixture carries a commit-graph file, and it would not help us if it + did.** A fresh clone has none — `git clone` does not write one, and + `gc --auto` does not fire on a single packfile — so this is what a user gets + on day one. It matters enormously to git: writing one for the kernel takes 14 + seconds, and `git log --topo-order -500` then drops from 9.51 s to **21 ms**. + It does not measurably help us. With the file present, our first page took + 63.4 s for four calls against 64.2 s without it: libgit2's revwalk does not + read it. That is the most useful single fact this benchmark produced, because + it rules out the cheap fix and says where the work actually has to go. + +## Results + + + +Measured on Apple M4 Pro (14 cores, 48 GB, macos/aarch64) with git version 2.50.1 (Apple Git-155), on 2026-09-17. Up to 10 repeats per operation, time-boxed to 20s each — so a cheap operation gets the full count and an expensive one gets at least three. The published record records how many each row actually took. + +The **`git` work** column is that baseline's wall clock with process start-up subtracted (12.4 ms per invocation on this machine, measured), because we pay none of it — the backend is libgit2, in process. That is deliberately the comparison that makes us look worse: against git's wall clock we would get a ten-millisecond head start on every row. **†** marks a baseline where start-up swamped the work, leaving a remainder too small to divide by; those rows print no ratio rather than a flattering one. + +### torvalds/linux + +A real clone of the Linux kernel: the repository people mean when they say a git client is slow. + +*1,482,923 commits · 96,034 tracked files · 946 tags · 13 changed entries* + +| Operation | Result size | First call | Repeat | p95 | `git` work | vs `git` | +| --- | --- | --- | --- | --- | --- | --- | +| Open the repository | a fresh handle | 0.26 ms | 0.11 ms | 0.13 ms | † | — | +| Everything the first screen needs, at once | 11 concurrent reads | 15.91 s | 15.84 s | 15.86 s | — | — | +| …including encoding it all for the webview | 11 concurrent reads | 15.80 s | 15.82 s | 15.86 s | — | — | +| Working-tree status | 26 entries | 1.15 s | 989 ms | 1.10 s | 517 ms | 1.9× | +| First page of history | 500 commits | 15.96 s | 15.95 s | 16.49 s | 9.51 s | 1.7× | +| Ten pages into history | 500 commits | 156.38 s | 157.67 s | 159.94 s | 9.68 s | 16× | +| List every branch | 3 branches | 1.65 ms | 0.87 ms | 0.89 ms | † | — | +| List every tag | 946 tags | 27.1 ms | 19.7 ms | 20.2 ms | 23.7 ms | 0.83× | +| Diff the selected commit | 3 files | 142 ms | 139 ms | 140 ms | 11.5 ms | 12× | +| Diff one modified file | 2 hunks | 68.2 ms | 2.14 ms | 2.22 ms | — | — | +| History of one file | 500 commits | 15.91 s | 15.42 s | 16.06 s | 5.65 s | 2.7× | +| Browse the whole tree | 96,034 files | 782 ms | 515 ms | 642 ms | 218 ms | 2.4× | + +### deep + +50,000 commits over a small tree — the size past which GitKraken's own users report native clients beating it. + +*50,000 commits · 16 tracked files* + +| Operation | Result size | First call | Repeat | p95 | `git` work | vs `git` | +| --- | --- | --- | --- | --- | --- | --- | +| Open the repository | a fresh handle | 0.14 ms | 0.11 ms | 0.12 ms | † | — | +| Everything the first screen needs, at once | 11 concurrent reads | 252 ms | 253 ms | 256 ms | — | — | +| …including encoding it all for the webview | 11 concurrent reads | 252 ms | 253 ms | 254 ms | — | — | +| Working-tree status | 0 entries | 0.69 ms | 0.53 ms | 0.55 ms | † | — | +| First page of history | 500 commits | 261 ms | 249 ms | 251 ms | 193 ms | 1.3× | +| Ten pages into history | 500 commits | 2.39 s | 2.39 s | 2.40 s | 192 ms | 12× | +| List every branch | 1 branch | 0.42 ms | 0.28 ms | 0.29 ms | † | — | +| List every tag | 0 tags | 0.17 ms | 0.15 ms | 0.15 ms | † | — | +| Diff the selected commit | 1 file | 0.48 ms | 0.34 ms | 0.35 ms | † | — | +| History of one file | 500 commits | 293 ms | 65.0 ms | 65.7 ms | 319 ms | 0.20× | +| Browse the whole tree | 16 files | 0.50 ms | 0.13 ms | 0.13 ms | † | — | + +**Soak.** 2,344 first-screen fan-outs over 10 minutes. Resident memory 67 MB → 69 MB (peak 69 MB). Median fan-out 252 ms in the first half, 252 ms in the second — -0.2%. + +### wide + +50,000 tracked files with every one of them modified, plus 5,000 untracked — a monorepo just after a codemod. + +*1 commits · 50,000 tracked files · 55,000 changed entries* + +| Operation | Result size | First call | Repeat | p95 | `git` work | vs `git` | +| --- | --- | --- | --- | --- | --- | --- | +| Open the repository | a fresh handle | 0.25 ms | 0.11 ms | 0.13 ms | † | — | +| Everything the first screen needs, at once | 11 concurrent reads | 5.42 s | 5.42 s | 5.62 s | — | — | +| …including encoding it all for the webview | 11 concurrent reads | 5.60 s | 5.43 s | 5.60 s | — | — | +| Working-tree status | 55,000 entries | 5.38 s | 5.42 s | 5.48 s | 2.98 s | 1.8× | +| First page of history | 1 commit | 0.35 ms | 0.25 ms | 0.26 ms | † | — | +| Ten pages into history | 1 commit | 0.31 ms | 0.25 ms | 0.26 ms | † | — | +| List every branch | 1 branch | 0.35 ms | 0.28 ms | 0.32 ms | † | — | +| List every tag | 0 tags | 0.17 ms | 0.14 ms | 0.15 ms | † | — | +| Diff the selected commit | 50,000 files | 1.53 s | 1.51 s | 1.67 s | 510 ms | 3.0× | +| Diff one modified file | 1 hunk | 17.2 ms | 0.63 ms | 0.64 ms | — | — | +| History of one file | 1 commit | 0.28 ms | 0.10 ms | 0.10 ms | † | — | +| Browse the whole tree | 55,000 files | 196 ms | 181 ms | 241 ms | 42.4 ms | 4.3× | + +### refs + +5,000 branches and 2,000 tags over a short history — a long-lived repository nobody prunes. + +*2,000 commits · 32 tracked files · 5,001 branches · 2,000 tags* + +| Operation | Result size | First call | Repeat | p95 | `git` work | vs `git` | +| --- | --- | --- | --- | --- | --- | --- | +| Open the repository | a fresh handle | 0.15 ms | 0.14 ms | 0.14 ms | † | — | +| Everything the first screen needs, at once | 11 concurrent reads | 221 ms | 219 ms | 221 ms | — | — | +| …including encoding it all for the webview | 11 concurrent reads | 221 ms | 220 ms | 224 ms | — | — | +| Working-tree status | 0 entries | 0.72 ms | 0.55 ms | 0.56 ms | † | — | +| First page of history | 500 commits | 787 ms | 135 ms | 138 ms | 8.48 ms | 16× | +| Ten pages into history | 500 commits | 526 ms | 525 ms | 551 ms | 7.25 ms | 72× | +| List every branch | 5,001 branches | 184 ms | 180 ms | 182 ms | 183 ms | 0.98× | +| List every tag | 2,000 tags | 148 ms | 148 ms | 151 ms | 43.3 ms | 3.4× | +| Diff the selected commit | 1 file | 0.40 ms | 0.28 ms | 0.29 ms | † | — | +| History of one file | 63 commits | 20.7 ms | 9.37 ms | 9.88 ms | 11.9 ms | 0.79× | +| Browse the whole tree | 32 files | 0.47 ms | 0.16 ms | 0.16 ms | † | — | + + diff --git a/docs/dev/testing.md b/docs/dev/testing.md index 87ea1000..6fcacf53 100644 --- a/docs/dev/testing.md +++ b/docs/dev/testing.md @@ -1,7 +1,16 @@ # Testing — four layers, Docker e2e, CI Part of the `docs/dev/` set (`architecture`, `testing`, `frontend`, `backend`, -`distribution`). `test/docs.test.ts` reads this set together with CLAUDE.md. +`distribution`, `performance`). `test/docs.test.ts` reads this set together with +CLAUDE.md. + +**Performance is measured, not tested** — the large-repo benchmark lives in +`docs/dev/performance.md` and is a fifth thing, deliberately outside all four +layers below and outside CI. Its numbers are wall clock, which on a shared +runner is a flake generator, and its fixtures take minutes to build. It is +something you run on a quiet machine when you have touched the log walk, +`status` or the refresh path; the committed results are what make a regression +in them visible. ## The four layers diff --git a/docs/superpowers/plans/2026-09-17-large-repo-benchmark.md b/docs/superpowers/plans/2026-09-17-large-repo-benchmark.md new file mode 100644 index 00000000..a9cb0509 --- /dev/null +++ b/docs/superpowers/plans/2026-09-17-large-repo-benchmark.md @@ -0,0 +1,150 @@ +# Large-repo benchmark — implementation plan (#257) + +Spec: `docs/superpowers/specs/2026-09-17-large-repo-benchmark-spec.md`. + +One PR. Nothing here changes app behaviour: every file is a new build artifact, +a new document, or a pointer to one. The two exceptions are the `js` path filter +in `tests.yml` (one path added) and a new section on the site's `/features` +page. + +## 1. Fixtures — `scripts/bench-fixtures.mjs` + +Generates `deep`, `wide` and `refs` from `git fast-import` streams; clones +`linux` on demand. Everything under `$PGBENCH_HOME`. + +* Deterministic: a seeded LCG for content, a fixed epoch for every timestamp, a + fixed author. Same parameters ⇒ same object ids on any machine. +* `--done` on the import, so a truncated stream is an error rather than a + quietly partial repository — a fixture that stopped at commit 31,000 would + publish numbers nobody can reproduce. +* Each fixture's shape is stamped to `.fixture.json` **beside** the + repository, never inside it: `wide` is measured partly by how many untracked + files `status` reports, and a stray file in the work tree would be one of them. +* Config left at git's defaults. `core.untrackedCache` and `core.fsmonitor` + both make `status` dramatically cheaper, and benchmarking with them on would + publish a number almost nobody's repository produces. `gc.auto=0` is the one + deviation, because a `gc --auto` firing mid-run is measured as whatever + operation happened to be in flight. + +**Done when** all three build in under a minute and the shapes verify: +50,000 commits / 50,000 files with 55,000 changed entries / 5,001 branches and +2,000 tags of which 500 annotated. + +## 2. Harness — `src-tauri/benches/repo_bench.rs` + +A `harness = false` bench target behind `required-features = ["bench"]`, so +`cargo check` and `cargo test` — the whole Rust CI gate — never build or link +it. + +Measures, per operation: one `first` call on a freshly constructed backend and +a freshly opened repository, then N repeats on one handle (median, min, p95, +max, and every sample kept). + +* `open_screen` runs the eleven `refreshAll` reads simultaneously behind a + `Barrier`. The barrier is load-bearing: spawning eleven threads without one + lets the first read finish before the last starts, which turns the one + measurement that exists to find lock contention into a measurement with none + in it. +* `open_screen_ipc` is the same fan-out plus `serde_json` encoding of every + payload. +* Every op a single `git` invocation can answer carries that invocation as a + baseline, chosen to ask the same question — `status`'s baseline is + `git status --porcelain` plus both `--numstat` diffs, because + `GitBackend::status` returns per-file line counts. +* `--soak N` repeats the fan-out for N minutes and reports RSS at start, end and + peak, plus the median fan-out in the first half of the run against the second. +* A `scale` string ("55,000 entries") travels with every timing, because a + benchmark that silently started measuring an empty result is the classic way + to publish an excellent number for nothing at all. +* Repeats are **time-boxed** rather than counted: `budget / first_call`, clamped + between three and `--iterations`. Ten repeats of an eight-second log page is + thirteen minutes for one table row; three is enough for a median when the + operation is that slow, and every row publishes its own sample count. +* The subject path for file history is the **most frequently changed path in the + last 2,000 commits**, not "a path HEAD touched". The obvious choice is + unusable: `file_history` stops at `limit` matches, so on a rarely-touched file + it never reaches 500 and walks the whole history with a tree comparison per + commit — on `torvalds/linux` the benchmark simply does not finish. Measured + the hard way, by watching it not finish. + +Output is hand-rolled JSON so key order is stable — a generated file that +reorders itself makes every diff of it unreadable, and this one is committed. + +**Done when** it compiles with `--features bench`, is absent from a plain +`cargo test`, and produces a plausible document for all four fixtures. + +## 3. Orchestration — `scripts/bench.sh`, `pnpm bench` + +Ensure fixtures → build the harness → run it per fixture → render. Flags: +`--linux`, `--fixture`, `--soak`, `--iterations`, `--force`, `--no-publish`. +Invokes the harness through `cargo bench` rather than a built path, because a +`harness = false` bench has no stable file name and globbing for it picks up +stale binaries. + +## 4. Rendering — `scripts/bench-report.mjs` + +Raw runs → `docs/dev/benchmark.json` → the markdown table block spliced between +the generated markers in `docs/dev/performance.md`. + +The direction matters: **the markdown is rendered from the published JSON**, not +from the raw runs. That is what gives the guard test leverage — it re-renders +from the committed JSON and compares to the committed markdown, so the two can +only agree if both came out of one run. Raw samples stay out of the repository. + +## 5. Documents + +* `docs/dev/performance.md` — method, fixtures, what is and is not included, the + baseline argument, the generated results, and what the numbers say. +* `CLAUDE.md` — one pointer in the doc list, one line in the command list. +* The spec, committed beside the other specs. + +## 6. Site — written, then held back + +A `Benchmark.astro` block on `/features` reading a typed `benchmark.ts`, printing +the machine beside the numbers, keeping the `git` column and printing the rows +where we lose. + +**Not shipped in this pass.** The first run's headline is 15.8 seconds to open +`torvalds/linux`, and the right response to that number is to fix it, not to put +it on a marketing page. It follows the log-walk work. Nothing benchmark-related +lives under `site/**` meanwhile, which also stops `pnpm bench` from triggering +the site deployment. + +## 7. Guards + +`test/benchmark.test.ts`: + +1. re-renders the markdown from `benchmark.json` and compares byte for byte; +2. requires a machine, a date and an iteration count; +3. requires `open`, `open_screen`, `status` and `log_first_page` on every + fixture, a non-empty `scale` and a positive median on every row; +4. rejects an operation key the renderer does not know how to order (it would + sort silently to the end of the table); +5. requires a recorded `git` command behind every printed ratio; +6. asserts `docs/dev/` is in the `js` path filter in `tests.yml` — a guard + skippable by the change it polices is the #210 failure mode, and it has + shipped here once. Both of its inputs live there, so no new filter entry is + needed. + +The hand-typed-figure check goes in with the site block, since it polices the +markup that block adds. + +## 8. Run it and publish + +A clean run of all four fixtures on a quiet machine, then commit +`benchmark.json` and `performance.md` together. + +Read the results before writing the prose. If a number is bad, the document says +so with the measurement attached and a follow-up issue gets filed — a benchmark's +job is to produce the number, and acting on a bad one is the next piece of work, +not this one. + +## Verification + +* `pnpm test` — the new guard plus the existing `docs`/`unit` projects. +* `pnpm tsc --noEmit`, and a site build for the new component. +* `cargo test` — unchanged, and specifically proving the bench target is not + built by it. +* `cargo build --features bench --bench repo_bench` — proving it still is when + asked. +* No e2e: nothing here reaches the app. diff --git a/docs/superpowers/specs/2026-09-17-large-repo-benchmark-spec.md b/docs/superpowers/specs/2026-09-17-large-repo-benchmark-spec.md new file mode 100644 index 00000000..87dd0572 --- /dev/null +++ b/docs/superpowers/specs/2026-09-17-large-repo-benchmark-spec.md @@ -0,0 +1,211 @@ +# Large-repo benchmark (#257) + +"Slow on big repositories" is the most consistent structural complaint about +every established GUI client, and being fast is one of this project's two +strongest claims. There are no numbers, so right now it is only a claim. + +This builds a benchmark anybody can run, publishes what it measures, and puts a +measured figure where an adjective used to be. + +## What "fast" has to mean before it can be measured + +The complaint is never about a microbenchmark. It is about four moments: + +1. **Opening a repository.** From double-click to a screen with history on it. +2. **Status on a dirty tree.** The auto-fetch and the file watcher both call + this, so on a big working tree it is not one cost but a recurring one. +3. **Scrolling history**, including past the first page. +4. **Jank after a while** — the Sourcetree complaint, the one nobody answers. + +Everything below is arranged around those four, and an operation that does not +serve one of them is not measured. A benchmark that reports thirty numbers is a +benchmark whose regressions nobody notices. + +## Three decisions that shape the rest + +### It measures the backend, and says so + +The harness drives `Libgit2Backend` through the real `GitBackend` trait. There +is no webview in it, so no number here includes React. + +That is a real limitation and the published document states it in those words. +It is nonetheless the right layer: + +* it is where a big repository is expensive — a 500-commit page and a 55,000- + entry status are git work, not render work; +* it is where a regression lands, and it is diffable; +* the render on top is bounded by the *window*, not by the repository (the log + is paged at 500, diff rows are windowed, long lists are virtualised), so it + does not grow with the fixture. + +Two measurements narrow the gap on purpose rather than pretending it is not +there. `open_screen` issues the **eleven reads `refreshAll` issues**, +simultaneously, from separate threads — which is the only shape that can catch +"a slow status blocks everything else on that repo", the trap +`git/repo_locks.rs` exists to avoid and the one an op-at-a-time benchmark is +structurally blind to. `open_screen_ipc` adds `serde_json` encoding of every +payload, because that encoding is real work on a 500-commit page and it happens +before the frontend sees a byte. + +An end-to-end number through the webview is a separate piece of work. It needs +the e2e harness, a Linux container and a multi-gigabyte fixture inside it, and it +would still be a worse regression detector than this because a wall-clock number +that includes WebDriver is dominated by WebDriver. + +### Four fixtures, three of them generated + +Breadth hurts differently from depth, so one "big repo" fixture would produce a +number that cannot say which dimension moved when it regresses. Each generated +fixture isolates exactly one: + +| fixture | shape | isolates | +| --- | --- | --- | +| `deep` | 50,000 commits, 16 files | the log walk | +| `wide` | 50,000 files, all modified, 5,000 untracked | `status` | +| `refs` | 5,000 branches, 2,000 tags, 2,000 commits | ref enumeration | +| `linux` | a real clone of `torvalds/linux` | all of it, for real | + +`deep` is 50,000 because that is the threshold GitKraken's own users name. +`wide` is a monorepo just after a codemod. `refs` is a long-lived repository +nobody prunes, which is most of them. + +The three generated ones are **deterministic**: fixed seed, fixed timestamps, +fixed author, so the same parameters produce the same object ids on every +machine. They build in seconds from `git fast-import`, which is what makes +"a script anyone can run" true rather than aspirational — nobody re-runs a +benchmark that starts with a six-gigabyte download. + +`linux` is not generated, because a synthetic repository cannot stand in for 1.4 +million real commits, 90,000 real paths and a real pack layout, and that is the +repository people mean. It is opt-in (`--linux`). + +Fixtures live under `$PGBENCH_HOME` (default `~/.cache/platypusgit-bench`), +never in the tree. + +### "First" and "repeat", not "cold" and "warm" + +`first` is one call on a freshly constructed backend and a freshly opened +repository — what happens when you open a repository in the app, with libgit2's +object database, ref database and pack indices all unbuilt. `repeat` is the +median of many calls on that handle. + +Neither purges the operating system's file cache. The words "cold" and "warm" +are avoided precisely because they would imply it did; a first-boot number is +larger than anything published here by an amount that depends on the disk rather +than on this code. Saying so is cheaper than being caught not having said so. + +## The `git` baseline + +Every operation a single `git` invocation can answer is measured as that +invocation too, and the table prints the ratio. + +The point is not to win. `git` is the floor, and a ratio near it is the good +outcome. The point is that **a ratio survives leaving this machine**: a +millisecond figure from somebody else's laptop tells a reader nothing they can +check, and a ratio that doubles is a regression even on hardware that got +faster. + +The baselines ask the *same* question, not the cheapest one sharing a name. +`status` is the clearest case: `GitBackend::status` returns per-file added and +removed counts, so its baseline is `git status --porcelain` **plus** both +`--numstat` diffs. Comparing it to a bare `git status` would be comparing it to +less work than it does, and flattering ourselves in public is the one thing a +benchmark must not do. + +**The rule cuts both ways, and it caught a false result here before publication.** +The first draft of the log baseline was a plain `git log --max-count=500`, which +made the first page of history look fourteen times slower than git on the `deep` +fixture. It is not. `log_page` walks with `Sort::TIME | Sort::TOPOLOGICAL` +because the commit graph's lane assignment depends on topological order, and a +default `git log` does not sort that way. Measured on `deep`: + +| | | +| --- | --- | +| `git log --max-count=500` | 41 ms | +| `git log --topo-order --max-count=500` | 284 ms | +| `log_page(None, None, 500)` | 275 ms | + +The first page is at **parity**. The wrong baseline would have published a +regression that does not exist, in the very document written to stop people +publishing numbers they had not checked — so the baselines are `--topo-order`, +and the reason is written beside them in the source. + +What survives the correction is sharper and still worth acting on: git pays for +that sort **once** and then skips, while `log_page` pays it per page. Page ten +is 2.61 s against `git log --topo-order --skip=4500 --max-count=500` at 219 ms. + +## Repeats are time-boxed, not counted + +A fixed repeat count is wrong at both ends. Ten repeats of a 0.15 ms operation +buys a rounding error's worth of extra confidence; ten repeats of an +eight-second log page on a kernel clone is thirteen minutes for one row of one +table, and a benchmark nobody has time to finish produces no numbers at all. + +So the count is derived: `budget / first_call`, clamped between three and the +requested maximum. Three is the floor because a median of two is the mean of +two. Every row records how many samples it actually took, so the published +table never implies a confidence it does not have — and the budget is soft by +construction, because the floor wins when one call already exceeds it. + +## The soak + +"Jank after a while" gets its own mode rather than being left out: `--soak N` +repeats the whole first-screen fan-out for N minutes and reports resident memory +at the start, at the end and at its peak, plus the median fan-out time in the +first half of the run against the second. + +Two halves rather than a fitted slope, because a slope invites reading a trend +into noise. It covers the backend only — a leak in React would not show here — +and the document says that. + +## Where the output goes + +Three artifacts, of which two are committed: + +* `$PGBENCH_HOME/results/.json` — raw, every sample, **not** committed. + It is what makes a result checkable, and it is also forty floating-point + numbers per run that nobody reads the diff of. +* `docs/dev/performance.md` — the developer-facing record: method, fixtures, + what the numbers mean, and a generated table block per fixture. +* `docs/dev/benchmark.json` — the published record, beside the document. + +Both committed artifacts are **generated and not hand-editable**, and +`test/benchmark.test.ts` re-renders the markdown from the JSON and fails when +they disagree. That guard is the point of the exercise: the way a measured +number turns back into an adjective is somebody nudging it in a hurry. + +Both live under `docs/dev/`, which is already in the `js` path filter in +`.github/workflows/tests.yml`. That is why the record lives there rather than +somewhere needing a filter entry of its own: a guard skippable by exactly the +change it polices is the failure mode #210 already shipped once. + +## What the site says — deferred, on purpose + +The plan was a "measured, not claimed" block on `/features`, printing real +figures from the JSON with the machine named beside them, under two rules taken +from the comparison table's house style: every figure comes from the JSON, and +it prints what was measured *including* where we are slow. + +**It is not shipping with the first run.** The measurement changed the decision: +the honest headline today is that opening `torvalds/linux` takes 15.8 seconds, +and a marketing page is not the right place to learn that. The block is written +and the rules above still hold; it ships once the log-walk finding is fixed, and +the record moves under `site/` at the same time. + +Holding it has a second benefit worth keeping either way: with no benchmark data +under `site/**`, a re-measurement cannot trigger the site deployment (`site.yml` +fires on any push to `main` touching that path), so `pnpm bench` can never +redeploy the website as a side effect. + +## Non-goals + +* **Comparing against other clients.** We cannot run GitKraken in a harness and + publish the result, and citing a forum post as a number would be the thing + this issue exists to stop doing. +* **A CI gate.** The numbers are wall clock on a shared runner, which is a + flake generator, and the fixtures take minutes to build. This is a thing you + run deliberately, and the published document is the record that makes a + regression visible. +* **Fixing what it finds.** A benchmark's job is to produce the number. Acting + on a bad one is the next piece of work, and it gets its own issue with the + measurement attached. From 481706572093b35eb4511bbb6422992f88921b77 Mon Sep 17 00:00:00 2001 From: Jonas Aasberg Date: Thu, 17 Sep 2026 13:45:29 +0200 Subject: [PATCH 3/3] test: hold the published performance numbers to one measured run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The way an adjective grows back is not malice. It is somebody with a release to cut, a figure that reads badly, and a two-character edit to a JSON file nobody diffs. So the document's table is RENDERED from the published record by the same module that writes it, and this re-renders it and compares byte for byte. The two can only agree if both came out of one pnpm bench. It also rejects a row whose median came from fewer than three samples, a ratio printed against a baseline that only measured process start-up, and an operation key the renderer does not know how to order — which would otherwise sort silently to the end of the table. It cannot check that the numbers are TRUE; nothing short of re-running the benchmark could. It checks that they are consistent and that they cover what the document claims. Both of its inputs live under docs/dev/, which is already in the js path filter in tests.yml, and the last assertion here is that it still is. A guard skippable by exactly the change it polices is the #210 failure mode, and it has shipped here once. Co-Authored-By: Claude Opus 5 (1M context) --- test/benchmark.test.ts | 249 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 249 insertions(+) create mode 100644 test/benchmark.test.ts diff --git a/test/benchmark.test.ts b/test/benchmark.test.ts new file mode 100644 index 00000000..a1b760dc --- /dev/null +++ b/test/benchmark.test.ts @@ -0,0 +1,249 @@ +/** + * @vitest-environment node + */ +// The published performance numbers are measured numbers (#257). +// +// The issue this whole benchmark answers is that "fast" was an adjective. The +// way an adjective grows back is not malice — it is somebody with a release to +// cut, a figure that reads badly, and a two-character edit to a JSON file that +// nobody diffs. So the two committed artifacts are generated from one run and +// this file is what makes that checkable: +// +// * `docs/dev/benchmark.json` is the published record, written by +// `scripts/bench-report.mjs`; +// * the table block in `docs/dev/performance.md` is RENDERED from that JSON by +// the same module. +// +// Re-rendering here and comparing byte for byte means the two can only agree if +// both came out of one `pnpm bench`. Edit either by hand and this fails, naming +// the command that fixes it. +// +// It cannot check that the numbers are TRUE — nothing short of re-running the +// benchmark could, and that takes minutes and a quiet machine. What it checks is +// that they are *consistent* and that they *cover* what the document claims to +// cover. That is the realistic accident. +// +// Every input it reads (`test/`, `scripts/`, `docs/dev/`) is already in the `js` +// filter in `tests.yml`, and the last assertion in this file is that `docs/dev/` +// really is — because a guard skippable by exactly the change it polices is the +// #210 failure mode, and it has shipped here once. + +import { readFileSync } from "node:fs"; +import { resolve } from "node:path"; +import { describe, expect, it } from "vitest"; + +import { + BEGIN, + END, + OP_ORDER, + renderMarkdown, + // @ts-expect-error — plain .mjs with JSDoc types, no .d.ts +} from "../scripts/bench-report.mjs"; + +const root = (rel: string) => resolve(process.cwd(), rel); +const read = (rel: string) => readFileSync(root(rel), "utf8"); + +const DATA_PATH = "docs/dev/benchmark.json"; +const DOC_PATH = "docs/dev/performance.md"; + +type Operation = { + op: string; + label: string; + scale: string; + samples: number; + firstMs: number | null; + repeatMedianMs: number | null; + repeatP95Ms: number | null; + gitCommand: string | null; + gitInvocations: number; + gitMedianMs: number | null; + gitWorkMs: number | null; + gitFloorBound: boolean; + ratioToGit: number | null; +}; + +type Fixture = { + key: string; + title: string; + kind: string; + blurb: string; + repository: { + fixture: string; + commits: number; + trackedFiles: number; + branches: number; + tags: number; + dirtyEntries: number; + }; + operations: Operation[]; +}; + +type Published = { + measuredOn: string; + machine: { + cpu: string; + cores: number; + memoryGb: number; + os: string; + gitVersion: string; + }; + iterations: number; + budgetSeconds: number; + gitSpawnFloorMs: number; + fixtures: Fixture[]; +}; + +const data: Published = JSON.parse(read(DATA_PATH)); +const doc = read(DOC_PATH); + +/** The operations every fixture must carry. Not the whole of `OP_ORDER`: two + * entries are conditional by design — `diff_workdir_file` needs a dirty tree + * and `file_history` needs HEAD to have touched a file — and a fixture that + * legitimately lacks one must not be made to fake it. These four are the ones + * the published claims rest on. */ +const REQUIRED_OPS = ["open", "open_screen", "status", "log_first_page"]; + +describe("the published benchmark numbers", () => { + it("renders exactly the table block that is committed in the doc", () => { + const a = doc.indexOf(BEGIN); + const b = doc.indexOf(END); + expect(a, `${DOC_PATH} is missing its BEGIN marker`).toBeGreaterThan(-1); + expect(b, `${DOC_PATH} is missing its END marker`).toBeGreaterThan(a); + + const committed = doc.slice(a + BEGIN.length, b).trim(); + expect( + committed, + `${DOC_PATH} and ${DATA_PATH} disagree. Neither is edited by hand — ` + + "re-run `pnpm bench` (or `pnpm bench --linux`) and commit both.", + ).toBe(renderMarkdown(data).trim()); + }); + + it("names the machine and the day it was measured", () => { + // A performance number with no machine beside it is not a measurement, it + // is a boast, and it is the first thing a reader checks. + expect(data.measuredOn).toMatch(/^\d{4}-\d{2}-\d{2}$/); + expect(data.machine.cpu.length).toBeGreaterThan(3); + expect(data.machine.gitVersion).toMatch(/^git version /); + expect(data.machine.cores).toBeGreaterThan(0); + expect(data.iterations).toBeGreaterThan(1); + expect(data.budgetSeconds).toBeGreaterThan(0); + }); + + it("measured something on every fixture", () => { + expect(data.fixtures.length).toBeGreaterThan(0); + for (const fixture of data.fixtures) { + const ops = new Set(fixture.operations.map((o) => o.op)); + const missing = REQUIRED_OPS.filter((op) => !ops.has(op)); + expect( + missing, + `${fixture.key} published no ${missing.join(", ")}. A run that measured ` + + "nothing publishes excellent numbers for nothing at all.", + ).toEqual([]); + + // The scale column is what lets a reader sanity-check a timing. A blank + // one, or a timing with no positive duration, means the harness measured + // an empty result and reported it as fast. + for (const op of fixture.operations) { + expect(op.scale, `${fixture.key}/${op.op} has no result size`).not.toBe(""); + expect( + op.repeatMedianMs, + `${fixture.key}/${op.op} has no repeat median`, + ).not.toBeNull(); + expect(op.repeatMedianMs!).toBeGreaterThan(0); + + // The time box can cut an expensive operation down to three repeats, + // and three is the floor on purpose: a median of two is the mean of + // two. A row below it means the box was misconfigured, not that the + // operation was fast. + expect( + op.samples, + `${fixture.key}/${op.op} published a median over ${op.samples} samples`, + ).toBeGreaterThanOrEqual(3); + } + } + }); + + it("publishes only operations the report knows how to order", () => { + // An op the renderer does not know about sorts to the end silently, which + // is how a new measurement gets added and then quietly read as the least + // important thing in the table. + const unknown = [ + ...new Set(data.fixtures.flatMap((f) => f.operations.map((o) => o.op))), + ].filter((op) => !OP_ORDER.includes(op)); + expect( + unknown, + `Not in OP_ORDER in scripts/bench-report.mjs: ${unknown.join(", ")}`, + ).toEqual([]); + }); + + it("prints no ratio when process start-up swamped the baseline", () => { + // The floor is real and large — over ten milliseconds on an M-series Mac, + // which is more than most of these operations take in total. A ratio + // against a baseline that is mostly `fork` would put a hundred-fold win in + // the table for an operation on an empty result: an adjective wearing a + // number. + expect(data.gitSpawnFloorMs).toBeGreaterThan(0); + for (const fixture of data.fixtures) { + for (const op of fixture.operations) { + if (!op.gitFloorBound) continue; + expect( + op.ratioToGit, + `${fixture.key}/${op.op} prints a ratio against a baseline at the ` + + "`git` start-up floor", + ).toBeNull(); + } + } + }); + + it("divides by git's work, never by its wall clock", () => { + // The ratio must be against the baseline MINUS process start-up, which is + // the harsher comparison — we pay no start-up, so dividing by git's wall + // clock would hand us a free head start on every row. + for (const fixture of data.fixtures) { + for (const op of fixture.operations) { + if (op.ratioToGit == null) continue; + expect(op.gitWorkMs, `${fixture.key}/${op.op}`).not.toBeNull(); + expect(op.gitWorkMs!).toBeLessThanOrEqual(op.gitMedianMs!); + expect( + op.ratioToGit, + `${fixture.key}/${op.op} did not divide by gitWorkMs`, + ).toBeCloseTo(op.repeatMedianMs! / op.gitWorkMs!, 5); + } + } + }); + + it("records the git command behind every ratio it prints", () => { + // A "3× git" with no command beside it is an argument nobody can check. + for (const fixture of data.fixtures) { + for (const op of fixture.operations) { + if (op.ratioToGit == null) continue; + expect( + op.gitCommand, + `${fixture.key}/${op.op} prints a ratio with no baseline command`, + ).toMatch(/^git /); + } + } + }); + + it("is reachable from the docs the assistant reads", () => { + // `docs.test.ts` pins the module and command lists; this pins the one + // pointer that makes the benchmark findable at all. + expect(read("CLAUDE.md")).toContain("docs/dev/performance.md"); + }); +}); + +describe("CI runs this guard when its inputs change", () => { + it("has docs/dev/ in the js path filter", () => { + // The #210 failure mode: a guard lands, its input is not in the filter, and + // the next PR touching only that input runs no suite at all and reports + // green. Both of this file's inputs — the document and the record beside it + // — are under `docs/dev/`, which is why the record lives there and not + // somewhere that would need a filter entry of its own. + const workflow = read(".github/workflows/tests.yml"); + expect( + workflow, + "`docs/dev/` left the `js` filter in tests.yml, so a benchmark-only " + + "commit now skips the guard that holds its numbers together.", + ).toContain("docs/dev/"); + }); +});