diff --git a/CLAUDE.md b/CLAUDE.md index 0d9ba6da..cdef78e3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -27,6 +27,10 @@ to what you are reading. A new section here needs a reason a pointer cannot serv and credentials, signing, stash, spawning processes, bisect, async/threading. - `docs/dev/distribution.md` — `pgit` CLI packaging per channel, the launch detach, Tauri permissions. +- `docs/dev/performance.md` — the large-repo benchmark: the fixtures, what each + number does and does not include, the baseline it is compared against, and + the published results. Read it before touching the log walk, `status`, or + the refresh path, and re-run `pnpm bench` when you do. - `docs/dev/releasing.md` — what a version number means and when to bump which part, the cut-a-release runbook (changelog lands on `main` FIRST), and the prerelease-promotion traps. Read it before tagging anything. @@ -58,6 +62,7 @@ pnpm test # vitest (unit logic + component tes pnpm test:e2e:docker # e2e — THE way to run e2e (headless, same stack as CI) pnpm test:e2e:docker run --spec e2e/specs/X.e2e.ts # ...one spec against this worktree's snapshot pnpm exec tsc -p e2e/tsconfig.json --noEmit # e2e typecheck gate (root tsc excludes e2e/) +pnpm bench # large-repo benchmark (docs/dev/performance.md) ``` **Local production builds need the updater signing key.** `tauri.conf.json` diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 611c752e..9c11a795 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -171,6 +171,21 @@ Add them alongside the code you change: - New component → `*.test.tsx`. - New user-facing flow, or a bug that reached the UI → an e2e spec. +### Measuring performance + +Performance is measured rather than tested, and it is not a CI gate — wall clock +on a shared runner is a flake generator. + +```bash +pnpm bench # three generated fixtures, built in seconds +pnpm bench --linux # …plus a real clone of torvalds/linux (multi-GB) +``` + +Run it on a quiet machine if you have touched the log walk, `status` or the +refresh path, and commit what it writes — `docs/dev/performance.md` explains +what each number does and does not include, and the figures on the website come +from the same run. + ## Production bundles ```bash diff --git a/docs/dev/benchmark.json b/docs/dev/benchmark.json new file mode 100644 index 00000000..6aa03780 --- /dev/null +++ b/docs/dev/benchmark.json @@ -0,0 +1,785 @@ +{ + "measuredOn": "2026-09-17", + "machine": { + "cpu": "Apple M4 Pro", + "cores": 14, + "memoryGb": 48, + "os": "macos/aarch64", + "gitVersion": "git version 2.50.1 (Apple Git-155)" + }, + "iterations": 10, + "budgetSeconds": 20, + "gitSpawnFloorMs": 12.411, + "fixtures": [ + { + "key": "linux", + "title": "torvalds/linux", + "kind": "real", + "blurb": "A real clone of the Linux kernel: the repository people mean when they say a git client is slow.", + "repository": { + "fixture": "linux", + "commits": 1482923, + "trackedFiles": 96034, + "branches": 1, + "tags": 946, + "dirtyEntries": 13 + }, + "gitSpawnFloorMs": 12.411, + "operations": [ + { + "op": "open", + "label": "Open the repository", + "scale": "a fresh handle", + "samples": 10, + "firstMs": 0.257, + "repeatMedianMs": 0.109, + "repeatP95Ms": 0.125, + "gitCommand": "git rev-parse HEAD", + "gitInvocations": 1, + "gitMedianMs": 12.24, + "gitWorkMs": 0, + "gitFloorBound": true, + "ratioToGit": null + }, + { + "op": "open_screen", + "label": "Everything the first screen needs, at once", + "scale": "11 concurrent reads", + "samples": 3, + "firstMs": 15905.468, + "repeatMedianMs": 15844.188, + "repeatP95Ms": 15855.63, + "gitCommand": null, + "gitInvocations": 0, + "gitMedianMs": null, + "gitWorkMs": null, + "gitFloorBound": false, + "ratioToGit": null + }, + { + "op": "open_screen_ipc", + "label": "…including encoding it all for the webview", + "scale": "11 concurrent reads", + "samples": 3, + "firstMs": 15799.373, + "repeatMedianMs": 15818.08, + "repeatP95Ms": 15856.463, + "gitCommand": null, + "gitInvocations": 0, + "gitMedianMs": null, + "gitWorkMs": null, + "gitFloorBound": false, + "ratioToGit": null + }, + { + "op": "status", + "label": "Working-tree status", + "scale": "26 entries", + "samples": 10, + "firstMs": 1150.178, + "repeatMedianMs": 989.035, + "repeatP95Ms": 1104.831, + "gitCommand": "git status --porcelain=v1 --untracked-files=all && git diff --numstat && git diff --cached --numstat", + "gitInvocations": 3, + "gitMedianMs": 554.112, + "gitWorkMs": 516.879, + "gitFloorBound": false, + "ratioToGit": 1.913474913857982 + }, + { + "op": "log_first_page", + "label": "First page of history", + "scale": "500 commits", + "samples": 3, + "firstMs": 15956.168, + "repeatMedianMs": 15952.356, + "repeatP95Ms": 16486.088, + "gitCommand": "git log --topo-order --max-count=500 --format=%H%n%an%n%ae%n%at%n%s", + "gitInvocations": 1, + "gitMedianMs": 9517.414, + "gitWorkMs": 9505.003, + "gitFloorBound": false, + "ratioToGit": 1.6783115165771119 + }, + { + "op": "log_page_deep", + "label": "Ten pages into history", + "scale": "500 commits", + "samples": 3, + "firstMs": 156377.157, + "repeatMedianMs": 157671.22, + "repeatP95Ms": 159939.268, + "gitCommand": "git log --topo-order --skip=4500 --max-count=500 --format=%H%n%an%n%ae%n%at%n%s", + "gitInvocations": 1, + "gitMedianMs": 9692.322, + "gitWorkMs": 9679.911, + "gitFloorBound": false, + "ratioToGit": 16.288498933512923 + }, + { + "op": "branches", + "label": "List every branch", + "scale": "3 branches", + "samples": 10, + "firstMs": 1.649, + "repeatMedianMs": 0.873, + "repeatP95Ms": 0.891, + "gitCommand": "git for-each-ref --format=%(refname)%(objectname)%(upstream) refs/heads refs/remotes", + "gitInvocations": 1, + "gitMedianMs": 12.951, + "gitWorkMs": 0.5400000000000009, + "gitFloorBound": true, + "ratioToGit": null + }, + { + "op": "tags", + "label": "List every tag", + "scale": "946 tags", + "samples": 10, + "firstMs": 27.099, + "repeatMedianMs": 19.709, + "repeatP95Ms": 20.166, + "gitCommand": "git for-each-ref --format=%(refname)%(objectname)%(*objectname) refs/tags", + "gitInvocations": 1, + "gitMedianMs": 36.108, + "gitWorkMs": 23.696999999999996, + "gitFloorBound": false, + "ratioToGit": 0.8317086551040217 + }, + { + "op": "diff_commit", + "label": "Diff the selected commit", + "scale": "3 files", + "samples": 10, + "firstMs": 141.665, + "repeatMedianMs": 138.7, + "repeatP95Ms": 140.429, + "gitCommand": "git show --format= --patch HEAD", + "gitInvocations": 1, + "gitMedianMs": 23.911, + "gitWorkMs": 11.500000000000002, + "gitFloorBound": false, + "ratioToGit": 12.060869565217388 + }, + { + "op": "diff_workdir_file", + "label": "Diff one modified file", + "scale": "2 hunks", + "samples": 10, + "firstMs": 68.161, + "repeatMedianMs": 2.136, + "repeatP95Ms": 2.219, + "gitCommand": null, + "gitInvocations": 0, + "gitMedianMs": null, + "gitWorkMs": null, + "gitFloorBound": false, + "ratioToGit": null + }, + { + "op": "file_history", + "label": "History of one file", + "scale": "500 commits", + "samples": 3, + "firstMs": 15913.163, + "repeatMedianMs": 15419.268, + "repeatP95Ms": 16055.124, + "gitCommand": "git log --topo-order --max-count=500 --format=%H%n%an%n%at%n%s -- MAINTAINERS", + "gitInvocations": 1, + "gitMedianMs": 5667.106, + "gitWorkMs": 5654.695, + "gitFloorBound": false, + "ratioToGit": 2.726808077181882 + }, + { + "op": "list_all_files", + "label": "Browse the whole tree", + "scale": "96,034 files", + "samples": 10, + "firstMs": 781.816, + "repeatMedianMs": 515.422, + "repeatP95Ms": 642.301, + "gitCommand": "git ls-files --cached --others --exclude-standard", + "gitInvocations": 1, + "gitMedianMs": 230.862, + "gitWorkMs": 218.451, + "gitFloorBound": false, + "ratioToGit": 2.359439874388307 + } + ], + "soak": null + }, + { + "key": "deep", + "title": "deep", + "kind": "generated", + "blurb": "50,000 commits over a small tree — the size past which GitKraken's own users report native clients beating it.", + "repository": { + "fixture": "deep", + "commits": 50000, + "trackedFiles": 16, + "branches": 1, + "tags": 0, + "dirtyEntries": 0 + }, + "gitSpawnFloorMs": 12.275, + "operations": [ + { + "op": "open", + "label": "Open the repository", + "scale": "a fresh handle", + "samples": 10, + "firstMs": 0.141, + "repeatMedianMs": 0.114, + "repeatP95Ms": 0.122, + "gitCommand": "git rev-parse HEAD", + "gitInvocations": 1, + "gitMedianMs": 12.244, + "gitWorkMs": 0, + "gitFloorBound": true, + "ratioToGit": null + }, + { + "op": "open_screen", + "label": "Everything the first screen needs, at once", + "scale": "11 concurrent reads", + "samples": 10, + "firstMs": 252.398, + "repeatMedianMs": 252.638, + "repeatP95Ms": 255.802, + "gitCommand": null, + "gitInvocations": 0, + "gitMedianMs": null, + "gitWorkMs": null, + "gitFloorBound": false, + "ratioToGit": null + }, + { + "op": "open_screen_ipc", + "label": "…including encoding it all for the webview", + "scale": "11 concurrent reads", + "samples": 10, + "firstMs": 251.635, + "repeatMedianMs": 252.584, + "repeatP95Ms": 254.367, + "gitCommand": null, + "gitInvocations": 0, + "gitMedianMs": null, + "gitWorkMs": null, + "gitFloorBound": false, + "ratioToGit": null + }, + { + "op": "status", + "label": "Working-tree status", + "scale": "0 entries", + "samples": 10, + "firstMs": 0.688, + "repeatMedianMs": 0.525, + "repeatP95Ms": 0.554, + "gitCommand": "git status --porcelain=v1 --untracked-files=all && git diff --numstat && git diff --cached --numstat", + "gitInvocations": 3, + "gitMedianMs": 39.411, + "gitWorkMs": 2.5859999999999985, + "gitFloorBound": true, + "ratioToGit": null + }, + { + "op": "log_first_page", + "label": "First page of history", + "scale": "500 commits", + "samples": 10, + "firstMs": 261.376, + "repeatMedianMs": 249.132, + "repeatP95Ms": 251.38, + "gitCommand": "git log --topo-order --max-count=500 --format=%H%n%an%n%ae%n%at%n%s", + "gitInvocations": 1, + "gitMedianMs": 205.036, + "gitWorkMs": 192.761, + "gitFloorBound": false, + "ratioToGit": 1.292439860760216 + }, + { + "op": "log_page_deep", + "label": "Ten pages into history", + "scale": "500 commits", + "samples": 8, + "firstMs": 2392.178, + "repeatMedianMs": 2393.558, + "repeatP95Ms": 2400.148, + "gitCommand": "git log --topo-order --skip=4500 --max-count=500 --format=%H%n%an%n%ae%n%at%n%s", + "gitInvocations": 1, + "gitMedianMs": 204.502, + "gitWorkMs": 192.227, + "gitFloorBound": false, + "ratioToGit": 12.4517263443741 + }, + { + "op": "branches", + "label": "List every branch", + "scale": "1 branch", + "samples": 10, + "firstMs": 0.422, + "repeatMedianMs": 0.281, + "repeatP95Ms": 0.29, + "gitCommand": "git for-each-ref --format=%(refname)%(objectname)%(upstream) refs/heads refs/remotes", + "gitInvocations": 1, + "gitMedianMs": 13.103, + "gitWorkMs": 0.8279999999999994, + "gitFloorBound": true, + "ratioToGit": null + }, + { + "op": "tags", + "label": "List every tag", + "scale": "0 tags", + "samples": 10, + "firstMs": 0.175, + "repeatMedianMs": 0.147, + "repeatP95Ms": 0.15, + "gitCommand": "git for-each-ref --format=%(refname)%(objectname)%(*objectname) refs/tags", + "gitInvocations": 1, + "gitMedianMs": 12.833, + "gitWorkMs": 0.5579999999999998, + "gitFloorBound": true, + "ratioToGit": null + }, + { + "op": "diff_commit", + "label": "Diff the selected commit", + "scale": "1 file", + "samples": 10, + "firstMs": 0.477, + "repeatMedianMs": 0.338, + "repeatP95Ms": 0.351, + "gitCommand": "git show --format= --patch HEAD", + "gitInvocations": 1, + "gitMedianMs": 13.195, + "gitWorkMs": 0.9199999999999999, + "gitFloorBound": true, + "ratioToGit": null + }, + { + "op": "file_history", + "label": "History of one file", + "scale": "500 commits", + "samples": 10, + "firstMs": 293.063, + "repeatMedianMs": 65.002, + "repeatP95Ms": 65.738, + "gitCommand": "git log --topo-order --max-count=500 --format=%H%n%an%n%at%n%s -- src/module_00.rs", + "gitInvocations": 1, + "gitMedianMs": 331.471, + "gitWorkMs": 319.196, + "gitFloorBound": false, + "ratioToGit": 0.20364290279326805 + }, + { + "op": "list_all_files", + "label": "Browse the whole tree", + "scale": "16 files", + "samples": 10, + "firstMs": 0.497, + "repeatMedianMs": 0.129, + "repeatP95Ms": 0.133, + "gitCommand": "git ls-files --cached --others --exclude-standard", + "gitInvocations": 1, + "gitMedianMs": 12.845, + "gitWorkMs": 0.5700000000000003, + "gitFloorBound": true, + "ratioToGit": null + } + ], + "soak": { + "minutes": 10, + "iterations": 2344, + "rssStartMb": 66.5, + "rssEndMb": 69.047, + "rssPeakMb": 69.047, + "firstHalfMedianMs": 252.302, + "secondHalfMedianMs": 251.693 + } + }, + { + "key": "wide", + "title": "wide", + "kind": "generated", + "blurb": "50,000 tracked files with every one of them modified, plus 5,000 untracked — a monorepo just after a codemod.", + "repository": { + "fixture": "wide", + "commits": 1, + "trackedFiles": 50000, + "branches": 1, + "tags": 0, + "dirtyEntries": 55000 + }, + "gitSpawnFloorMs": 12.058, + "operations": [ + { + "op": "open", + "label": "Open the repository", + "scale": "a fresh handle", + "samples": 10, + "firstMs": 0.25, + "repeatMedianMs": 0.114, + "repeatP95Ms": 0.134, + "gitCommand": "git rev-parse HEAD", + "gitInvocations": 1, + "gitMedianMs": 12.46, + "gitWorkMs": 0.402000000000001, + "gitFloorBound": true, + "ratioToGit": null + }, + { + "op": "open_screen", + "label": "Everything the first screen needs, at once", + "scale": "11 concurrent reads", + "samples": 3, + "firstMs": 5420.711, + "repeatMedianMs": 5420.4, + "repeatP95Ms": 5622.622, + "gitCommand": null, + "gitInvocations": 0, + "gitMedianMs": null, + "gitWorkMs": null, + "gitFloorBound": false, + "ratioToGit": null + }, + { + "op": "open_screen_ipc", + "label": "…including encoding it all for the webview", + "scale": "11 concurrent reads", + "samples": 3, + "firstMs": 5602.161, + "repeatMedianMs": 5432.862, + "repeatP95Ms": 5601.451, + "gitCommand": null, + "gitInvocations": 0, + "gitMedianMs": null, + "gitWorkMs": null, + "gitFloorBound": false, + "ratioToGit": null + }, + { + "op": "status", + "label": "Working-tree status", + "scale": "55,000 entries", + "samples": 3, + "firstMs": 5380.675, + "repeatMedianMs": 5419.887, + "repeatP95Ms": 5482.045, + "gitCommand": "git status --porcelain=v1 --untracked-files=all && git diff --numstat && git diff --cached --numstat", + "gitInvocations": 3, + "gitMedianMs": 3014.503, + "gitWorkMs": 2978.329, + "gitFloorBound": false, + "ratioToGit": 1.8197744439919161 + }, + { + "op": "log_first_page", + "label": "First page of history", + "scale": "1 commit", + "samples": 10, + "firstMs": 0.353, + "repeatMedianMs": 0.251, + "repeatP95Ms": 0.256, + "gitCommand": "git log --topo-order --max-count=500 --format=%H%n%an%n%ae%n%at%n%s", + "gitInvocations": 1, + "gitMedianMs": 13.179, + "gitWorkMs": 1.1210000000000004, + "gitFloorBound": true, + "ratioToGit": null + }, + { + "op": "log_page_deep", + "label": "Ten pages into history", + "scale": "1 commit", + "samples": 10, + "firstMs": 0.312, + "repeatMedianMs": 0.25, + "repeatP95Ms": 0.256, + "gitCommand": "git log --topo-order --skip=4500 --max-count=500 --format=%H%n%an%n%ae%n%at%n%s", + "gitInvocations": 1, + "gitMedianMs": 13.081, + "gitWorkMs": 1.0229999999999997, + "gitFloorBound": true, + "ratioToGit": null + }, + { + "op": "branches", + "label": "List every branch", + "scale": "1 branch", + "samples": 10, + "firstMs": 0.352, + "repeatMedianMs": 0.275, + "repeatP95Ms": 0.317, + "gitCommand": "git for-each-ref --format=%(refname)%(objectname)%(upstream) refs/heads refs/remotes", + "gitInvocations": 1, + "gitMedianMs": 12.756, + "gitWorkMs": 0.6980000000000004, + "gitFloorBound": true, + "ratioToGit": null + }, + { + "op": "tags", + "label": "List every tag", + "scale": "0 tags", + "samples": 10, + "firstMs": 0.171, + "repeatMedianMs": 0.145, + "repeatP95Ms": 0.147, + "gitCommand": "git for-each-ref --format=%(refname)%(objectname)%(*objectname) refs/tags", + "gitInvocations": 1, + "gitMedianMs": 12.756, + "gitWorkMs": 0.6980000000000004, + "gitFloorBound": true, + "ratioToGit": null + }, + { + "op": "diff_commit", + "label": "Diff the selected commit", + "scale": "50,000 files", + "samples": 10, + "firstMs": 1530.386, + "repeatMedianMs": 1510.808, + "repeatP95Ms": 1674.506, + "gitCommand": "git show --format= --patch HEAD", + "gitInvocations": 1, + "gitMedianMs": 522.06, + "gitWorkMs": 510.00199999999995, + "gitFloorBound": false, + "ratioToGit": 2.9623570103646655 + }, + { + "op": "diff_workdir_file", + "label": "Diff one modified file", + "scale": "1 hunk", + "samples": 10, + "firstMs": 17.18, + "repeatMedianMs": 0.632, + "repeatP95Ms": 0.639, + "gitCommand": null, + "gitInvocations": 0, + "gitMedianMs": null, + "gitWorkMs": null, + "gitFloorBound": false, + "ratioToGit": null + }, + { + "op": "file_history", + "label": "History of one file", + "scale": "1 commit", + "samples": 10, + "firstMs": 0.276, + "repeatMedianMs": 0.1, + "repeatP95Ms": 0.105, + "gitCommand": "git log --topo-order --max-count=500 --format=%H%n%an%n%at%n%s -- pkg/000/gen_000000.ts", + "gitInvocations": 1, + "gitMedianMs": 13.32, + "gitWorkMs": 1.2620000000000005, + "gitFloorBound": true, + "ratioToGit": null + }, + { + "op": "list_all_files", + "label": "Browse the whole tree", + "scale": "55,000 files", + "samples": 10, + "firstMs": 195.887, + "repeatMedianMs": 181.487, + "repeatP95Ms": 240.574, + "gitCommand": "git ls-files --cached --others --exclude-standard", + "gitInvocations": 1, + "gitMedianMs": 54.416, + "gitWorkMs": 42.358, + "gitFloorBound": false, + "ratioToGit": 4.284597950800321 + } + ], + "soak": null + }, + { + "key": "refs", + "title": "refs", + "kind": "generated", + "blurb": "5,000 branches and 2,000 tags over a short history — a long-lived repository nobody prunes.", + "repository": { + "fixture": "refs", + "commits": 2000, + "trackedFiles": 32, + "branches": 5001, + "tags": 2000, + "dirtyEntries": 0 + }, + "gitSpawnFloorMs": 12.386, + "operations": [ + { + "op": "open", + "label": "Open the repository", + "scale": "a fresh handle", + "samples": 10, + "firstMs": 0.153, + "repeatMedianMs": 0.138, + "repeatP95Ms": 0.143, + "gitCommand": "git rev-parse HEAD", + "gitInvocations": 1, + "gitMedianMs": 12.342, + "gitWorkMs": 0, + "gitFloorBound": true, + "ratioToGit": null + }, + { + "op": "open_screen", + "label": "Everything the first screen needs, at once", + "scale": "11 concurrent reads", + "samples": 10, + "firstMs": 220.893, + "repeatMedianMs": 219.434, + "repeatP95Ms": 220.996, + "gitCommand": null, + "gitInvocations": 0, + "gitMedianMs": null, + "gitWorkMs": null, + "gitFloorBound": false, + "ratioToGit": null + }, + { + "op": "open_screen_ipc", + "label": "…including encoding it all for the webview", + "scale": "11 concurrent reads", + "samples": 10, + "firstMs": 220.518, + "repeatMedianMs": 219.773, + "repeatP95Ms": 224.361, + "gitCommand": null, + "gitInvocations": 0, + "gitMedianMs": null, + "gitWorkMs": null, + "gitFloorBound": false, + "ratioToGit": null + }, + { + "op": "status", + "label": "Working-tree status", + "scale": "0 entries", + "samples": 10, + "firstMs": 0.724, + "repeatMedianMs": 0.549, + "repeatP95Ms": 0.562, + "gitCommand": "git status --porcelain=v1 --untracked-files=all && git diff --numstat && git diff --cached --numstat", + "gitInvocations": 3, + "gitMedianMs": 38.918, + "gitWorkMs": 1.759999999999998, + "gitFloorBound": true, + "ratioToGit": null + }, + { + "op": "log_first_page", + "label": "First page of history", + "scale": "500 commits", + "samples": 10, + "firstMs": 786.833, + "repeatMedianMs": 135.184, + "repeatP95Ms": 138.328, + "gitCommand": "git log --topo-order --max-count=500 --format=%H%n%an%n%ae%n%at%n%s", + "gitInvocations": 1, + "gitMedianMs": 20.862, + "gitWorkMs": 8.475999999999999, + "gitFloorBound": false, + "ratioToGit": 15.949032562529496 + }, + { + "op": "log_page_deep", + "label": "Ten pages into history", + "scale": "500 commits", + "samples": 10, + "firstMs": 526.236, + "repeatMedianMs": 524.766, + "repeatP95Ms": 550.759, + "gitCommand": "git log --topo-order --skip=4500 --max-count=500 --format=%H%n%an%n%ae%n%at%n%s", + "gitInvocations": 1, + "gitMedianMs": 19.632, + "gitWorkMs": 7.246000000000002, + "gitFloorBound": false, + "ratioToGit": 72.42147391664363 + }, + { + "op": "branches", + "label": "List every branch", + "scale": "5,001 branches", + "samples": 10, + "firstMs": 184.019, + "repeatMedianMs": 180.081, + "repeatP95Ms": 181.962, + "gitCommand": "git for-each-ref --format=%(refname)%(objectname)%(upstream) refs/heads refs/remotes", + "gitInvocations": 1, + "gitMedianMs": 195.41, + "gitWorkMs": 183.024, + "gitFloorBound": false, + "ratioToGit": 0.983920141620771 + }, + { + "op": "tags", + "label": "List every tag", + "scale": "2,000 tags", + "samples": 10, + "firstMs": 148.003, + "repeatMedianMs": 148.38, + "repeatP95Ms": 150.987, + "gitCommand": "git for-each-ref --format=%(refname)%(objectname)%(*objectname) refs/tags", + "gitInvocations": 1, + "gitMedianMs": 55.724, + "gitWorkMs": 43.337999999999994, + "gitFloorBound": false, + "ratioToGit": 3.423785130832065 + }, + { + "op": "diff_commit", + "label": "Diff the selected commit", + "scale": "1 file", + "samples": 10, + "firstMs": 0.404, + "repeatMedianMs": 0.282, + "repeatP95Ms": 0.293, + "gitCommand": "git show --format= --patch HEAD", + "gitInvocations": 1, + "gitMedianMs": 12.971, + "gitWorkMs": 0.5850000000000009, + "gitFloorBound": true, + "ratioToGit": null + }, + { + "op": "file_history", + "label": "History of one file", + "scale": "63 commits", + "samples": 10, + "firstMs": 20.661, + "repeatMedianMs": 9.374, + "repeatP95Ms": 9.883, + "gitCommand": "git log --topo-order --max-count=500 --format=%H%n%an%n%at%n%s -- src/file_0.txt", + "gitInvocations": 1, + "gitMedianMs": 24.316, + "gitWorkMs": 11.93, + "gitFloorBound": false, + "ratioToGit": 0.7857502095557419 + }, + { + "op": "list_all_files", + "label": "Browse the whole tree", + "scale": "32 files", + "samples": 10, + "firstMs": 0.475, + "repeatMedianMs": 0.158, + "repeatP95Ms": 0.161, + "gitCommand": "git ls-files --cached --others --exclude-standard", + "gitInvocations": 1, + "gitMedianMs": 12.878, + "gitWorkMs": 0.4920000000000009, + "gitFloorBound": true, + "ratioToGit": null + } + ], + "soak": null + } + ] +} diff --git a/docs/dev/performance.md b/docs/dev/performance.md new file mode 100644 index 00000000..933311cb --- /dev/null +++ b/docs/dev/performance.md @@ -0,0 +1,336 @@ +# Performance — the large-repo benchmark + +"Slow on big repositories" is the most consistent structural complaint about +every established GUI client, and being fast is one of the two strongest claims +this project makes. This file is what turns that claim into a number somebody +else can check, and what makes a regression in it visible before a stranger +finds it (#257). + +Read the spec for the reasoning behind every choice below: +`docs/superpowers/specs/2026-09-17-large-repo-benchmark-spec.md`. + +## Running it + +```bash +pnpm bench # the three generated fixtures, ~1 minute to build +pnpm bench --linux # …plus a real clone of torvalds/linux (multi-GB) +pnpm bench --fixture deep # one fixture +pnpm bench --soak 60 # add a 60-minute soak per fixture +pnpm bench --no-publish # measure and print; write nothing +``` + +Three pieces, each runnable on its own: + +| piece | what it does | +| --- | --- | +| `scripts/bench-fixtures.mjs` | materialises the repositories under `$PGBENCH_HOME` (default `~/.cache/platypusgit-bench`) | +| `src-tauri/benches/repo_bench.rs` | measures one repository, writes a JSON document of raw samples | +| `scripts/bench-report.mjs` | renders those into `benchmark.json` beside this file, and the table block below | + +**Run it on a quiet machine.** Every number is wall clock, so a compile in +another window is recorded as this program being slow — and these are the +figures a stranger will check. + +The harness is behind `--features bench` and `required-features` in +`src-tauri/Cargo.toml`, so `cargo check` and `cargo test` — the whole Rust CI +gate — never build or link it. + +## The fixtures + +Breadth hurts differently from depth, so one "big repo" fixture would give a +number that cannot say which dimension moved when it regresses. Three generated +fixtures isolate one dimension each; the fourth is the real thing. + +| fixture | shape | isolates | +| --- | --- | --- | +| `deep` | 50,000 commits, 16 files | the log walk | +| `wide` | 50,000 files, all modified, 5,000 untracked | `status` | +| `refs` | 5,000 branches, 2,000 tags, 2,000 commits | ref enumeration | +| `linux` | a real clone of `torvalds/linux` | all of it, for real | + +The generated three are **deterministic** — fixed seed, fixed timestamps, fixed +author — so the same parameters produce the same object ids on every machine, +and they build in seconds from `git fast-import`. That is what makes "a script +anyone can run" true rather than aspirational: nobody re-runs a benchmark that +starts with a six-gigabyte download. `linux` is opt-in for exactly that reason, +and it is not generated because a synthetic repository cannot stand in for 1.4 +million real commits, 90,000 real paths and a real pack layout. + +Fixtures are build artifacts. They live under `$PGBENCH_HOME`, never in the +tree, and a fixture whose recorded shape no longer matches +`scripts/bench-fixtures.mjs` is regenerated rather than reused — comparing +today's numbers against a differently-shaped repository is worse than having no +previous numbers at all. + +## What is measured, and what is therefore not + +The harness drives `Libgit2Backend` through the real `GitBackend` trait. **No +number here includes React**, because there is no webview in it. That is a real +limitation, stated plainly, and it is still the right layer: it is where a big +repository is actually expensive, it is where a regression lands, and the render +on top is bounded by the *window* rather than by the repository — the log is +paged at 500, diff rows are windowed, long lists are virtualised — so it does +not grow with the fixture. + +Two measurements narrow the gap on purpose rather than pretending it is not +there: + +* **`open_screen`** issues the eleven reads `useRepoStore.refreshAll` issues, + simultaneously, from separate threads. A composite is the only measurement + that can catch "a slow status blocks everything else on that repo" — the trap + `git/repo_locks.rs` exists to avoid, and the one an op-at-a-time benchmark is + structurally blind to. +* **`open_screen_ipc`** is the same fan-out plus `serde_json` encoding of every + payload, because that encoding is real work on a 500-commit page and it + happens before the frontend sees a byte. + +### "First" and "repeat", not "cold" and "warm" + +**First** is one call on a freshly constructed backend and a freshly opened +repository — what happens when you open a repository in the app, with libgit2's +object database, ref database and pack indices all unbuilt. **Repeat** is the +median over many calls on that handle; the p95 beside it is nearest-rank. + +Neither purges the operating system's file cache. The words "cold" and "warm" +are avoided precisely because they would imply it did. A first-boot number is +larger than anything below, by an amount that depends on the disk rather than on +this code. + +### The `git` baseline + +Every operation a single `git` invocation can answer is measured as that +invocation too, and the table prints the ratio. + +The point is not to win. `git` is the floor, and a ratio near it is the good +outcome. The point is that **a ratio survives leaving this machine**: a +millisecond figure from somebody else's laptop tells a reader nothing they can +check, and a ratio that doubles is a regression even on hardware that got +faster. + +The baselines ask the *same* question, not the cheapest one sharing a name. +`status` is the clearest case: `GitBackend::status` returns per-file added and +removed counts, so its baseline is `git status --porcelain` **plus** both +`--numstat` diffs. Comparing it to a bare `git status` would be comparing it to +less work than it does. Every baseline command is recorded in +`docs/dev/benchmark.json` under `gitCommand`, so the comparison can be +disputed with evidence rather than in the abstract. + +## Where the results go + +| artifact | committed | what it is | +| --- | --- | --- | +| `$PGBENCH_HOME/results/.json` | no | raw, every sample — what makes a result checkable | +| `docs/dev/benchmark.json` | yes | the published record, summary statistics only | +| the table block below | yes | the same numbers, as a document | + +Both committed artifacts are **generated and not hand-editable**. +`test/benchmark.test.ts` re-renders the markdown from the JSON and fails when +they disagree. That guard is the point of the whole exercise: the way a measured +number turns back into an adjective is somebody nudging it in a hurry. + +The record sits beside this file rather than under `site/`, and both are covered +by the `docs/dev/` entry already in the `js` path filter in +`.github/workflows/tests.yml`. Without that coverage the guard would be +skippable by exactly the change it polices — the failure mode #210 already +shipped once. + +### The marketing site does not print these yet + +#257 asks for a measured figure on the site in place of an adjective, and the +block to do it is written. It is **deliberately not shipped yet**: the honest +headline today is that opening `torvalds/linux` takes 15.8 seconds, and the +right response to that is to fix it rather than to publish it as a selling +point. It ships once the log-walk work in the findings below lands — at which +point the record moves to `site/src/data/` beside `comparison.json`, which is +where this repository keeps published records the site reads. + +Until then nothing under `site/**` is touched by a re-measurement, which also +means `pnpm bench` cannot redeploy the website by accident. + +## What the numbers say + +The first run of this benchmark found three things. They are recorded here +because the tables above will move and the reasoning will not, and because a +number with no reading beside it is a number nobody acts on. + +### 1. We are fine until history gets very deep — and then we are not + +The whole first screen — the eleven reads `refreshAll` issues, all at once — +costs **255 ms** on a 50,000-commit repository and **219 ms** on one with 5,001 +branches and 2,000 tags. That is the size at which GitKraken's own users report +it falling over, and it is a good answer. + +On `torvalds/linux` the same screen costs **15.8 seconds**, and scrolling ten +pages into its history costs **two minutes and thirty-eight seconds**. That is +not a good answer, and publishing it is the point of the exercise: the user who +opens a 1.4-million-commit repository and waits is the user this project was +written for. + +### 2. The log walk is not slow — the topological SORT is, and it is re-paid per page + +The obvious reading of "our 500-commit page takes 252 ms and `git log -500` +takes 41 ms" is that the walk is six times too slow. It is wrong, and it nearly +reached this document. + +`log_page` walks with `Sort::TIME | Sort::TOPOLOGICAL`, because the commit +graph's lane assignment depends on topological order. A default `git log` does +not sort that way. Asked the same question, `git log --topo-order -500` costs +284 ms on the `deep` fixture against our 275 ms — **parity**. The baselines in +this benchmark are `--topo-order` for exactly that reason, and the near miss is +written up in the spec. + +What survives is sharper. git pays for that sort **once and then skips**; we pay +it per page: + +| | `deep` (50k commits) | `torvalds/linux` (1.5M commits) | +| --- | --- | --- | +| our first page | 252 ms | 15.95 s | +| our tenth page | 2.40 s | 157.67 s | +| `git log --topo-order --skip=4500 -500` | 193 ms | 9.68 s | + +The per-page cost is flat in depth — ten pages cost ten times one page — which +is the signature of restarting the walk rather than continuing it. The cursor +already carries the frontier, so the walk logically continues; it is the sort +that is rebuilt. + +### 3. The ref map is rebuilt on every page, too + +`log_page` calls `collect_ref_map(repo)` per call, enumerating and peeling every +ref so the page can decorate its 500 commits. Two fixtures isolate it, and the +variable between them is refs rather than history: + +| fixture | history | refs | ours | `git` work | ratio | +| --- | --- | --- | --- | --- | --- | +| `deep` | 50,000 commits | 1 | 252 ms | 194 ms | 1.3× | +| `refs` | 2,000 commits | 7,001 | 135 ms | 8.5 ms | **16×** | + +Twenty-five times *less* history, and still most of the cost. This one looks +much cheaper to fix than the sort: cache the map per repository and invalidate +it on ref writes. + +### What is already good, and worth not breaking + +* **Opening a repository is free** — 0.11 ms on the kernel, a fresh libgit2 + handle and nothing else. Every "first call" number is measured against one. +* **The concurrent fan-out really is concurrent.** On `wide`, `open_screen` + costs 5.42 s and `status` alone costs 5.42 s: eleven reads cost what the + slowest one costs, not the sum. That is `git/repo_locks.rs` doing its job, and + it is the single thing most worth not regressing. +* **IPC encoding is not a cost worth optimising.** `open_screen_ipc` is within + noise of `open_screen` on every fixture, including a 500-commit page. +* **`status` tracks git.** 5.42 s against 2.98 s of git's work on 55,000 + changed entries — 1.8×, on an operation where git itself takes three seconds. + It is slow because the question is expensive, not because of how we ask it. + +### Known characteristics that are not on the tables + +* **File history is unbounded on a cold path.** `file_history` stops at `limit` + matches, so on a file with fewer than 500 commits it walks to the root of + history with a tree comparison per commit. On the kernel that is 1.5 million + of them for one click. The benchmark measures the *most frequently changed* + path precisely so it terminates; see `hottest_path` in the harness. It also + takes the exclusive lock (`with_repo`, not `with_repo_read`), so it blocks + every other read on that repository while it runs. +* **No fixture carries a commit-graph file, and it would not help us if it + did.** A fresh clone has none — `git clone` does not write one, and + `gc --auto` does not fire on a single packfile — so this is what a user gets + on day one. It matters enormously to git: writing one for the kernel takes 14 + seconds, and `git log --topo-order -500` then drops from 9.51 s to **21 ms**. + It does not measurably help us. With the file present, our first page took + 63.4 s for four calls against 64.2 s without it: libgit2's revwalk does not + read it. That is the most useful single fact this benchmark produced, because + it rules out the cheap fix and says where the work actually has to go. + +## Results + + + +Measured on Apple M4 Pro (14 cores, 48 GB, macos/aarch64) with git version 2.50.1 (Apple Git-155), on 2026-09-17. Up to 10 repeats per operation, time-boxed to 20s each — so a cheap operation gets the full count and an expensive one gets at least three. The published record records how many each row actually took. + +The **`git` work** column is that baseline's wall clock with process start-up subtracted (12.4 ms per invocation on this machine, measured), because we pay none of it — the backend is libgit2, in process. That is deliberately the comparison that makes us look worse: against git's wall clock we would get a ten-millisecond head start on every row. **†** marks a baseline where start-up swamped the work, leaving a remainder too small to divide by; those rows print no ratio rather than a flattering one. + +### torvalds/linux + +A real clone of the Linux kernel: the repository people mean when they say a git client is slow. + +*1,482,923 commits · 96,034 tracked files · 946 tags · 13 changed entries* + +| Operation | Result size | First call | Repeat | p95 | `git` work | vs `git` | +| --- | --- | --- | --- | --- | --- | --- | +| Open the repository | a fresh handle | 0.26 ms | 0.11 ms | 0.13 ms | † | — | +| Everything the first screen needs, at once | 11 concurrent reads | 15.91 s | 15.84 s | 15.86 s | — | — | +| …including encoding it all for the webview | 11 concurrent reads | 15.80 s | 15.82 s | 15.86 s | — | — | +| Working-tree status | 26 entries | 1.15 s | 989 ms | 1.10 s | 517 ms | 1.9× | +| First page of history | 500 commits | 15.96 s | 15.95 s | 16.49 s | 9.51 s | 1.7× | +| Ten pages into history | 500 commits | 156.38 s | 157.67 s | 159.94 s | 9.68 s | 16× | +| List every branch | 3 branches | 1.65 ms | 0.87 ms | 0.89 ms | † | — | +| List every tag | 946 tags | 27.1 ms | 19.7 ms | 20.2 ms | 23.7 ms | 0.83× | +| Diff the selected commit | 3 files | 142 ms | 139 ms | 140 ms | 11.5 ms | 12× | +| Diff one modified file | 2 hunks | 68.2 ms | 2.14 ms | 2.22 ms | — | — | +| History of one file | 500 commits | 15.91 s | 15.42 s | 16.06 s | 5.65 s | 2.7× | +| Browse the whole tree | 96,034 files | 782 ms | 515 ms | 642 ms | 218 ms | 2.4× | + +### deep + +50,000 commits over a small tree — the size past which GitKraken's own users report native clients beating it. + +*50,000 commits · 16 tracked files* + +| Operation | Result size | First call | Repeat | p95 | `git` work | vs `git` | +| --- | --- | --- | --- | --- | --- | --- | +| Open the repository | a fresh handle | 0.14 ms | 0.11 ms | 0.12 ms | † | — | +| Everything the first screen needs, at once | 11 concurrent reads | 252 ms | 253 ms | 256 ms | — | — | +| …including encoding it all for the webview | 11 concurrent reads | 252 ms | 253 ms | 254 ms | — | — | +| Working-tree status | 0 entries | 0.69 ms | 0.53 ms | 0.55 ms | † | — | +| First page of history | 500 commits | 261 ms | 249 ms | 251 ms | 193 ms | 1.3× | +| Ten pages into history | 500 commits | 2.39 s | 2.39 s | 2.40 s | 192 ms | 12× | +| List every branch | 1 branch | 0.42 ms | 0.28 ms | 0.29 ms | † | — | +| List every tag | 0 tags | 0.17 ms | 0.15 ms | 0.15 ms | † | — | +| Diff the selected commit | 1 file | 0.48 ms | 0.34 ms | 0.35 ms | † | — | +| History of one file | 500 commits | 293 ms | 65.0 ms | 65.7 ms | 319 ms | 0.20× | +| Browse the whole tree | 16 files | 0.50 ms | 0.13 ms | 0.13 ms | † | — | + +**Soak.** 2,344 first-screen fan-outs over 10 minutes. Resident memory 67 MB → 69 MB (peak 69 MB). Median fan-out 252 ms in the first half, 252 ms in the second — -0.2%. + +### wide + +50,000 tracked files with every one of them modified, plus 5,000 untracked — a monorepo just after a codemod. + +*1 commits · 50,000 tracked files · 55,000 changed entries* + +| Operation | Result size | First call | Repeat | p95 | `git` work | vs `git` | +| --- | --- | --- | --- | --- | --- | --- | +| Open the repository | a fresh handle | 0.25 ms | 0.11 ms | 0.13 ms | † | — | +| Everything the first screen needs, at once | 11 concurrent reads | 5.42 s | 5.42 s | 5.62 s | — | — | +| …including encoding it all for the webview | 11 concurrent reads | 5.60 s | 5.43 s | 5.60 s | — | — | +| Working-tree status | 55,000 entries | 5.38 s | 5.42 s | 5.48 s | 2.98 s | 1.8× | +| First page of history | 1 commit | 0.35 ms | 0.25 ms | 0.26 ms | † | — | +| Ten pages into history | 1 commit | 0.31 ms | 0.25 ms | 0.26 ms | † | — | +| List every branch | 1 branch | 0.35 ms | 0.28 ms | 0.32 ms | † | — | +| List every tag | 0 tags | 0.17 ms | 0.14 ms | 0.15 ms | † | — | +| Diff the selected commit | 50,000 files | 1.53 s | 1.51 s | 1.67 s | 510 ms | 3.0× | +| Diff one modified file | 1 hunk | 17.2 ms | 0.63 ms | 0.64 ms | — | — | +| History of one file | 1 commit | 0.28 ms | 0.10 ms | 0.10 ms | † | — | +| Browse the whole tree | 55,000 files | 196 ms | 181 ms | 241 ms | 42.4 ms | 4.3× | + +### refs + +5,000 branches and 2,000 tags over a short history — a long-lived repository nobody prunes. + +*2,000 commits · 32 tracked files · 5,001 branches · 2,000 tags* + +| Operation | Result size | First call | Repeat | p95 | `git` work | vs `git` | +| --- | --- | --- | --- | --- | --- | --- | +| Open the repository | a fresh handle | 0.15 ms | 0.14 ms | 0.14 ms | † | — | +| Everything the first screen needs, at once | 11 concurrent reads | 221 ms | 219 ms | 221 ms | — | — | +| …including encoding it all for the webview | 11 concurrent reads | 221 ms | 220 ms | 224 ms | — | — | +| Working-tree status | 0 entries | 0.72 ms | 0.55 ms | 0.56 ms | † | — | +| First page of history | 500 commits | 787 ms | 135 ms | 138 ms | 8.48 ms | 16× | +| Ten pages into history | 500 commits | 526 ms | 525 ms | 551 ms | 7.25 ms | 72× | +| List every branch | 5,001 branches | 184 ms | 180 ms | 182 ms | 183 ms | 0.98× | +| List every tag | 2,000 tags | 148 ms | 148 ms | 151 ms | 43.3 ms | 3.4× | +| Diff the selected commit | 1 file | 0.40 ms | 0.28 ms | 0.29 ms | † | — | +| History of one file | 63 commits | 20.7 ms | 9.37 ms | 9.88 ms | 11.9 ms | 0.79× | +| Browse the whole tree | 32 files | 0.47 ms | 0.16 ms | 0.16 ms | † | — | + + diff --git a/docs/dev/testing.md b/docs/dev/testing.md index 87ea1000..6fcacf53 100644 --- a/docs/dev/testing.md +++ b/docs/dev/testing.md @@ -1,7 +1,16 @@ # Testing — four layers, Docker e2e, CI Part of the `docs/dev/` set (`architecture`, `testing`, `frontend`, `backend`, -`distribution`). `test/docs.test.ts` reads this set together with CLAUDE.md. +`distribution`, `performance`). `test/docs.test.ts` reads this set together with +CLAUDE.md. + +**Performance is measured, not tested** — the large-repo benchmark lives in +`docs/dev/performance.md` and is a fifth thing, deliberately outside all four +layers below and outside CI. Its numbers are wall clock, which on a shared +runner is a flake generator, and its fixtures take minutes to build. It is +something you run on a quiet machine when you have touched the log walk, +`status` or the refresh path; the committed results are what make a regression +in them visible. ## The four layers diff --git a/docs/superpowers/plans/2026-09-17-large-repo-benchmark.md b/docs/superpowers/plans/2026-09-17-large-repo-benchmark.md new file mode 100644 index 00000000..a9cb0509 --- /dev/null +++ b/docs/superpowers/plans/2026-09-17-large-repo-benchmark.md @@ -0,0 +1,150 @@ +# Large-repo benchmark — implementation plan (#257) + +Spec: `docs/superpowers/specs/2026-09-17-large-repo-benchmark-spec.md`. + +One PR. Nothing here changes app behaviour: every file is a new build artifact, +a new document, or a pointer to one. The two exceptions are the `js` path filter +in `tests.yml` (one path added) and a new section on the site's `/features` +page. + +## 1. Fixtures — `scripts/bench-fixtures.mjs` + +Generates `deep`, `wide` and `refs` from `git fast-import` streams; clones +`linux` on demand. Everything under `$PGBENCH_HOME`. + +* Deterministic: a seeded LCG for content, a fixed epoch for every timestamp, a + fixed author. Same parameters ⇒ same object ids on any machine. +* `--done` on the import, so a truncated stream is an error rather than a + quietly partial repository — a fixture that stopped at commit 31,000 would + publish numbers nobody can reproduce. +* Each fixture's shape is stamped to `.fixture.json` **beside** the + repository, never inside it: `wide` is measured partly by how many untracked + files `status` reports, and a stray file in the work tree would be one of them. +* Config left at git's defaults. `core.untrackedCache` and `core.fsmonitor` + both make `status` dramatically cheaper, and benchmarking with them on would + publish a number almost nobody's repository produces. `gc.auto=0` is the one + deviation, because a `gc --auto` firing mid-run is measured as whatever + operation happened to be in flight. + +**Done when** all three build in under a minute and the shapes verify: +50,000 commits / 50,000 files with 55,000 changed entries / 5,001 branches and +2,000 tags of which 500 annotated. + +## 2. Harness — `src-tauri/benches/repo_bench.rs` + +A `harness = false` bench target behind `required-features = ["bench"]`, so +`cargo check` and `cargo test` — the whole Rust CI gate — never build or link +it. + +Measures, per operation: one `first` call on a freshly constructed backend and +a freshly opened repository, then N repeats on one handle (median, min, p95, +max, and every sample kept). + +* `open_screen` runs the eleven `refreshAll` reads simultaneously behind a + `Barrier`. The barrier is load-bearing: spawning eleven threads without one + lets the first read finish before the last starts, which turns the one + measurement that exists to find lock contention into a measurement with none + in it. +* `open_screen_ipc` is the same fan-out plus `serde_json` encoding of every + payload. +* Every op a single `git` invocation can answer carries that invocation as a + baseline, chosen to ask the same question — `status`'s baseline is + `git status --porcelain` plus both `--numstat` diffs, because + `GitBackend::status` returns per-file line counts. +* `--soak N` repeats the fan-out for N minutes and reports RSS at start, end and + peak, plus the median fan-out in the first half of the run against the second. +* A `scale` string ("55,000 entries") travels with every timing, because a + benchmark that silently started measuring an empty result is the classic way + to publish an excellent number for nothing at all. +* Repeats are **time-boxed** rather than counted: `budget / first_call`, clamped + between three and `--iterations`. Ten repeats of an eight-second log page is + thirteen minutes for one table row; three is enough for a median when the + operation is that slow, and every row publishes its own sample count. +* The subject path for file history is the **most frequently changed path in the + last 2,000 commits**, not "a path HEAD touched". The obvious choice is + unusable: `file_history` stops at `limit` matches, so on a rarely-touched file + it never reaches 500 and walks the whole history with a tree comparison per + commit — on `torvalds/linux` the benchmark simply does not finish. Measured + the hard way, by watching it not finish. + +Output is hand-rolled JSON so key order is stable — a generated file that +reorders itself makes every diff of it unreadable, and this one is committed. + +**Done when** it compiles with `--features bench`, is absent from a plain +`cargo test`, and produces a plausible document for all four fixtures. + +## 3. Orchestration — `scripts/bench.sh`, `pnpm bench` + +Ensure fixtures → build the harness → run it per fixture → render. Flags: +`--linux`, `--fixture`, `--soak`, `--iterations`, `--force`, `--no-publish`. +Invokes the harness through `cargo bench` rather than a built path, because a +`harness = false` bench has no stable file name and globbing for it picks up +stale binaries. + +## 4. Rendering — `scripts/bench-report.mjs` + +Raw runs → `docs/dev/benchmark.json` → the markdown table block spliced between +the generated markers in `docs/dev/performance.md`. + +The direction matters: **the markdown is rendered from the published JSON**, not +from the raw runs. That is what gives the guard test leverage — it re-renders +from the committed JSON and compares to the committed markdown, so the two can +only agree if both came out of one run. Raw samples stay out of the repository. + +## 5. Documents + +* `docs/dev/performance.md` — method, fixtures, what is and is not included, the + baseline argument, the generated results, and what the numbers say. +* `CLAUDE.md` — one pointer in the doc list, one line in the command list. +* The spec, committed beside the other specs. + +## 6. Site — written, then held back + +A `Benchmark.astro` block on `/features` reading a typed `benchmark.ts`, printing +the machine beside the numbers, keeping the `git` column and printing the rows +where we lose. + +**Not shipped in this pass.** The first run's headline is 15.8 seconds to open +`torvalds/linux`, and the right response to that number is to fix it, not to put +it on a marketing page. It follows the log-walk work. Nothing benchmark-related +lives under `site/**` meanwhile, which also stops `pnpm bench` from triggering +the site deployment. + +## 7. Guards + +`test/benchmark.test.ts`: + +1. re-renders the markdown from `benchmark.json` and compares byte for byte; +2. requires a machine, a date and an iteration count; +3. requires `open`, `open_screen`, `status` and `log_first_page` on every + fixture, a non-empty `scale` and a positive median on every row; +4. rejects an operation key the renderer does not know how to order (it would + sort silently to the end of the table); +5. requires a recorded `git` command behind every printed ratio; +6. asserts `docs/dev/` is in the `js` path filter in `tests.yml` — a guard + skippable by the change it polices is the #210 failure mode, and it has + shipped here once. Both of its inputs live there, so no new filter entry is + needed. + +The hand-typed-figure check goes in with the site block, since it polices the +markup that block adds. + +## 8. Run it and publish + +A clean run of all four fixtures on a quiet machine, then commit +`benchmark.json` and `performance.md` together. + +Read the results before writing the prose. If a number is bad, the document says +so with the measurement attached and a follow-up issue gets filed — a benchmark's +job is to produce the number, and acting on a bad one is the next piece of work, +not this one. + +## Verification + +* `pnpm test` — the new guard plus the existing `docs`/`unit` projects. +* `pnpm tsc --noEmit`, and a site build for the new component. +* `cargo test` — unchanged, and specifically proving the bench target is not + built by it. +* `cargo build --features bench --bench repo_bench` — proving it still is when + asked. +* No e2e: nothing here reaches the app. diff --git a/docs/superpowers/specs/2026-09-17-large-repo-benchmark-spec.md b/docs/superpowers/specs/2026-09-17-large-repo-benchmark-spec.md new file mode 100644 index 00000000..87dd0572 --- /dev/null +++ b/docs/superpowers/specs/2026-09-17-large-repo-benchmark-spec.md @@ -0,0 +1,211 @@ +# Large-repo benchmark (#257) + +"Slow on big repositories" is the most consistent structural complaint about +every established GUI client, and being fast is one of this project's two +strongest claims. There are no numbers, so right now it is only a claim. + +This builds a benchmark anybody can run, publishes what it measures, and puts a +measured figure where an adjective used to be. + +## What "fast" has to mean before it can be measured + +The complaint is never about a microbenchmark. It is about four moments: + +1. **Opening a repository.** From double-click to a screen with history on it. +2. **Status on a dirty tree.** The auto-fetch and the file watcher both call + this, so on a big working tree it is not one cost but a recurring one. +3. **Scrolling history**, including past the first page. +4. **Jank after a while** — the Sourcetree complaint, the one nobody answers. + +Everything below is arranged around those four, and an operation that does not +serve one of them is not measured. A benchmark that reports thirty numbers is a +benchmark whose regressions nobody notices. + +## Three decisions that shape the rest + +### It measures the backend, and says so + +The harness drives `Libgit2Backend` through the real `GitBackend` trait. There +is no webview in it, so no number here includes React. + +That is a real limitation and the published document states it in those words. +It is nonetheless the right layer: + +* it is where a big repository is expensive — a 500-commit page and a 55,000- + entry status are git work, not render work; +* it is where a regression lands, and it is diffable; +* the render on top is bounded by the *window*, not by the repository (the log + is paged at 500, diff rows are windowed, long lists are virtualised), so it + does not grow with the fixture. + +Two measurements narrow the gap on purpose rather than pretending it is not +there. `open_screen` issues the **eleven reads `refreshAll` issues**, +simultaneously, from separate threads — which is the only shape that can catch +"a slow status blocks everything else on that repo", the trap +`git/repo_locks.rs` exists to avoid and the one an op-at-a-time benchmark is +structurally blind to. `open_screen_ipc` adds `serde_json` encoding of every +payload, because that encoding is real work on a 500-commit page and it happens +before the frontend sees a byte. + +An end-to-end number through the webview is a separate piece of work. It needs +the e2e harness, a Linux container and a multi-gigabyte fixture inside it, and it +would still be a worse regression detector than this because a wall-clock number +that includes WebDriver is dominated by WebDriver. + +### Four fixtures, three of them generated + +Breadth hurts differently from depth, so one "big repo" fixture would produce a +number that cannot say which dimension moved when it regresses. Each generated +fixture isolates exactly one: + +| fixture | shape | isolates | +| --- | --- | --- | +| `deep` | 50,000 commits, 16 files | the log walk | +| `wide` | 50,000 files, all modified, 5,000 untracked | `status` | +| `refs` | 5,000 branches, 2,000 tags, 2,000 commits | ref enumeration | +| `linux` | a real clone of `torvalds/linux` | all of it, for real | + +`deep` is 50,000 because that is the threshold GitKraken's own users name. +`wide` is a monorepo just after a codemod. `refs` is a long-lived repository +nobody prunes, which is most of them. + +The three generated ones are **deterministic**: fixed seed, fixed timestamps, +fixed author, so the same parameters produce the same object ids on every +machine. They build in seconds from `git fast-import`, which is what makes +"a script anyone can run" true rather than aspirational — nobody re-runs a +benchmark that starts with a six-gigabyte download. + +`linux` is not generated, because a synthetic repository cannot stand in for 1.4 +million real commits, 90,000 real paths and a real pack layout, and that is the +repository people mean. It is opt-in (`--linux`). + +Fixtures live under `$PGBENCH_HOME` (default `~/.cache/platypusgit-bench`), +never in the tree. + +### "First" and "repeat", not "cold" and "warm" + +`first` is one call on a freshly constructed backend and a freshly opened +repository — what happens when you open a repository in the app, with libgit2's +object database, ref database and pack indices all unbuilt. `repeat` is the +median of many calls on that handle. + +Neither purges the operating system's file cache. The words "cold" and "warm" +are avoided precisely because they would imply it did; a first-boot number is +larger than anything published here by an amount that depends on the disk rather +than on this code. Saying so is cheaper than being caught not having said so. + +## The `git` baseline + +Every operation a single `git` invocation can answer is measured as that +invocation too, and the table prints the ratio. + +The point is not to win. `git` is the floor, and a ratio near it is the good +outcome. The point is that **a ratio survives leaving this machine**: a +millisecond figure from somebody else's laptop tells a reader nothing they can +check, and a ratio that doubles is a regression even on hardware that got +faster. + +The baselines ask the *same* question, not the cheapest one sharing a name. +`status` is the clearest case: `GitBackend::status` returns per-file added and +removed counts, so its baseline is `git status --porcelain` **plus** both +`--numstat` diffs. Comparing it to a bare `git status` would be comparing it to +less work than it does, and flattering ourselves in public is the one thing a +benchmark must not do. + +**The rule cuts both ways, and it caught a false result here before publication.** +The first draft of the log baseline was a plain `git log --max-count=500`, which +made the first page of history look fourteen times slower than git on the `deep` +fixture. It is not. `log_page` walks with `Sort::TIME | Sort::TOPOLOGICAL` +because the commit graph's lane assignment depends on topological order, and a +default `git log` does not sort that way. Measured on `deep`: + +| | | +| --- | --- | +| `git log --max-count=500` | 41 ms | +| `git log --topo-order --max-count=500` | 284 ms | +| `log_page(None, None, 500)` | 275 ms | + +The first page is at **parity**. The wrong baseline would have published a +regression that does not exist, in the very document written to stop people +publishing numbers they had not checked — so the baselines are `--topo-order`, +and the reason is written beside them in the source. + +What survives the correction is sharper and still worth acting on: git pays for +that sort **once** and then skips, while `log_page` pays it per page. Page ten +is 2.61 s against `git log --topo-order --skip=4500 --max-count=500` at 219 ms. + +## Repeats are time-boxed, not counted + +A fixed repeat count is wrong at both ends. Ten repeats of a 0.15 ms operation +buys a rounding error's worth of extra confidence; ten repeats of an +eight-second log page on a kernel clone is thirteen minutes for one row of one +table, and a benchmark nobody has time to finish produces no numbers at all. + +So the count is derived: `budget / first_call`, clamped between three and the +requested maximum. Three is the floor because a median of two is the mean of +two. Every row records how many samples it actually took, so the published +table never implies a confidence it does not have — and the budget is soft by +construction, because the floor wins when one call already exceeds it. + +## The soak + +"Jank after a while" gets its own mode rather than being left out: `--soak N` +repeats the whole first-screen fan-out for N minutes and reports resident memory +at the start, at the end and at its peak, plus the median fan-out time in the +first half of the run against the second. + +Two halves rather than a fitted slope, because a slope invites reading a trend +into noise. It covers the backend only — a leak in React would not show here — +and the document says that. + +## Where the output goes + +Three artifacts, of which two are committed: + +* `$PGBENCH_HOME/results/.json` — raw, every sample, **not** committed. + It is what makes a result checkable, and it is also forty floating-point + numbers per run that nobody reads the diff of. +* `docs/dev/performance.md` — the developer-facing record: method, fixtures, + what the numbers mean, and a generated table block per fixture. +* `docs/dev/benchmark.json` — the published record, beside the document. + +Both committed artifacts are **generated and not hand-editable**, and +`test/benchmark.test.ts` re-renders the markdown from the JSON and fails when +they disagree. That guard is the point of the exercise: the way a measured +number turns back into an adjective is somebody nudging it in a hurry. + +Both live under `docs/dev/`, which is already in the `js` path filter in +`.github/workflows/tests.yml`. That is why the record lives there rather than +somewhere needing a filter entry of its own: a guard skippable by exactly the +change it polices is the failure mode #210 already shipped once. + +## What the site says — deferred, on purpose + +The plan was a "measured, not claimed" block on `/features`, printing real +figures from the JSON with the machine named beside them, under two rules taken +from the comparison table's house style: every figure comes from the JSON, and +it prints what was measured *including* where we are slow. + +**It is not shipping with the first run.** The measurement changed the decision: +the honest headline today is that opening `torvalds/linux` takes 15.8 seconds, +and a marketing page is not the right place to learn that. The block is written +and the rules above still hold; it ships once the log-walk finding is fixed, and +the record moves under `site/` at the same time. + +Holding it has a second benefit worth keeping either way: with no benchmark data +under `site/**`, a re-measurement cannot trigger the site deployment (`site.yml` +fires on any push to `main` touching that path), so `pnpm bench` can never +redeploy the website as a side effect. + +## Non-goals + +* **Comparing against other clients.** We cannot run GitKraken in a harness and + publish the result, and citing a forum post as a number would be the thing + this issue exists to stop doing. +* **A CI gate.** The numbers are wall clock on a shared runner, which is a + flake generator, and the fixtures take minutes to build. This is a thing you + run deliberately, and the published document is the record that makes a + regression visible. +* **Fixing what it finds.** A benchmark's job is to produce the number. Acting + on a bad one is the next piece of work, and it gets its own issue with the + measurement attached. diff --git a/package.json b/package.json index daa29aa8..72dbf82c 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,8 @@ "test:e2e": "pnpm test:e2e:build && pnpm test:e2e:run", "test:e2e:build": "pnpm tauri build --debug --no-bundle --features tauri/custom-protocol,e2e --config src-tauri/tauri.e2e.conf.json && mkdir -p e2e/.bin && cp src-tauri/target/debug/platypusgit e2e/.bin/platypusgit", "test:e2e:run": "wdio run e2e/wdio.conf.ts", - "test:e2e:docker": "e2e/e2e-docker.sh" + "test:e2e:docker": "e2e/e2e-docker.sh", + "bench": "scripts/bench.sh" }, "dependencies": { "@codemirror/commands": "^6.11.0", diff --git a/scripts/bench-fixtures.mjs b/scripts/bench-fixtures.mjs new file mode 100755 index 00000000..137f6b9f --- /dev/null +++ b/scripts/bench-fixtures.mjs @@ -0,0 +1,412 @@ +#!/usr/bin/env node +// Fixture repositories for the large-repo benchmark (issue 257). +// +// "Slow on big repositories" is the most consistent complaint about every +// established GUI client, and "we are fast" is one of this project's two +// strongest claims. A claim with no number behind it is an adjective, so the +// benchmark needs repositories big enough to hurt — and they have to be the +// SAME repositories on every machine, or the numbers cannot be compared to +// each other, let alone to a previous run. +// +// Hence generated rather than downloaded, for three of the four. A generated +// fixture is deterministic (fixed seed, fixed timestamps, fixed author), costs +// no network, and isolates ONE dimension each: +// +// * `deep` — many commits over a small tree. The log walk, and nothing else. +// * `wide` — one commit, an enormous tree, every file dirty. `status`, and +// nothing else. +// * `refs` — thousands of branches and tags over a short history. Ref +// enumeration, and nothing else. +// +// Breadth hurts differently from depth, which is exactly why they are separate +// repositories: a single "big repo" fixture would give one number that cannot +// say which dimension moved when it regresses. +// +// The fourth, `linux`, is a real clone of torvalds/linux and is NOT generated — +// a synthetic repository cannot stand in for 1.4 million real commits, 90k real +// paths and a real pack layout, and that is the repository people actually mean +// when they say a client is slow. It is opt-in (`--linux`) because it is a +// multi-gigabyte download. +// +// Everything is written under `$PGBENCH_HOME` (default +// `~/.cache/platypusgit-bench`), never inside the repository: a fixture is a +// build artifact, it is regenerable, and `wide` alone is 50,000 files. +// +// Determinism, stated precisely. Identical inputs produce byte-identical object +// ids: the content comes from a seeded LCG, every timestamp is derived from a +// fixed epoch, and the author never varies. So `deep`'s HEAD sha is the same on +// your machine as on mine, and a fixture that drifted is visible rather than +// silent — a `.fixture.json` stamp beside each one records the parameters +// and the resulting HEAD, and a shape that no longer matches is regenerated +// rather than reused. + +import { spawn, spawnSync } from "node:child_process"; +import { + mkdirSync, + rmSync, + renameSync, + existsSync, + writeFileSync, + readFileSync, +} from "node:fs"; +import { join } from "node:path"; +import { homedir } from "node:os"; + +/** Bumped when a change here would produce different repositories. A fixture + * whose recorded version is older is regenerated rather than reused, because + * comparing today's numbers against a differently-shaped repository is worse + * than having no previous numbers at all. */ +export const FIXTURE_VERSION = 1; + +/** The shapes, and why each number is the number. + * + * `deep` at 50,000 commits is not arbitrary: it is the size at which + * GitKraken's own users report it falling over ("past ~50,000 commits, native + * clients beat it"), so it is the threshold the market has already identified. + * `wide` at 50,000 files with every one of them modified is the shape of a + * generated-code monorepo after a formatter run — the case where `status` is + * the whole cost. `refs` at 5,000 branches is a long-lived repository nobody + * prunes, which is most of them. */ +export const SHAPES = { + deep: { commits: 50_000, files: 16 }, + wide: { files: 50_000, dirs: 250, untracked: 5_000 }, + refs: { commits: 2_000, branches: 5_000, lightweightTags: 1_500, annotatedTags: 500 }, +}; + +const AUTHOR = "PlatypusGit Bench "; +/** 2023-11-14T22:13:20Z. Fixed so object ids do not depend on the clock. */ +const EPOCH = 1_700_000_000; + +export function benchHome() { + return process.env.PGBENCH_HOME || join(homedir(), ".cache", "platypusgit-bench"); +} + +/** A tiny LCG. Deterministic across Node versions in a way `Math.random` with a + * seed shim is not, and the statistical quality is irrelevant here — this only + * has to produce bytes that do not compress to nothing. */ +function lcg(seed) { + let s = seed >>> 0; + return () => { + s = (Math.imul(s, 1_664_525) + 1_013_904_223) >>> 0; + return s; + }; +} + +const WORDS = [ + "handle", "buffer", "commit", "index", "refspec", "packet", "stream", "node", + "cursor", "window", "lane", "hunk", "blob", "tree", "ref", "oid", "walk", + "stage", "merge", "rebase", "fetch", "prune", "shard", "lock", "queue", +]; + +/** Plausible source-ish text. Real-looking lines matter: a file of one repeated + * byte deltas and packs unlike anything a user has, which would flatter every + * number measured against it. */ +function makeLines(rand, count) { + const out = []; + for (let i = 0; i < count; i++) { + const a = WORDS[rand() % WORDS.length]; + const b = WORDS[rand() % WORDS.length]; + const c = WORDS[rand() % WORDS.length]; + out.push(` let ${a}_${i} = ${b}(${c}, ${rand() % 9973});`); + } + return out.join("\n") + "\n"; +} + +function run(cmd, args, opts = {}) { + const r = spawnSync(cmd, args, { stdio: "inherit", ...opts }); + if (r.status !== 0) { + throw new Error(`${cmd} ${args.join(" ")} exited ${r.status ?? r.signal}`); + } +} + +function capture(cmd, args, opts = {}) { + const r = spawnSync(cmd, args, { encoding: "utf8", ...opts }); + if (r.status !== 0) { + throw new Error(`${cmd} ${args.join(" ")} exited ${r.status ?? r.signal}: ${r.stderr}`); + } + return r.stdout.trim(); +} + +/** Create the repository and pin every config the numbers could otherwise + * depend on. Left at git's DEFAULTS on purpose: `core.untrackedCache` and + * `core.fsmonitor` both make `status` dramatically cheaper, and benchmarking + * with them on would publish a number almost nobody's repository produces. */ +function initRepo(dir) { + rmSync(dir, { recursive: true, force: true }); + mkdirSync(dir, { recursive: true }); + run("git", ["init", "--quiet", "--initial-branch=main", dir]); + const cfg = (k, v) => run("git", ["-C", dir, "config", k, v]); + cfg("user.name", "PlatypusGit Bench"); + cfg("user.email", "bench@platypusgit.invalid"); + cfg("commit.gpgsign", "false"); + cfg("tag.gpgsign", "false"); + cfg("core.autocrlf", "false"); + // No background repack mid-benchmark: a `gc --auto` firing during a timed run + // is a measurement of gc, attributed to whatever op happened to be running. + cfg("gc.auto", "0"); + cfg("gc.autoDetach", "false"); +} + +/** Feed a fast-import stream, respecting backpressure. The streams here reach + * tens of megabytes; writing them to a temp file first would double the io for + * no benefit, and buffering them in memory is how the wide fixture OOMs. */ +function fastImport(dir, produce) { + return new Promise((resolve, reject) => { + const child = spawn( + "git", + ["-C", dir, "fast-import", "--quiet", "--done"], + { stdio: ["pipe", "inherit", "inherit"] }, + ); + child.on("error", reject); + child.on("close", (code) => + code === 0 ? resolve() : reject(new Error(`fast-import exited ${code}`)), + ); + // `--done` above makes a truncated stream an ERROR rather than a silent + // partial import, which costs exactly this one line and is worth it: a + // fixture that quietly stopped at commit 31,000 would publish a number for + // a repository nobody can reproduce. + produce(child.stdin).then( + () => child.stdin.end("done\n"), + (e) => { + child.stdin.destroy(); + reject(e); + }, + ); + }); +} + +/** `write` that awaits drain. Without this the wide fixture's 50,000 blobs + * queue in Node's heap faster than git reads them. */ +function writer(stream) { + return (chunk) => + stream.write(chunk) ? Promise.resolve() : new Promise((r) => stream.once("drain", r)); +} + +function dataBlock(text) { + return `data ${Buffer.byteLength(text)}\n${text}\n`; +} + +// --------------------------------------------------------------------------- +// deep — 50,000 commits over 16 files. +// --------------------------------------------------------------------------- + +async function buildDeep(dir) { + const { commits, files } = SHAPES.deep; + initRepo(dir); + const rand = lcg(0x0dee9); + + await fastImport(dir, async (stdin) => { + const w = writer(stdin); + for (let i = 0; i < commits; i++) { + const path = `src/module_${String(i % files).padStart(2, "0")}.rs`; + const body = makeLines(rand, 40); + const when = EPOCH + i * 60; + await w(`blob\nmark :${i * 2 + 1}\n${dataBlock(body)}`); + const msg = + `feat(module ${i % files}): revision ${i}\n\n` + + `Generated fixture commit ${i} of ${commits}.\n`; + await w( + `commit refs/heads/main\nmark :${i * 2 + 2}\n` + + `author ${AUTHOR} ${when} +0000\n` + + `committer ${AUTHOR} ${when} +0000\n` + + dataBlock(msg) + + (i === 0 ? "" : `from :${i * 2}\n`) + + `M 100644 :${i * 2 + 1} ${path}\n\n`, + ); + } + }); + + run("git", ["-C", dir, "reset", "--hard", "main", "--quiet"]); + run("git", ["-C", dir, "repack", "-adq"]); +} + +// --------------------------------------------------------------------------- +// wide — 50,000 files, every one of them modified, plus 5,000 untracked. +// --------------------------------------------------------------------------- + +function widePath(i, dirs) { + const d = i % dirs; + return `pkg/${String(d).padStart(3, "0")}/gen_${String(i).padStart(6, "0")}.ts`; +} + +async function buildWide(dir) { + const { files, dirs, untracked } = SHAPES.wide; + initRepo(dir); + const rand = lcg(0x21de); + + await fastImport(dir, async (stdin) => { + const w = writer(stdin); + const marks = []; + for (let i = 0; i < files; i++) { + await w(`blob\nmark :${i + 1}\n${dataBlock(makeLines(rand, 12))}`); + marks.push(i + 1); + } + await w( + `commit refs/heads/main\nmark :${files + 1}\n` + + `author ${AUTHOR} ${EPOCH} +0000\n` + + `committer ${AUTHOR} ${EPOCH} +0000\n` + + dataBlock("chore: the generated tree\n"), + ); + for (let i = 0; i < files; i++) { + await w(`M 100644 :${marks[i]} ${widePath(i, dirs)}\n`); + } + await w("\n"); + }); + + run("git", ["-C", dir, "reset", "--hard", "main", "--quiet"]); + run("git", ["-C", dir, "repack", "-adq"]); + + // Now dirty it. Appending rather than rewriting so the modification is a real + // content change with an unchanged size prefix — the shape a formatter run or + // a codemod leaves, which is the case people describe when they say a client + // hangs on `status`. + for (let i = 0; i < files; i++) { + const p = join(dir, widePath(i, dirs)); + writeFileSync(p, readFileSync(p, "utf8") + "// touched by the fixture\n"); + } + for (let i = 0; i < untracked; i++) { + writeFileSync(join(dir, `pkg/${String(i % dirs).padStart(3, "0")}/new_${i}.ts`), "// new\n"); + } +} + +// --------------------------------------------------------------------------- +// refs — 5,000 branches and 2,000 tags over 2,000 commits. +// --------------------------------------------------------------------------- + +async function buildRefs(dir) { + const { commits, branches, lightweightTags, annotatedTags } = SHAPES.refs; + initRepo(dir); + const rand = lcg(0x4e75); + + await fastImport(dir, async (stdin) => { + const w = writer(stdin); + for (let i = 0; i < commits; i++) { + const when = EPOCH + i * 3600; + await w(`blob\nmark :${i * 2 + 1}\n${dataBlock(makeLines(rand, 8))}`); + await w( + `commit refs/heads/main\nmark :${i * 2 + 2}\n` + + `author ${AUTHOR} ${when} +0000\n` + + `committer ${AUTHOR} ${when} +0000\n` + + dataBlock(`chore: commit ${i}\n`) + + (i === 0 ? "" : `from :${i * 2}\n`) + + `M 100644 :${i * 2 + 1} src/file_${i % 32}.txt\n\n`, + ); + } + // Spread the refs over the whole history rather than piling them on the + // tip: a client that enumerates refs has to peel each one, and refs that + // all point at the same commit is the case where every cache hits. + const at = (n) => `:${(n % commits) * 2 + 2}`; + for (let i = 0; i < branches; i++) { + // Foldered names, because that is what a repository with 5,000 branches + // really looks like and it is what the branch tree has to group. + const name = `team-${i % 40}/feature/${String(i).padStart(5, "0")}`; + await w(`reset refs/heads/${name}\nfrom ${at(i * 7)}\n\n`); + } + for (let i = 0; i < lightweightTags; i++) { + await w(`reset refs/tags/build-${String(i).padStart(5, "0")}\nfrom ${at(i * 3)}\n\n`); + } + for (let i = 0; i < annotatedTags; i++) { + await w( + `tag v1.${i}.0\nfrom ${at(i * 5)}\n` + + `tagger ${AUTHOR} ${EPOCH + i * 7200} +0000\n` + + dataBlock(`Release v1.${i}.0\n`), + ); + } + }); + + run("git", ["-C", dir, "reset", "--hard", "main", "--quiet"]); + run("git", ["-C", dir, "repack", "-adq"]); +} + +// --------------------------------------------------------------------------- +// linux — the real thing. +// --------------------------------------------------------------------------- + +function buildLinux(dir) { + if (existsSync(join(dir, ".git"))) { + console.log(` linux: already present at ${dir}`); + return; + } + const tmp = `${dir}.partial`; + rmSync(tmp, { recursive: true, force: true }); + console.log(" linux: cloning torvalds/linux (several GB, this takes a while)…"); + // A full clone on purpose. `--filter=blob:none` would make every diff in the + // benchmark trigger a lazy fetch, so the numbers would be measuring the + // network. + run("git", ["clone", "--quiet", "https://github.com/torvalds/linux.git", tmp]); + run("git", ["-C", tmp, "config", "gc.auto", "0"]); + run("git", ["-C", tmp, "config", "gc.autoDetach", "false"]); + run("git", ["-C", tmp, "config", "user.name", "PlatypusGit Bench"]); + run("git", ["-C", tmp, "config", "user.email", "bench@platypusgit.invalid"]); + renameSync(tmp, dir); +} + +// --------------------------------------------------------------------------- + +const BUILDERS = { deep: buildDeep, wide: buildWide, refs: buildRefs }; + +/** The stamp lives BESIDE the repository, never inside it: `wide` is measured + * partly by how many untracked files `status` has to report, and a stray + * `fixture.json` in the work tree would be one of them. */ +export function stampPath(home, name) { + return join(home, `${name}.fixture.json`); +} + +/** What `bench.sh` compares against to decide whether a fixture is reusable. */ +function stamp(name, dir) { + return { + name, + version: FIXTURE_VERSION, + shape: SHAPES[name] ?? null, + head: capture("git", ["-C", dir, "rev-parse", "HEAD"]), + generated: new Date().toISOString(), + }; +} + +export function isFresh(name, home) { + const path = stampPath(home, name); + if (!existsSync(path)) return false; + try { + const have = JSON.parse(readFileSync(path, "utf8")); + return ( + have.version === FIXTURE_VERSION && + JSON.stringify(have.shape) === JSON.stringify(SHAPES[name] ?? null) + ); + } catch { + return false; + } +} + +async function main() { + const args = process.argv.slice(2); + const force = args.includes("--force"); + const wanted = args.filter((a) => !a.startsWith("--")); + const names = wanted.length ? wanted : Object.keys(BUILDERS); + const home = benchHome(); + mkdirSync(home, { recursive: true }); + + for (const name of names) { + const dir = join(home, name); + if (name === "linux") { + buildLinux(dir); + continue; + } + if (!BUILDERS[name]) throw new Error(`unknown fixture: ${name}`); + if (!force && isFresh(name, home)) { + console.log(` ${name}: up to date at ${dir}`); + continue; + } + const started = Date.now(); + console.log(` ${name}: generating…`); + await BUILDERS[name](dir); + writeFileSync(stampPath(home, name), JSON.stringify(stamp(name, dir), null, 2) + "\n"); + console.log(` ${name}: ready in ${((Date.now() - started) / 1000).toFixed(1)}s (${dir})`); + } +} + +if (import.meta.url === `file://${process.argv[1]}`) { + main().catch((e) => { + console.error(e.message); + process.exit(1); + }); +} diff --git a/scripts/bench-report.mjs b/scripts/bench-report.mjs new file mode 100755 index 00000000..dde5e57c --- /dev/null +++ b/scripts/bench-report.mjs @@ -0,0 +1,424 @@ +#!/usr/bin/env node +// Render benchmark runs into the two things that get committed (issue 257). +// +// `src-tauri/benches/repo_bench.rs` writes one JSON document per fixture, full +// of raw samples. This turns the set of them into: +// +// * `docs/dev/benchmark.json` — the published record, summary statistics only, +// read by `test/benchmark.test.ts`; +// * the table block inside `docs/dev/performance.md`, between its generated +// markers — the developer-facing record, with every operation, the `git` +// baseline beside it and the ratio between them. +// +// **Both are generated, and neither is hand-editable.** That is the whole point +// of the exercise: the site is supposed to print a number somebody measured, +// and the way a measured number turns back into an adjective is somebody +// nudging it in a hurry. +// +// The markdown is rendered from the PUBLISHED record rather than from the raw +// runs, and that indirection is the guard test's whole leverage: the test +// re-renders it from the committed JSON and compares it to the committed +// markdown, so the two can only agree if both came from one run. Rendering from +// the raw results — which are not committed — would leave the guard able to +// check only that the markdown parses. +// +// The raw sample arrays are deliberately NOT committed. They are what makes a +// result checkable, so they stay in `$PGBENCH_HOME/results/`, but a committed +// file that churns forty floating-point numbers per run is a file nobody reads +// the diff of. + +import { readFileSync, writeFileSync, existsSync } from "node:fs"; +import { join } from "node:path"; + +/** Operations in the order the tables print them: roughly the order a user + * meets them, opening a repository first and browsing the tree last, rather + * than the order the harness happens to measure. */ +export const OP_ORDER = [ + "open", + "open_screen", + "open_screen_ipc", + "status", + "log_first_page", + "log_page_deep", + "branches", + "tags", + "diff_commit", + "diff_workdir_file", + "file_history", + "list_all_files", +]; + +/** Title and one-line framing per fixture. Lives here rather than in the JSON + * because it is editorial — what the repository is FOR — while everything the + * harness writes is measured. */ +export const FIXTURES = { + linux: { + title: "torvalds/linux", + kind: "real", + blurb: + "A real clone of the Linux kernel: the repository people mean when they " + + "say a git client is slow.", + }, + deep: { + title: "deep", + kind: "generated", + blurb: + "50,000 commits over a small tree — the size past which GitKraken's own " + + "users report native clients beating it.", + }, + wide: { + title: "wide", + kind: "generated", + blurb: + "50,000 tracked files with every one of them modified, plus 5,000 " + + "untracked — a monorepo just after a codemod.", + }, + refs: { + title: "refs", + kind: "generated", + blurb: + "5,000 branches and 2,000 tags over a short history — a long-lived " + + "repository nobody prunes.", + }, +}; + +/** + * What each row is CALLED, as opposed to what it measured. + * + * Editorial, so it lives here beside `FIXTURES` rather than in the harness, for + * the same reason the fixture blurbs do: rewording a table heading should not + * mean recompiling a benchmark and re-running it against a six-gigabyte clone. + * The harness's own label is the fallback for an op this map has not met. + * + * `log_page_deep` is why the rule earned its keep. Its harness label named the + * commit range it reaches — which is a lie on any fixture with less history + * than that, and `wide` has exactly one commit. + */ +export const OP_LABELS = { + open: "Open the repository", + open_screen: "Everything the first screen needs, at once", + open_screen_ipc: "…including encoding it all for the webview", + status: "Working-tree status", + log_first_page: "First page of history", + log_page_deep: "Ten pages into history", + branches: "List every branch", + tags: "List every tag", + diff_commit: "Diff the selected commit", + diff_workdir_file: "Diff one modified file", + file_history: "History of one file", + list_all_files: "Browse the whole tree", +}; + +/** + * "1 commits" is the sort of thing that makes a published table look + * unproofread, and the counts come from the harness as free text. Fixing it + * where it is READ rather than where it is measured keeps the harness's job to + * measuring. + */ +export function tidyScale(scale) { + return String(scale).replace(/^1 ([a-z]+)$/, (whole, word) => { + if (word.endsWith("ies")) return `1 ${word.slice(0, -3)}y`; + if (/(ch|sh|x|s)es$/.test(word)) return `1 ${word.slice(0, -2)}`; + if (word.endsWith("s")) return `1 ${word.slice(0, -1)}`; + return whole; + }); +} + +export const BEGIN = + ""; +export const END = ""; + +/** Milliseconds, at a precision the measurement can actually support. + * + * Three significant figures at most, and seconds once past a thousand + * milliseconds: "5728.58 ms" implies a repeatability no wall-clock number on a + * general-purpose machine has, and reading it as 5.73 s is how anybody would + * say it out loud anyway. */ +export function fmtMs(v) { + if (v == null) return "—"; + if (v >= 1000) return `${(v / 1000).toFixed(2)} s`; + if (v >= 100) return `${v.toFixed(0)} ms`; + if (v >= 10) return `${v.toFixed(1)} ms`; + return `${v.toFixed(2)} ms`; +} + +export function fmtRatio(v) { + if (v == null) return "—"; + if (v >= 10) return `${v.toFixed(0)}×`; + if (v >= 1) return `${v.toFixed(1)}×`; + return `${v.toFixed(2)}×`; +} + +export function fmtMb(v) { + return v == null ? "—" : `${v.toFixed(0)} MB`; +} + +function thousands(n) { + return Number(n).toLocaleString("en-US"); +} + +/** One line describing the repository, for the table caption. Built from what + * the harness counted rather than from the fixture definition, so a fixture + * that generated wrong is visible in the published document. */ +export function describeRepo(r) { + const parts = [ + `${thousands(r.commits)} commits`, + `${thousands(r.trackedFiles)} tracked files`, + ]; + if (r.branches > 1) parts.push(`${thousands(r.branches)} branches`); + if (r.tags > 0) parts.push(`${thousands(r.tags)} tags`); + if (r.dirtyEntries > 0) parts.push(`${thousands(r.dirtyEntries)} changed entries`); + return parts.join(" · "); +} + +/** + * What `git` spent on the WORK, with process start-up taken back out. + * + * Every `git` invocation pays for a `fork`, an `exec`, the dynamic loader and + * git's own start-up before it reads a byte — over ten milliseconds on an + * M-series Mac, which is more than most of the operations here take in total. + * We pay none of it: the backend is libgit2, in process. + * + * So the ratio is against git's work, not its wall clock, and that choice is + * deliberately the one that makes US look worse. Comparing against the wall + * clock would hand us a ten-millisecond head start on every row and quietly + * turn a loss into a win on any operation git finishes quickly. + * + * Scaled by the invocation COUNT, because a baseline can be several processes — + * `status`'s is three (see the module doc on why), so it pays the floor three + * times. + */ +export function gitWorkMs(op, floorMs) { + if (op.gitMedianMs == null || !op.gitInvocations) return null; + return Math.max(0, op.gitMedianMs - floorMs * op.gitInvocations); +} + +/** + * Is what is left big enough to divide by? + * + * When start-up is most of the baseline, the remainder is the difference of two + * similar noisy numbers and a ratio against it is arithmetic, not measurement. + * A quarter is the line: below it the row prints † and no ratio, which is the + * honest answer for `status` on a clean tree, where git's three processes cost + * 40 ms and 37 ms of that is launching them. + */ +export function isFloorBound(op, floorMs) { + const work = gitWorkMs(op, floorMs); + if (work == null || op.gitMedianMs == null) return false; + return work <= 0 || work < op.gitMedianMs * 0.25; +} + +/** The ratio against git's work, or null when that work is unmeasurable. */ +export function ratioToGit(op, floorMs) { + if (op.repeatMedianMs == null || isFloorBound(op, floorMs)) return null; + const work = gitWorkMs(op, floorMs); + return work && work > 0 ? op.repeatMedianMs / work : null; +} + +function orderOps(ops) { + const rank = (op) => { + const i = OP_ORDER.indexOf(op); + return i === -1 ? OP_ORDER.length : i; + }; + return [...ops].sort((a, b) => rank(a.op) - rank(b.op)); +} + +export function renderTable(fixture) { + const head = [ + "Operation", + "Result size", + "First call", + "Repeat", + "p95", + "`git` work", + "vs `git`", + ]; + const rows = fixture.operations.map((o) => [ + o.label, + tidyScale(o.scale), + fmtMs(o.firstMs), + fmtMs(o.repeatMedianMs), + fmtMs(o.repeatP95Ms), + // git's WORK, not its wall clock — see `gitWorkMs`. The dagger marks a + // baseline where start-up swamped the work, so there was nothing left to + // compare; that row's ratio is deliberately absent rather than impressive. + o.gitFloorBound ? "†" : fmtMs(o.gitWorkMs), + fmtRatio(o.ratioToGit), + ]); + const line = (cells) => `| ${cells.join(" | ")} |`; + return [line(head), line(head.map(() => "---")), ...rows.map(line)].join("\n"); +} + +export function renderSoak(soak) { + if (!soak) return null; + const drift = + soak.firstHalfMedianMs && soak.secondHalfMedianMs + ? (soak.secondHalfMedianMs / soak.firstHalfMedianMs - 1) * 100 + : null; + return ( + `${thousands(soak.iterations)} first-screen fan-outs over ${soak.minutes} minutes. ` + + `Resident memory ${fmtMb(soak.rssStartMb)} → ${fmtMb(soak.rssEndMb)} ` + + `(peak ${fmtMb(soak.rssPeakMb)}). ` + + `Median fan-out ${fmtMs(soak.firstHalfMedianMs)} in the first half, ` + + `${fmtMs(soak.secondHalfMedianMs)} in the second` + + (drift == null ? "." : ` — ${drift >= 0 ? "+" : ""}${drift.toFixed(1)}%.`) + ); +} + +/** The table block, rendered from the published record. */ +export function renderMarkdown(data) { + const out = []; + const m = data.machine; + out.push( + `Measured on ${m.cpu} (${m.cores} cores, ${m.memoryGb} GB, ${m.os}) with ` + + `${m.gitVersion}, on ${data.measuredOn}. Up to ${data.iterations} repeats ` + + `per operation, time-boxed to ${data.budgetSeconds}s each — so a cheap ` + + `operation gets the full count and an expensive one gets at least three. ` + + `The published record records how many each row actually took.`, + ); + out.push(""); + out.push( + "The **`git` work** column is that baseline's wall clock with process " + + `start-up subtracted (${fmtMs(data.gitSpawnFloorMs)} per invocation on ` + + "this machine, measured), because we pay none of it — the backend is " + + "libgit2, in process. That is deliberately the comparison that makes us " + + "look worse: against git's wall clock we would get a ten-millisecond " + + "head start on every row. **†** marks a baseline where start-up swamped " + + "the work, leaving a remainder too small to divide by; those rows print " + + "no ratio rather than a flattering one.", + ); + out.push(""); + for (const fixture of data.fixtures) { + out.push(`### ${fixture.title}`); + out.push(""); + if (fixture.blurb) out.push(fixture.blurb); + out.push(""); + out.push(`*${describeRepo(fixture.repository)}*`); + out.push(""); + out.push(renderTable(fixture)); + const soak = renderSoak(fixture.soak); + if (soak) { + out.push(""); + out.push(`**Soak.** ${soak}`); + } + out.push(""); + } + return out.join("\n").trimEnd(); +} + +/** The published record. Summary statistics only — see the header note on why + * the samples stay out of the repository. + * + * It lives beside `performance.md` rather than under `site/` because nothing + * renders it yet: publishing these figures on the marketing site waits on the + * log-walk work, and a data file under `site/**` would redeploy the site on + * every re-measurement for no visible change. */ +export function buildPublishedRecord(runs) { + const m = runs[0].machine; + return { + measuredOn: new Date().toISOString().slice(0, 10), + machine: { + cpu: m.cpu, + cores: m.cores, + memoryGb: m.memoryGb, + os: `${m.os}/${m.arch}`, + gitVersion: m.git, + }, + iterations: runs[0].iterations, + budgetSeconds: runs[0].budgetSeconds, + /** What one `git` invocation costs before doing anything, measured. */ + gitSpawnFloorMs: runs[0].gitSpawnFloorMs, + fixtures: runs.map((run) => ({ + key: run.fixture, + title: FIXTURES[run.fixture]?.title ?? run.fixture, + kind: FIXTURES[run.fixture]?.kind ?? "generated", + blurb: FIXTURES[run.fixture]?.blurb ?? "", + repository: run.repository, + /** Measured per run, because it is a property of the machine at the time + * and not a constant. The note under the tables quotes the first. */ + gitSpawnFloorMs: run.gitSpawnFloorMs, + operations: orderOps(run.operations).map((o) => ({ + op: o.op, + label: OP_LABELS[o.op] ?? o.label, + scale: tidyScale(o.scale), + /** How many repeats this row actually took — see the time box. */ + samples: o.samples, + firstMs: o.firstMs, + repeatMedianMs: o.repeatMedianMs, + repeatP95Ms: o.repeatP95Ms, + gitCommand: o.gitCommand, + gitInvocations: o.gitInvocations, + /** The baseline's wall clock, start-up included. */ + gitMedianMs: o.gitMedianMs, + /** …and with start-up subtracted, which is what the ratio divides by. */ + gitWorkMs: gitWorkMs(o, run.gitSpawnFloorMs), + // Derived HERE rather than in the harness: whether a baseline is + // comparable is a judgement about what may be published, and it belongs + // with the thing that publishes it. + gitFloorBound: isFloorBound(o, run.gitSpawnFloorMs), + ratioToGit: ratioToGit(o, run.gitSpawnFloorMs), + })), + soak: run.soak ?? null, + })), + }; +} + +export function splice(doc, block) { + const a = doc.indexOf(BEGIN); + const b = doc.indexOf(END); + if (a === -1 || b === -1) { + throw new Error("docs/dev/performance.md is missing its generated markers"); + } + return doc.slice(0, a + BEGIN.length) + "\n\n" + block + "\n\n" + doc.slice(b); +} + +function arg(argv, name, fallback) { + const at = argv.indexOf(name); + return at === -1 ? fallback : argv[at + 1]; +} + +function main() { + const argv = process.argv.slice(2); + const results = arg(argv, "--results"); + const root = arg(argv, "--root", process.cwd()); + const only = arg(argv, "--fixtures", "").split(",").filter(Boolean); + const print = argv.includes("--print"); + + // Ordered so the real repository leads and the generated ones follow, which + // is the order a reader wants: the headline first, then the dimensions that + // explain it. + const order = ["linux", "deep", "wide", "refs"]; + const names = (only.length ? only : order).sort( + (a, b) => order.indexOf(a) - order.indexOf(b), + ); + + const runs = []; + for (const name of names) { + const path = join(results, `${name}.json`); + if (!existsSync(path)) { + console.error(` no results for ${name} (${path}) — skipping`); + continue; + } + runs.push(JSON.parse(readFileSync(path, "utf8"))); + } + if (!runs.length) throw new Error("no results to render"); + + const data = buildPublishedRecord(runs); + const markdown = renderMarkdown(data); + if (print) { + console.log(markdown); + return; + } + + const dataPath = join(root, "docs/dev/benchmark.json"); + writeFileSync(dataPath, JSON.stringify(data, null, 2) + "\n"); + console.log(` wrote ${dataPath}`); + + const docPath = join(root, "docs/dev/performance.md"); + writeFileSync(docPath, splice(readFileSync(docPath, "utf8"), markdown)); + console.log(` wrote ${docPath}`); +} + +if (import.meta.url === `file://${process.argv[1]}`) { + main(); +} diff --git a/scripts/bench.sh b/scripts/bench.sh new file mode 100755 index 00000000..12fec0e0 --- /dev/null +++ b/scripts/bench.sh @@ -0,0 +1,101 @@ +#!/usr/bin/env bash +# The large-repo benchmark, end to end (issue 257). +# +# pnpm bench # the three generated fixtures, published +# pnpm bench --linux # …and a real clone of torvalds/linux +# pnpm bench --fixture deep # one fixture only +# pnpm bench --soak 60 # add a 60-minute soak per fixture +# pnpm bench --no-publish # measure and print; write nothing +# +# Three steps, each of which can be run on its own: +# +# 1. `scripts/bench-fixtures.mjs` materialises the repositories under +# $PGBENCH_HOME (default ~/.cache/platypusgit-bench). Generated ones are +# reused when their recorded shape still matches; `--force` regenerates. +# 2. `src-tauri/benches/repo_bench.rs` measures one repository and writes a +# JSON document per fixture. It is behind `--features bench` so the Rust CI +# gate never builds it. +# 3. `scripts/bench-report.mjs` renders those documents into +# `docs/dev/performance.md` and `docs/dev/benchmark.json`, which is +# what `test/benchmark.test.ts` reads. +# +# Publishing is the default because a benchmark nobody publishes is a benchmark +# nobody runs twice. `--no-publish` is for the case you are iterating on the +# harness itself and do not want a dirty tree. +# +# Run it on a QUIET machine. Every number here is wall clock, so a compile in +# another window is measured as this program being slow — and the published +# figures are the ones a stranger will check. +set -euo pipefail + +root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +home="${PGBENCH_HOME:-$HOME/.cache/platypusgit-bench}" +results="$home/results" + +fixtures=(deep wide refs) +want_linux=0 +publish=1 +soak="" +iterations=10 +force="" + +while [ $# -gt 0 ]; do + case "$1" in + --linux) want_linux=1 ;; + --fixture) fixtures=("$2"); shift ;; + --soak) soak="$2"; shift ;; + --iterations) iterations="$2"; shift ;; + --force) force="--force" ;; + --no-publish) publish=0 ;; + -h|--help) sed -n '2,25p' "${BASH_SOURCE[0]}" | sed 's|^# \{0,1\}||'; exit 0 ;; + *) echo "unknown option: $1" >&2; exit 2 ;; + esac + shift +done + +[ "$want_linux" = 1 ] && fixtures+=(linux) + +# `cargo` and `node` are not on a non-interactive PATH on every machine this +# runs on; the repo's own toolchain locations are the fallback. +CARGO="${CARGO:-$(command -v cargo || echo "$HOME/.cargo/bin/cargo")}" + +echo "==> fixtures ($home)" +node "$root/scripts/bench-fixtures.mjs" $force "${fixtures[@]}" + +echo "==> building the harness" +"$CARGO" build --release --manifest-path "$root/src-tauri/Cargo.toml" \ + --features bench --bench repo_bench + +mkdir -p "$results" +for name in "${fixtures[@]}"; do + echo "==> $name" + soak_args=() + [ -n "$soak" ] && soak_args=(--soak-minutes "$soak") + # Via `cargo bench` rather than the built path: a `harness = false` bench has + # no stable file name (cargo appends a hash), and re-resolving it by globbing + # picks up stale binaries from an earlier build. + # + # `${a[@]+"${a[@]}"}` rather than `"${a[@]}"`: under `set -u`, bash 3.2 — which + # is what macOS ships — treats an EMPTY array's expansion as an unbound + # variable and aborts. Without a soak, that is every run. + "$CARGO" bench --quiet --manifest-path "$root/src-tauri/Cargo.toml" \ + --features bench --bench repo_bench -- \ + --repo "$home/$name" --name "$name" \ + --iterations "$iterations" \ + ${soak_args[@]+"${soak_args[@]}"} \ + --out "$results/$name.json" +done + +# Deliberately NOT `--fixtures "${fixtures[*]}"`. The renderer publishes every +# result it finds, so `pnpm bench --fixture deep` refreshes deep and leaves the +# other three standing. Narrowing it to this run's list would have silently +# deleted `torvalds/linux` from the published record the first time somebody +# re-measured one fixture — the results are a record, not a snapshot of the last +# command anyone happened to type. +if [ "$publish" = 1 ]; then + echo "==> publishing" + node "$root/scripts/bench-report.mjs" --results "$results" --root "$root" +else + echo "==> not publishing (--no-publish); raw results in $results" + node "$root/scripts/bench-report.mjs" --results "$results" --root "$root" --print +fi diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index 07e31e09..005bd556 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -77,6 +77,21 @@ libc = "0.2" [features] # Compile + wire the WebDriver plugin. Set only by the e2e debug build. e2e = ["dep:tauri-plugin-wdio-webdriver"] +# Compile the large-repo benchmark (#257). Adds no code to the crate itself — +# it exists only so the `[[bench]]` target below can be gated off, because the +# Rust CI gate is `cargo test` and a benchmark that links the whole lib would +# put its link time on every backend PR for a target nobody runs there. +bench = [] + +# `scripts/bench.sh` builds and runs this; `docs/dev/performance.md` explains +# what it measures. `harness = false` because it is a program with its own +# argument parsing rather than a set of `#[bench]` functions, and `test = false` +# so `cargo test` cannot decide to execute that program as a test binary. +[[bench]] +name = "repo_bench" +harness = false +test = false +required-features = ["bench"] [dev-dependencies] tempfile = "3" diff --git a/src-tauri/benches/repo_bench.rs b/src-tauri/benches/repo_bench.rs new file mode 100644 index 00000000..87dd39a9 --- /dev/null +++ b/src-tauri/benches/repo_bench.rs @@ -0,0 +1,1229 @@ +//! The large-repo benchmark (issue 257). +//! +//! "Slow on big repositories" is the one structural complaint every established +//! GUI client shares, and "fast" is one of this project's two strongest claims. +//! Until this file existed the claim was an adjective. What it produces is a +//! number per operation per fixture, in a JSON document `scripts/bench.sh` +//! renders into `docs/dev/performance.md` and into the figure the marketing site +//! prints — so a regression shows up as a smaller claim rather than as a +//! stranger's bug report. +//! +//! ## What it measures, and what it therefore does not +//! +//! It drives `Libgit2Backend` — the real backend, through the real `GitBackend` +//! trait, with no test doubles anywhere. That is the layer where a big +//! repository is actually expensive, and it is the layer a regression lands in. +//! +//! It stops at the IPC boundary. There is no webview here, so nothing below +//! measures React rendering, and the honest name for every number is "what the +//! backend costs", not "what the user waits". Two things narrow that gap on +//! purpose: +//! +//! * `open_screen` runs the ELEVEN reads `useRepoStore.refreshAll` issues, +//! simultaneously, from separate threads — the real fan-out, including +//! whatever the per-repository lock does to it. A composite is the only +//! number that can catch "a slow status blocks everything else on that +//! repo", which is the trap `git/repo_locks.rs` exists to avoid and the one +//! an op-at-a-time benchmark is structurally blind to. +//! * `open_screen_ipc` is the same fan-out plus `serde_json` encoding of every +//! payload, because that encoding is real work on a 500-commit page and it +//! happens before the frontend sees a byte. +//! +//! The render on top of those is bounded by the window, not by the repository: +//! the log is paged at 500, diff rows are windowed, and long lists are +//! virtualised. That is the argument for why the backend number is the +//! interesting one — it is not a proof, and `docs/dev/performance.md` says so. +//! +//! ## "first" and "repeat", not "cold" and "warm" +//! +//! `first` is one call on a freshly constructed backend and a freshly opened +//! repository, which is what happens when you open a repository in the app: +//! libgit2's object database, ref database and pack indices are all unbuilt. +//! `repeat` is the median of many calls on that same handle. +//! +//! Neither purges the operating system's file cache, and the words "cold" and +//! "warm" are avoided because they would imply it did. A first-boot number is +//! larger than anything here, by an amount that depends on the disk rather than +//! on this code. +//! +//! ## The `git` baseline +//! +//! Every op that a single `git` invocation can answer is also measured as that +//! invocation. The point is NOT to win: `git` is the floor, and a ratio near it +//! is the good outcome. The point is that a ratio is a comparison a reader can +//! check on their own machine, in a way that a bare millisecond figure from +//! somebody else's laptop is not — and a ratio that doubles is a regression +//! even on a machine that got faster. +//! +//! The baselines are chosen to ask the SAME question, not the cheapest one that +//! shares a name. `status` here is the clearest case: `GitBackend::status` +//! returns per-file added/removed counts, so its baseline is +//! `git status --porcelain` plus both `--numstat` diffs, and comparing it to a +//! bare `git status` would be comparing it to less work than it does. +//! +//! ## Why this lives in `benches/` and spawns processes directly +//! +//! `benches/` is outside the tree `tests/spawn_no_window.rs` guards, and the +//! `Command::new` calls below are deliberate rather than an oversight of that +//! rule: the rule exists so no SHIPPED spawn opens a console window on Windows, +//! and nothing in this file ships. It is behind `required-features = ["bench"]` +//! for the same reason — `cargo test` and `cargo check` must not pay to build +//! it. + +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::sync::{Arc, Barrier}; +use std::time::{Duration, Instant}; + +use platypusgit_lib::git::libgit2::Libgit2Backend; +use platypusgit_lib::git::types::{DiffKind, RepoId}; +use platypusgit_lib::git::GitBackend; + +/// The log page size the frontend asks for (`PAGE_SIZE` in `useRepoStore.ts`). +/// Hard-coded rather than shared because the two trees do not share constants — +/// if that one changes, this one has to change with it or the benchmark stops +/// measuring the thing the app does. +const PAGE_SIZE: usize = 500; + +/// How deep `log_page_deep` walks before timing a page. Ten pages in is past +/// anything a first screen touches, which is the point: the paged tail is where +/// a naive implementation re-walks history from the top every time. +const DEEP_PAGES: usize = 10; + +/// Diff context lines, matching the app's default. +const CONTEXT: u32 = 3; + +// --------------------------------------------------------------------------- +// Timing +// --------------------------------------------------------------------------- + +fn ms(d: Duration) -> f64 { + d.as_secs_f64() * 1000.0 +} + +fn time(f: impl FnOnce() -> T) -> (f64, T) { + let at = Instant::now(); + let out = f(); + (ms(at.elapsed()), out) +} + +/// Summary statistics over the repeat samples. +/// +/// Median rather than mean, and p95 alongside it, because the distribution is +/// not symmetric: one sample in a run lands on a page fault or a scheduler +/// hiccup and drags a mean somewhere the operation never actually was. The +/// samples are kept in the JSON too, so anyone who disagrees with the summary +/// can compute their own. +#[derive(Clone)] +struct Stats { + n: usize, + min: f64, + median: f64, + p95: f64, + max: f64, +} + +impl Stats { + fn of(samples: &[f64]) -> Option { + if samples.is_empty() { + return None; + } + let mut s = samples.to_vec(); + s.sort_by(|a, b| a.partial_cmp(b).unwrap()); + // Nearest-rank p95: with ten samples that is the worst one, which is + // the honest reading of "95th percentile of ten measurements". + let rank = ((0.95 * s.len() as f64).ceil() as usize).max(1) - 1; + Some(Stats { + n: s.len(), + min: s[0], + median: if s.len() % 2 == 1 { + s[s.len() / 2] + } else { + (s[s.len() / 2 - 1] + s[s.len() / 2]) / 2.0 + }, + p95: s[rank], + max: s[s.len() - 1], + }) + } +} + +/// One measured operation. +struct Measured { + /// Stable machine key. The doc renderer and the guard test both key on it. + key: &'static str, + /// What the operation is, in the words the published table uses. + label: &'static str, + /// One call on a fresh handle. `None` when the op has no meaningful + /// first-call form (the composites open their own handle). + first_ms: Option, + samples: Vec, + /// What came back — "55,000 entries", "500 commits". A timing with no size + /// beside it cannot be sanity-checked by a reader, and a benchmark that + /// silently started measuring an empty result is the classic way to publish + /// an excellent number for nothing at all. + scale: String, + /// The `git` invocation asking the same question, and what it cost. + baseline: Option, +} + +struct Baseline { + command: String, + /// How many processes one sample launched. A baseline can be several + /// invocations (see `status`), and each one pays the spawn floor, so the + /// count is what makes "is this comparable at all?" answerable downstream. + invocations: usize, + samples: Vec, +} + +// --------------------------------------------------------------------------- +// The git baseline +// --------------------------------------------------------------------------- + +/// Run one or more `git` invocations and return their combined wall time. +/// +/// A list rather than one command because some of our single calls genuinely +/// are several of git's — see the `status` note in the module doc. Output is +/// discarded to a null sink so the measurement is git's work rather than the +/// terminal's. +fn git_time(repo: &Path, invocations: &[&[&str]]) -> f64 { + let at = Instant::now(); + for args in invocations { + let status = Command::new("git") + .arg("-C") + .arg(repo) + .args(*args) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .status() + .expect("spawn git"); + assert!(status.success(), "git {args:?} failed in {}", repo.display()); + } + ms(at.elapsed()) +} + +/// What one `git` invocation costs before it has done anything. +/// +/// Every baseline below pays for a `fork`, an `exec`, the dynamic loader and +/// git's own start-up, and on this class of machine that is over ten +/// milliseconds — more than most of the operations being measured take in +/// total. Without this number the table would report that we open a repository +/// a hundred times faster than `git`, which is true and meaningless: what it +/// measures is process creation. +/// +/// `rev-parse --git-dir` is the cheapest invocation that still opens the +/// repository, so it is the floor rather than a lower bound nothing can reach. +/// `measured_json` drops the ratio for any baseline within twice it. +fn spawn_floor_ms(repo: &Path) -> f64 { + let once = || git_time(repo, &[&["rev-parse", "--git-dir"]]); + once(); + let mut samples: Vec = (0..10).map(|_| once()).collect(); + samples.sort_by(|a, b| a.partial_cmp(b).unwrap()); + samples[samples.len() / 2] +} + +fn render_command(invocations: &[&[&str]]) -> String { + invocations + .iter() + .map(|a| format!("git {}", a.join(" "))) + .collect::>() + .join(" && ") +} + +fn measure_baseline(repo: &Path, cfg: &Config, invocations: &[&[&str]]) -> Option { + if !cfg.baseline { + return None; + } + // One untimed run first: the first `git` of a run pays for loading the + // binary and its libraries, which is a cost the app never pays twice + // either. It doubles as the estimate the repeat count comes from. + let estimate = git_time(repo, invocations); + let samples = (0..repeats(estimate, cfg)) + .map(|_| git_time(repo, invocations)) + .collect(); + Some(Baseline { + command: render_command(invocations), + invocations: invocations.len(), + samples, + }) +} + +// --------------------------------------------------------------------------- +// The measured operations +// --------------------------------------------------------------------------- + +/// Everything the harness needs to know about the repository before it can +/// choose paths and revisions to measure against. Derived from the repository +/// rather than configured, so the same command works on any fixture. +struct Subject { + path: PathBuf, + head_oid: String, + /// The path changed most often in recent history. + /// + /// NOT simply "a path HEAD touched", which is the obvious choice and is + /// unusable: `file_history` filters a walk by path and stops at `limit` + /// matches, so on a rarely-touched file it never reaches 500 and walks the + /// WHOLE history with a tree comparison per commit. On `torvalds/linux` + /// that is 1.4 million commits and the benchmark simply does not finish. + /// + /// A hot file is also the honest subject: file history is a thing people + /// open on files that change. The pathological case is real and is written + /// up in `docs/dev/performance.md` rather than measured here, because + /// "unbounded" is not a number. + hot_path: Option, + /// A path the working tree has modified, if any. `None` on a clean fixture, + /// which simply skips the worktree diff. + dirty_path: Option, +} + +/// The path touched by the most of the last 2,000 commits. +/// +/// Asked of `git` rather than computed here: this is fixture selection, not a +/// measurement, and one `git log --name-only` is both faster and less code than +/// walking it ourselves. A window of 2,000 keeps it cheap even on a repository +/// with over a million commits, and any file that is hot in the last 2,000 is +/// hot enough to bound the walk being measured. +fn hottest_path(repo: &Path) -> Option { + let out = Command::new("git") + .arg("-C") + .arg(repo) + .args(["log", "--max-count=2000", "--name-only", "--format=", "HEAD"]) + .output() + .ok()?; + let text = String::from_utf8_lossy(&out.stdout); + let mut counts: HashMap<&str, usize> = HashMap::new(); + for line in text.lines().filter(|l| !l.trim().is_empty()) { + *counts.entry(line).or_default() += 1; + } + // Ties broken by name so the choice is deterministic across runs on the + // same repository — otherwise the fixture silently changes under the + // numbers. + counts + .into_iter() + .max_by(|a, b| a.1.cmp(&b.1).then_with(|| b.0.cmp(a.0))) + .map(|(p, _)| PathBuf::from(p)) +} + +fn probe(path: &Path) -> Subject { + let backend = Libgit2Backend::new(); + let handle = backend.open(path).expect("open the repository"); + let head = backend.head_info(&handle.id).expect("head_info"); + let head_oid = head.head_oid.clone().expect("the fixture has commits"); + + let hot_path = hottest_path(path).or_else(|| { + backend + .diff_commit(&handle.id, &head_oid, 0, false) + .ok() + .and_then(|diffs| diffs.into_iter().next()) + .map(|d| PathBuf::from(d.path)) + }); + + let dirty_path = backend.status(&handle.id).ok().and_then(|entries| { + entries + .into_iter() + .find(|e| matches!(e.worktree, platypusgit_lib::git::types::StatusFlag::Modified)) + .map(|e| PathBuf::from(e.path)) + }); + + Subject { + path: path.to_path_buf(), + head_oid, + hot_path, + dirty_path, + } +} + +struct Config { + /// The MOST repeats any operation takes. What it actually takes is + /// `repeats()`, below. + iterations: usize, + /// `spawn_floor_ms`, measured once per run. A baseline within twice it is + /// reported without a ratio. + floor_ms: f64, + warmup: usize, + baseline: bool, + /// Roughly how long one operation's repeats may take, in milliseconds. + budget_ms: f64, +} + +/// Never fewer than this many repeats: a median of two is the mean of two, and +/// a single sample is not a measurement at all. +const MIN_REPEATS: usize = 3; + +/// How many repeats to actually take, given what the first call cost. +/// +/// A fixed count is wrong at both ends. Ten repeats of a 0.15 ms operation is a +/// rounding error's worth of extra confidence; ten repeats of an eight-second +/// log page on a kernel clone is thirteen minutes for one row of one table, and +/// a benchmark nobody has time to finish produces no numbers at all. +/// +/// The budget is per operation, not per run, so the cheap operations keep the +/// sample count their variance actually needs. Each row records how many +/// samples it took, so the table never implies a confidence it does not have. +fn repeats(first_ms: f64, cfg: &Config) -> usize { + if first_ms <= 0.0 { + return cfg.iterations; + } + let affordable = (cfg.budget_ms / first_ms).floor() as usize; + affordable.clamp(MIN_REPEATS, cfg.iterations) +} + +/// Warm-up is skipped outright once one call would eat half the budget. Its +/// purpose is to settle caches that a slow operation has already settled by +/// being slow. +fn warmups(first_ms: f64, cfg: &Config) -> usize { + if first_ms * cfg.warmup as f64 > cfg.budget_ms / 2.0 { + 0 + } else { + cfg.warmup + } +} + +/// Measure one op: a first call on a brand-new backend, then `iterations` +/// repeats on a single handle. +/// +/// The fresh backend is what makes `first` mean anything — `Libgit2Backend` +/// caches a `Repository` per `RepoId`, so reusing one would measure libgit2's +/// caches rather than the work of building them. +fn measure( + key: &'static str, + label: &'static str, + subject: &Subject, + cfg: &Config, + scale: impl Fn(&T) -> String, + op: impl Fn(&Libgit2Backend, &RepoId) -> T, +) -> Measured { + // Progress on stderr, never on stdout: stdout is the JSON document when no + // `--out` is given. A run against a real kernel clone takes minutes, and a + // benchmark that spends them in silence is a benchmark people kill halfway + // and conclude is hung. + eprint!(" {key} … "); + let started = Instant::now(); + let first_ms = { + let backend = Libgit2Backend::new(); + let handle = backend.open(&subject.path).expect("open"); + let (t, _) = time(|| op(&backend, &handle.id)); + t + }; + + let backend = Libgit2Backend::new(); + let handle = backend.open(&subject.path).expect("open"); + for _ in 0..warmups(first_ms, cfg) { + op(&backend, &handle.id); + } + let n = repeats(first_ms, cfg); + let mut samples = Vec::with_capacity(n); + let mut last = None; + for _ in 0..n { + let (t, out) = time(|| op(&backend, &handle.id)); + samples.push(t); + last = Some(out); + } + + let scale = last.as_ref().map(scale).unwrap_or_default(); + eprintln!("{scale} in {:.1}s", started.elapsed().as_secs_f64()); + + Measured { + key, + label, + first_ms: Some(first_ms), + samples, + scale, + baseline: None, + } +} + +fn with_baseline(mut m: Measured, b: Option) -> Measured { + m.baseline = b; + m +} + +/// The eleven reads `refreshAll` issues, all at once. +/// +/// A `Barrier` rather than "spawn eleven threads and hope": spawning is slow +/// enough that the first read can finish before the last starts, which would +/// quietly turn the one measurement that exists to find lock contention into a +/// measurement with no contention in it. +fn open_screen(subject: &Subject, encode: bool) -> f64 { + let backend = Arc::new(Libgit2Backend::new()); + let handle = backend.open(&subject.path).expect("open"); + let id = handle.id.clone(); + + const N: usize = 11; + let gate = Arc::new(Barrier::new(N + 1)); + let mut threads = Vec::with_capacity(N); + for slot in 0..N { + let backend = Arc::clone(&backend); + let gate = Arc::clone(&gate); + let id = id.clone(); + threads.push(std::thread::spawn(move || { + gate.wait(); + // Each arm produces the JSON the command handler would return, so + // `encode` can charge for the same bytes the webview receives. + let json = match slot { + 0 => serde_json::to_string(&backend.status(&id).expect("status")), + 1 => serde_json::to_string(&backend.branches(&id).expect("branches")), + 2 => serde_json::to_string(&backend.tags(&id).expect("tags")), + 3 => serde_json::to_string(&backend.stashes(&id).expect("stashes")), + 4 => serde_json::to_string(&backend.remotes(&id).expect("remotes")), + 5 => serde_json::to_string( + &backend + .log_page(&id, None, None, PAGE_SIZE) + .expect("log_page"), + ), + 6 => serde_json::to_string(&backend.repo_state(&id).expect("repo_state")), + 7 => serde_json::to_string(&backend.rebase_status(&id).expect("rebase_status")), + 8 => serde_json::to_string(&backend.bisect_status(&id).expect("bisect_status")), + 9 => serde_json::to_string(&backend.head_info(&id).expect("head_info")), + _ => serde_json::to_string(&backend.shallow_info(&id).expect("shallow_info")), + }; + if encode { + // Touch the string so the encode cannot be optimised away. + std::hint::black_box(json.expect("encode").len()); + } + })); + } + + gate.wait(); + let at = Instant::now(); + for t in threads { + t.join().expect("a read panicked"); + } + ms(at.elapsed()) +} + +fn measure_open_screen(key: &'static str, label: &'static str, subject: &Subject, cfg: &Config, encode: bool) -> Measured { + eprint!(" {key} … "); + let started = Instant::now(); + let first_ms = open_screen(subject, encode); + for _ in 0..warmups(first_ms, cfg) { + open_screen(subject, encode); + } + let samples: Vec = (0..repeats(first_ms, cfg)) + .map(|_| open_screen(subject, encode)) + .collect(); + eprintln!("11 reads in {:.1}s", started.elapsed().as_secs_f64()); + Measured { + key, + label, + first_ms: Some(first_ms), + samples, + scale: "11 concurrent reads".to_string(), + baseline: None, + } +} + +fn thousands(n: usize) -> String { + let s = n.to_string(); + let mut out = String::new(); + for (i, c) in s.chars().enumerate() { + if i > 0 && (s.len() - i) % 3 == 0 { + out.push(','); + } + out.push(c); + } + out +} + +fn run_suite(subject: &Subject, cfg: &Config) -> Vec { + let repo = subject.path.as_path(); + let mut out = Vec::new(); + + // Opening the repository is its own cost, and on a repository with 5,000 + // refs it is not a rounding error. + // + // Measured on its own rather than through `measure`, because `open` is the + // one op for which "repeat it on an existing handle" is meaningless: every + // call has to build a new one, so `first` and `repeat` measure the same + // thing and the samples are simply that thing many times. + let open_first = { + let backend = Libgit2Backend::new(); + time(|| backend.open(&subject.path).expect("open")).0 + }; + let open_samples: Vec = (0..repeats(open_first, cfg)) + .map(|_| { + let backend = Libgit2Backend::new(); + time(|| backend.open(&subject.path).expect("open")).0 + }) + .collect(); + out.push(Measured { + key: "open", + label: "Open the repository", + first_ms: Some(open_first), + samples: open_samples, + scale: "a fresh handle".to_string(), + baseline: measure_baseline(repo, cfg, &[&["rev-parse", "HEAD"]]), + }); + + out.push(with_baseline( + measure( + "status", + "Working-tree status", + subject, + cfg, + |v: &Vec| { + format!("{} entries", thousands(v.len())) + }, + |b, id| b.status(id).expect("status"), + ), + // Three invocations because one `GitBackend::status` answers all three + // questions: what changed, and how many lines on each side. + measure_baseline( + repo, + cfg, + &[ + &["status", "--porcelain=v1", "--untracked-files=all"], + &["diff", "--numstat"], + &["diff", "--cached", "--numstat"], + ], + ), + )); + + out.push(with_baseline( + measure( + "log_first_page", + "First page of history (500 commits)", + subject, + cfg, + |p: &platypusgit_lib::git::types::LogPage| { + format!("{} commits", thousands(p.commits.len())) + }, + |b, id| b.log_page(id, None, None, PAGE_SIZE).expect("log_page"), + ), + // `--topo-order`, because `log_page` walks with + // `Sort::TIME | Sort::TOPOLOGICAL` and the commit graph's lanes depend + // on it — a plain `git log` is a strictly easier question and quoting + // it here would be the `status` mistake in the module doc, made the + // other way round. + // + // Measured, not assumed: on the `deep` fixture a default `git log -500` + // is 41 ms and `--topo-order` is 284 ms, against our 275 ms. The first + // page is at PARITY. Comparing against the 41 ms would have published a + // fourteen-fold regression that does not exist. + measure_baseline( + repo, + cfg, + &[&[ + "log", + "--topo-order", + "--max-count=500", + "--format=%H%n%an%n%ae%n%at%n%s", + ]], + ), + )); + + // The paged tail. `s.commits` is a prefix of history, so scrolling past the + // first screen is a real backend call, and it is the one that gets slower + // the further in you are if the walk is restarted each time. + out.push(with_baseline( + measure( + "log_page_deep", + "Page 10 of history (commits 4,501–5,000)", + subject, + cfg, + |p: &platypusgit_lib::git::types::LogPage| { + format!("{} commits", thousands(p.commits.len())) + }, + |b, id| { + let mut page = b.log_page(id, None, None, PAGE_SIZE).expect("log_page"); + for _ in 1..DEEP_PAGES { + // A fixture shallower than ten pages simply stops early and + // reports the last page it reached — the `scale` column says + // how many commits that was, so a short result is visible + // rather than passed off as a fast one. + let Some(cursor) = page.next_cursor.clone() else { break }; + page = b + .log_page(id, None, Some(&cursor), PAGE_SIZE) + .expect("log_page"); + } + page + }, + ), + // Same order, and `--skip` rather than ten invocations on purpose: git + // pays for the topological sort ONCE and then skips. That asymmetry is + // the point of this row — it is what turns "we are at parity on page + // one" into a number for what paging actually costs. + measure_baseline( + repo, + cfg, + &[&[ + "log", + "--topo-order", + "--skip=4500", + "--max-count=500", + "--format=%H%n%an%n%ae%n%at%n%s", + ]], + ), + )); + + out.push(with_baseline( + measure( + "branches", + "List every branch", + subject, + cfg, + |v: &Vec| { + format!("{} branches", thousands(v.len())) + }, + |b, id| b.branches(id).expect("branches"), + ), + measure_baseline( + repo, + cfg, + &[&[ + "for-each-ref", + "--format=%(refname)%(objectname)%(upstream)", + "refs/heads", + "refs/remotes", + ]], + ), + )); + + out.push(with_baseline( + measure( + "tags", + "List every tag", + subject, + cfg, + |v: &Vec| format!("{} tags", thousands(v.len())), + |b, id| b.tags(id).expect("tags"), + ), + measure_baseline( + repo, + cfg, + &[&[ + "for-each-ref", + "--format=%(refname)%(objectname)%(*objectname)", + "refs/tags", + ]], + ), + )); + + let head_oid = subject.head_oid.clone(); + out.push(with_baseline( + measure( + "diff_commit", + "Diff the selected commit", + subject, + cfg, + |v: &Vec| format!("{} files", thousands(v.len())), + move |b, id| b.diff_commit(id, &head_oid, CONTEXT, false).expect("diff_commit"), + ), + measure_baseline( + repo, + cfg, + &[&["show", "--format=", "--patch", "HEAD"]], + ), + )); + + if let Some(path) = subject.hot_path.clone() { + let for_history = path.clone(); + out.push(with_baseline( + measure( + "file_history", + "History of one file (500 commits)", + subject, + cfg, + |v: &Vec| { + format!("{} commits", thousands(v.len())) + }, + move |b, id| b.file_history(id, &for_history, PAGE_SIZE).expect("file_history"), + ), + // Deliberately NOT `--follow`. `Libgit2Backend::file_history` is a + // plain path filter over the walk — it does not detect renames — so + // a `--follow` baseline would be asking the harder question and + // flattering us with the difference. + measure_baseline( + repo, + cfg, + &[&[ + "log", + "--topo-order", + "--max-count=500", + "--format=%H%n%an%n%at%n%s", + "--", + path.to_str().expect("utf-8 fixture path"), + ]], + ), + )); + } + + if let Some(path) = subject.dirty_path.clone() { + out.push(with_baseline( + measure( + "diff_workdir_file", + "Diff one modified file", + subject, + cfg, + |d: &platypusgit_lib::git::types::FileDiff| format!("{} hunks", d.hunks.len()), + move |b, id| { + b.diff(id, &path, DiffKind::WorktreeToIndex, CONTEXT, false) + .expect("diff") + }, + ), + None, + )); + } + + out.push(with_baseline( + measure( + "list_all_files", + "Browse the whole tree", + subject, + cfg, + |v: &Vec| { + format!("{} files", thousands(v.len())) + }, + |b, id| b.list_all_files(id).expect("list_all_files"), + ), + measure_baseline( + repo, + cfg, + &[&["ls-files", "--cached", "--others", "--exclude-standard"]], + ), + )); + + out.push(measure_open_screen( + "open_screen", + "Everything the first screen needs, at once", + subject, + cfg, + false, + )); + out.push(measure_open_screen( + "open_screen_ipc", + "…including encoding it all for the webview", + subject, + cfg, + true, + )); + + out +} + +// --------------------------------------------------------------------------- +// The soak +// --------------------------------------------------------------------------- + +/// Resident set size of this process, in megabytes. +/// +/// `ps` rather than a crate: adding a dependency to the shipped manifest for a +/// benchmark would put it in the dependency tree the privacy guard reads, and +/// one subprocess every few seconds is beneath the noise of what is being +/// measured. +fn rss_mb() -> Option { + let out = Command::new("ps") + .args(["-o", "rss=", "-p", &std::process::id().to_string()]) + .output() + .ok()?; + let kb: f64 = String::from_utf8_lossy(&out.stdout).trim().parse().ok()?; + Some(kb / 1024.0) +} + +/// "Jank after a while" is the complaint nobody else answers, so it gets its own +/// mode: repeat the whole first-screen fan-out for `minutes` and report both +/// what memory did and whether the operation itself got slower. +/// +/// Reported as two halves rather than a slope, because a slope invites reading +/// a trend into noise. First half versus second half of the same run, on the +/// same machine, is a comparison that either shows something or does not. +fn soak(subject: &Subject, minutes: f64) -> JsonValue { + let until = Instant::now() + Duration::from_secs_f64(minutes * 60.0); + let start_rss = rss_mb(); + let mut peak = start_rss.unwrap_or(0.0); + let mut samples: Vec = Vec::new(); + + while Instant::now() < until { + samples.push(open_screen(subject, true)); + if let Some(r) = rss_mb() { + peak = peak.max(r); + } + } + + let end_rss = rss_mb(); + let half = samples.len() / 2; + let first_half = Stats::of(&samples[..half]); + let second_half = Stats::of(&samples[half..]); + + JsonValue::Ordered(vec![ + ("minutes".into(), json_num(minutes)), + ("iterations".into(), json_num(samples.len() as f64)), + ("rssStartMb".into(), start_rss.map(json_num).unwrap_or(json_null())), + ("rssEndMb".into(), end_rss.map(json_num).unwrap_or(json_null())), + ("rssPeakMb".into(), json_num(peak)), + ( + "firstHalfMedianMs".into(), + first_half.map(|s| json_num(s.median)).unwrap_or(json_null()), + ), + ( + "secondHalfMedianMs".into(), + second_half.map(|s| json_num(s.median)).unwrap_or(json_null()), + ), + ]) +} + +// --------------------------------------------------------------------------- +// JSON output +// +// Written by hand rather than with a `Serialize` derive so the key order in the +// published file is the order it is written here. A generated file that +// reorders itself between runs makes every diff of it unreadable, and this one +// is committed. +// --------------------------------------------------------------------------- + +enum JsonValue { + Null, + Num(f64), + Str(String), + Array(Vec), + /// Every object here is written in the order its fields should be READ, so + /// a map that sorts them is the wrong container. + Ordered(Vec<(String, JsonValue)>), +} + +fn json_num(v: f64) -> JsonValue { + // Three decimals is past the precision anything here is repeatable to, and + // it keeps the committed file from churning on the last bit. + JsonValue::Num((v * 1000.0).round() / 1000.0) +} +fn json_str(v: impl Into) -> JsonValue { + JsonValue::Str(v.into()) +} +fn json_null() -> JsonValue { + JsonValue::Null +} + +fn escape(s: &str) -> String { + let mut out = String::with_capacity(s.len() + 2); + for c in s.chars() { + match c { + '"' => out.push_str("\\\""), + '\\' => out.push_str("\\\\"), + '\n' => out.push_str("\\n"), + '\t' => out.push_str("\\t"), + c if (c as u32) < 0x20 => out.push_str(&format!("\\u{:04x}", c as u32)), + c => out.push(c), + } + } + out +} + +fn render(v: &JsonValue, indent: usize, out: &mut String) { + let pad = " ".repeat(indent); + let inner = " ".repeat(indent + 1); + match v { + JsonValue::Null => out.push_str("null"), + JsonValue::Num(n) => { + if n.fract() == 0.0 && n.abs() < 1e15 { + out.push_str(&format!("{}", *n as i64)); + } else { + out.push_str(&format!("{n}")); + } + } + JsonValue::Str(s) => out.push_str(&format!("\"{}\"", escape(s))), + JsonValue::Array(items) => { + if items.is_empty() { + out.push_str("[]"); + return; + } + out.push_str("[\n"); + for (i, item) in items.iter().enumerate() { + out.push_str(&inner); + render(item, indent + 1, out); + if i + 1 < items.len() { + out.push(','); + } + out.push('\n'); + } + out.push_str(&pad); + out.push(']'); + } + JsonValue::Ordered(pairs) => { + let pairs: Vec<(String, &JsonValue)> = + pairs.iter().map(|(k, v)| (k.clone(), v)).collect(); + render_pairs(&pairs, indent, out); + } + } +} + +fn render_pairs(pairs: &[(String, &JsonValue)], indent: usize, out: &mut String) { + let pad = " ".repeat(indent); + let inner = " ".repeat(indent + 1); + if pairs.is_empty() { + out.push_str("{}"); + return; + } + out.push_str("{\n"); + for (i, (k, v)) in pairs.iter().enumerate() { + out.push_str(&inner); + out.push_str(&format!("\"{}\": ", escape(k))); + render(v, indent + 1, out); + if i + 1 < pairs.len() { + out.push(','); + } + out.push('\n'); + } + out.push_str(&pad); + out.push('}'); +} + +fn json_render(v: &JsonValue) -> String { + let mut s = String::new(); + render(v, 0, &mut s); + s +} + +fn measured_json(m: &Measured) -> JsonValue { + let stats = Stats::of(&m.samples); + let mut pairs: Vec<(String, JsonValue)> = vec![ + ("op".into(), json_str(m.key)), + ("label".into(), json_str(m.label)), + ("scale".into(), json_str(&m.scale)), + ( + "firstMs".into(), + m.first_ms.map(json_num).unwrap_or(json_null()), + ), + ( + "repeatMedianMs".into(), + stats.as_ref().map(|s| json_num(s.median)).unwrap_or(json_null()), + ), + ( + "repeatMinMs".into(), + stats.as_ref().map(|s| json_num(s.min)).unwrap_or(json_null()), + ), + ( + "repeatP95Ms".into(), + stats.as_ref().map(|s| json_num(s.p95)).unwrap_or(json_null()), + ), + ( + "repeatMaxMs".into(), + stats.as_ref().map(|s| json_num(s.max)).unwrap_or(json_null()), + ), + ( + "samples".into(), + JsonValue::Num(stats.as_ref().map(|s| s.n as f64).unwrap_or(0.0)), + ), + ( + "sampleMs".into(), + JsonValue::Array(m.samples.iter().copied().map(json_num).collect()), + ), + ]; + // No ratio is computed here, on purpose. Whether a baseline is comparable + // at all depends on the spawn floor and on how many processes the baseline + // is, and that judgement belongs with the thing that PUBLISHES the number — + // `scripts/bench-report.mjs`. This file's job is to measure and to report + // what it measured, including the invocation count that makes the judgement + // possible. + match &m.baseline { + Some(b) => { + let bs = Stats::of(&b.samples); + pairs.push(("gitCommand".into(), json_str(&b.command))); + pairs.push(("gitInvocations".into(), json_num(b.invocations as f64))); + pairs.push(( + "gitMedianMs".into(), + bs.as_ref().map(|s| json_num(s.median)).unwrap_or(json_null()), + )); + } + None => { + pairs.push(("gitCommand".into(), json_null())); + pairs.push(("gitInvocations".into(), json_num(0.0))); + pairs.push(("gitMedianMs".into(), json_null())); + } + } + JsonValue::Ordered(pairs) +} + +// --------------------------------------------------------------------------- +// Machine description +// --------------------------------------------------------------------------- + +fn sysctl(key: &str) -> Option { + let out = Command::new("sysctl").args(["-n", key]).output().ok()?; + let s = String::from_utf8_lossy(&out.stdout).trim().to_string(); + (!s.is_empty()).then_some(s) +} + +fn cpu_model() -> String { + if let Some(v) = sysctl("machdep.cpu.brand_string") { + return v; + } + if let Ok(info) = std::fs::read_to_string("/proc/cpuinfo") { + for line in info.lines() { + if let Some(v) = line.strip_prefix("model name") { + return v.trim_start_matches([' ', ':']).trim().to_string(); + } + } + } + "unknown".into() +} + +fn memory_gb() -> Option { + if let Some(v) = sysctl("hw.memsize").and_then(|s| s.parse::().ok()) { + return Some(v / 1024.0 / 1024.0 / 1024.0); + } + let info = std::fs::read_to_string("/proc/meminfo").ok()?; + let line = info.lines().find(|l| l.starts_with("MemTotal:"))?; + let kb: f64 = line.split_whitespace().nth(1)?.parse().ok()?; + Some(kb / 1024.0 / 1024.0) +} + +fn git_version() -> String { + Command::new("git") + .arg("--version") + .output() + .ok() + .map(|o| String::from_utf8_lossy(&o.stdout).trim().to_string()) + .unwrap_or_else(|| "unknown".into()) +} + +fn machine_json() -> JsonValue { + JsonValue::Ordered(vec![ + ("os".into(), json_str(std::env::consts::OS)), + ("arch".into(), json_str(std::env::consts::ARCH)), + ("cpu".into(), json_str(cpu_model())), + ( + "cores".into(), + json_num( + std::thread::available_parallelism() + .map(|n| n.get() as f64) + .unwrap_or(0.0), + ), + ), + ( + "memoryGb".into(), + memory_gb().map(|g| json_num(g.round())).unwrap_or(json_null()), + ), + ("git".into(), json_str(git_version())), + ]) +} + +// --------------------------------------------------------------------------- +// Repository description +// --------------------------------------------------------------------------- + +fn count(repo: &Path, args: &[&str]) -> usize { + Command::new("git") + .arg("-C") + .arg(repo) + .args(args) + .output() + .ok() + .map(|o| String::from_utf8_lossy(&o.stdout).lines().count()) + .unwrap_or(0) +} + +fn repo_json(name: &str, subject: &Subject) -> JsonValue { + let repo = subject.path.as_path(); + let commits = Command::new("git") + .arg("-C") + .arg(repo) + .args(["rev-list", "--count", "HEAD"]) + .output() + .ok() + .and_then(|o| String::from_utf8_lossy(&o.stdout).trim().parse::().ok()) + .unwrap_or(0.0); + JsonValue::Ordered(vec![ + ("fixture".into(), json_str(name)), + ("commits".into(), json_num(commits)), + ( + "trackedFiles".into(), + json_num(count(repo, &["ls-files"]) as f64), + ), + ( + "branches".into(), + json_num(count(repo, &["for-each-ref", "--format=%(refname)", "refs/heads"]) as f64), + ), + ( + "tags".into(), + json_num(count(repo, &["for-each-ref", "--format=%(refname)", "refs/tags"]) as f64), + ), + ( + "dirtyEntries".into(), + json_num(count(repo, &["status", "--porcelain=v1", "--untracked-files=all"]) as f64), + ), + ]) +} + +// --------------------------------------------------------------------------- +// Entry point +// --------------------------------------------------------------------------- + +fn arg(args: &[String], name: &str) -> Option { + let at = args.iter().position(|a| a == name)?; + args.get(at + 1).cloned() +} + +fn usage() -> ! { + eprintln!( + "usage: repo_bench --repo --name \ + [--iterations N] [--warmup N] [--budget-seconds S] [--no-baseline] \ + [--soak-minutes M] [--out ]" + ); + std::process::exit(2) +} + +fn main() { + let args: Vec = std::env::args().collect(); + let Some(repo) = arg(&args, "--repo") else { usage() }; + let name = arg(&args, "--name").unwrap_or_else(|| "repo".into()); + let mut cfg = Config { + floor_ms: 0.0, + iterations: arg(&args, "--iterations") + .and_then(|v| v.parse().ok()) + .unwrap_or(10), + warmup: arg(&args, "--warmup").and_then(|v| v.parse().ok()).unwrap_or(2), + baseline: !args.iter().any(|a| a == "--no-baseline"), + budget_ms: arg(&args, "--budget-seconds") + .and_then(|v| v.parse::().ok()) + .unwrap_or(20.0) + * 1000.0, + }; + + let path = PathBuf::from(&repo); + if !path.join(".git").exists() { + eprintln!("not a git repository: {repo}"); + std::process::exit(1); + } + + eprintln!("benchmarking {name} at {repo}"); + cfg.floor_ms = spawn_floor_ms(&path); + eprintln!(" one `git` invocation costs {:.1} ms before doing anything", cfg.floor_ms); + let subject = probe(&path); + let measured = run_suite(&subject, &cfg); + + let soaked = arg(&args, "--soak-minutes") + .and_then(|v| v.parse::().ok()) + .map(|m| { + eprintln!(" soaking for {m} minute(s)…"); + soak(&subject, m) + }); + + let doc = JsonValue::Ordered(vec![ + ("fixture".into(), json_str(&name)), + ("repository".into(), repo_json(&name, &subject)), + ("machine".into(), machine_json()), + ("iterations".into(), json_num(cfg.iterations as f64)), + ("warmup".into(), json_num(cfg.warmup as f64)), + ("budgetSeconds".into(), json_num(cfg.budget_ms / 1000.0)), + ("gitSpawnFloorMs".into(), json_num(cfg.floor_ms)), + ( + "operations".into(), + JsonValue::Array( + measured + .iter() + .map(measured_json) + .collect(), + ), + ), + ("soak".into(), soaked.unwrap_or_else(json_null)), + ]); + + let rendered = json_render(&doc); + match arg(&args, "--out") { + Some(out) => { + std::fs::write(&out, format!("{rendered}\n")).expect("write results"); + eprintln!(" wrote {out}"); + } + None => println!("{rendered}"), + } +} diff --git a/test/benchmark.test.ts b/test/benchmark.test.ts new file mode 100644 index 00000000..a1b760dc --- /dev/null +++ b/test/benchmark.test.ts @@ -0,0 +1,249 @@ +/** + * @vitest-environment node + */ +// The published performance numbers are measured numbers (#257). +// +// The issue this whole benchmark answers is that "fast" was an adjective. The +// way an adjective grows back is not malice — it is somebody with a release to +// cut, a figure that reads badly, and a two-character edit to a JSON file that +// nobody diffs. So the two committed artifacts are generated from one run and +// this file is what makes that checkable: +// +// * `docs/dev/benchmark.json` is the published record, written by +// `scripts/bench-report.mjs`; +// * the table block in `docs/dev/performance.md` is RENDERED from that JSON by +// the same module. +// +// Re-rendering here and comparing byte for byte means the two can only agree if +// both came out of one `pnpm bench`. Edit either by hand and this fails, naming +// the command that fixes it. +// +// It cannot check that the numbers are TRUE — nothing short of re-running the +// benchmark could, and that takes minutes and a quiet machine. What it checks is +// that they are *consistent* and that they *cover* what the document claims to +// cover. That is the realistic accident. +// +// Every input it reads (`test/`, `scripts/`, `docs/dev/`) is already in the `js` +// filter in `tests.yml`, and the last assertion in this file is that `docs/dev/` +// really is — because a guard skippable by exactly the change it polices is the +// #210 failure mode, and it has shipped here once. + +import { readFileSync } from "node:fs"; +import { resolve } from "node:path"; +import { describe, expect, it } from "vitest"; + +import { + BEGIN, + END, + OP_ORDER, + renderMarkdown, + // @ts-expect-error — plain .mjs with JSDoc types, no .d.ts +} from "../scripts/bench-report.mjs"; + +const root = (rel: string) => resolve(process.cwd(), rel); +const read = (rel: string) => readFileSync(root(rel), "utf8"); + +const DATA_PATH = "docs/dev/benchmark.json"; +const DOC_PATH = "docs/dev/performance.md"; + +type Operation = { + op: string; + label: string; + scale: string; + samples: number; + firstMs: number | null; + repeatMedianMs: number | null; + repeatP95Ms: number | null; + gitCommand: string | null; + gitInvocations: number; + gitMedianMs: number | null; + gitWorkMs: number | null; + gitFloorBound: boolean; + ratioToGit: number | null; +}; + +type Fixture = { + key: string; + title: string; + kind: string; + blurb: string; + repository: { + fixture: string; + commits: number; + trackedFiles: number; + branches: number; + tags: number; + dirtyEntries: number; + }; + operations: Operation[]; +}; + +type Published = { + measuredOn: string; + machine: { + cpu: string; + cores: number; + memoryGb: number; + os: string; + gitVersion: string; + }; + iterations: number; + budgetSeconds: number; + gitSpawnFloorMs: number; + fixtures: Fixture[]; +}; + +const data: Published = JSON.parse(read(DATA_PATH)); +const doc = read(DOC_PATH); + +/** The operations every fixture must carry. Not the whole of `OP_ORDER`: two + * entries are conditional by design — `diff_workdir_file` needs a dirty tree + * and `file_history` needs HEAD to have touched a file — and a fixture that + * legitimately lacks one must not be made to fake it. These four are the ones + * the published claims rest on. */ +const REQUIRED_OPS = ["open", "open_screen", "status", "log_first_page"]; + +describe("the published benchmark numbers", () => { + it("renders exactly the table block that is committed in the doc", () => { + const a = doc.indexOf(BEGIN); + const b = doc.indexOf(END); + expect(a, `${DOC_PATH} is missing its BEGIN marker`).toBeGreaterThan(-1); + expect(b, `${DOC_PATH} is missing its END marker`).toBeGreaterThan(a); + + const committed = doc.slice(a + BEGIN.length, b).trim(); + expect( + committed, + `${DOC_PATH} and ${DATA_PATH} disagree. Neither is edited by hand — ` + + "re-run `pnpm bench` (or `pnpm bench --linux`) and commit both.", + ).toBe(renderMarkdown(data).trim()); + }); + + it("names the machine and the day it was measured", () => { + // A performance number with no machine beside it is not a measurement, it + // is a boast, and it is the first thing a reader checks. + expect(data.measuredOn).toMatch(/^\d{4}-\d{2}-\d{2}$/); + expect(data.machine.cpu.length).toBeGreaterThan(3); + expect(data.machine.gitVersion).toMatch(/^git version /); + expect(data.machine.cores).toBeGreaterThan(0); + expect(data.iterations).toBeGreaterThan(1); + expect(data.budgetSeconds).toBeGreaterThan(0); + }); + + it("measured something on every fixture", () => { + expect(data.fixtures.length).toBeGreaterThan(0); + for (const fixture of data.fixtures) { + const ops = new Set(fixture.operations.map((o) => o.op)); + const missing = REQUIRED_OPS.filter((op) => !ops.has(op)); + expect( + missing, + `${fixture.key} published no ${missing.join(", ")}. A run that measured ` + + "nothing publishes excellent numbers for nothing at all.", + ).toEqual([]); + + // The scale column is what lets a reader sanity-check a timing. A blank + // one, or a timing with no positive duration, means the harness measured + // an empty result and reported it as fast. + for (const op of fixture.operations) { + expect(op.scale, `${fixture.key}/${op.op} has no result size`).not.toBe(""); + expect( + op.repeatMedianMs, + `${fixture.key}/${op.op} has no repeat median`, + ).not.toBeNull(); + expect(op.repeatMedianMs!).toBeGreaterThan(0); + + // The time box can cut an expensive operation down to three repeats, + // and three is the floor on purpose: a median of two is the mean of + // two. A row below it means the box was misconfigured, not that the + // operation was fast. + expect( + op.samples, + `${fixture.key}/${op.op} published a median over ${op.samples} samples`, + ).toBeGreaterThanOrEqual(3); + } + } + }); + + it("publishes only operations the report knows how to order", () => { + // An op the renderer does not know about sorts to the end silently, which + // is how a new measurement gets added and then quietly read as the least + // important thing in the table. + const unknown = [ + ...new Set(data.fixtures.flatMap((f) => f.operations.map((o) => o.op))), + ].filter((op) => !OP_ORDER.includes(op)); + expect( + unknown, + `Not in OP_ORDER in scripts/bench-report.mjs: ${unknown.join(", ")}`, + ).toEqual([]); + }); + + it("prints no ratio when process start-up swamped the baseline", () => { + // The floor is real and large — over ten milliseconds on an M-series Mac, + // which is more than most of these operations take in total. A ratio + // against a baseline that is mostly `fork` would put a hundred-fold win in + // the table for an operation on an empty result: an adjective wearing a + // number. + expect(data.gitSpawnFloorMs).toBeGreaterThan(0); + for (const fixture of data.fixtures) { + for (const op of fixture.operations) { + if (!op.gitFloorBound) continue; + expect( + op.ratioToGit, + `${fixture.key}/${op.op} prints a ratio against a baseline at the ` + + "`git` start-up floor", + ).toBeNull(); + } + } + }); + + it("divides by git's work, never by its wall clock", () => { + // The ratio must be against the baseline MINUS process start-up, which is + // the harsher comparison — we pay no start-up, so dividing by git's wall + // clock would hand us a free head start on every row. + for (const fixture of data.fixtures) { + for (const op of fixture.operations) { + if (op.ratioToGit == null) continue; + expect(op.gitWorkMs, `${fixture.key}/${op.op}`).not.toBeNull(); + expect(op.gitWorkMs!).toBeLessThanOrEqual(op.gitMedianMs!); + expect( + op.ratioToGit, + `${fixture.key}/${op.op} did not divide by gitWorkMs`, + ).toBeCloseTo(op.repeatMedianMs! / op.gitWorkMs!, 5); + } + } + }); + + it("records the git command behind every ratio it prints", () => { + // A "3× git" with no command beside it is an argument nobody can check. + for (const fixture of data.fixtures) { + for (const op of fixture.operations) { + if (op.ratioToGit == null) continue; + expect( + op.gitCommand, + `${fixture.key}/${op.op} prints a ratio with no baseline command`, + ).toMatch(/^git /); + } + } + }); + + it("is reachable from the docs the assistant reads", () => { + // `docs.test.ts` pins the module and command lists; this pins the one + // pointer that makes the benchmark findable at all. + expect(read("CLAUDE.md")).toContain("docs/dev/performance.md"); + }); +}); + +describe("CI runs this guard when its inputs change", () => { + it("has docs/dev/ in the js path filter", () => { + // The #210 failure mode: a guard lands, its input is not in the filter, and + // the next PR touching only that input runs no suite at all and reports + // green. Both of this file's inputs — the document and the record beside it + // — are under `docs/dev/`, which is why the record lives there and not + // somewhere that would need a filter entry of its own. + const workflow = read(".github/workflows/tests.yml"); + expect( + workflow, + "`docs/dev/` left the `js` filter in tests.yml, so a benchmark-only " + + "commit now skips the guard that holds its numbers together.", + ).toContain("docs/dev/"); + }); +});