diff --git a/benchmarkFixture.js b/benchmarkFixture.js index 859eb45..ca49dec 100644 --- a/benchmarkFixture.js +++ b/benchmarkFixture.js @@ -171,18 +171,31 @@ async function writeRegistryConfig (pm, cwd, opts) { * directory the benchmark doesn't control. When the scenario offloads * resolution, `pnprServer` names the server it happens on. * - * `minimumReleaseAge` is deliberately left at pnpm's default here, unlike - * the directories the managers are installed into. The hold — and the - * lockfile verification pass it drives — is pnpm's default behavior, so it - * belongs in the measurement; the client sends the setting along with its - * resolve and verify-lockfile requests, so the server applies the same - * cutoff and both pnpm columns still install the same graph. Setting it to - * zero would leave no verifier configured at all, silently deleting the - * verification work whose offload to the server the accelerated column - * exists to measure. + * `minimumReleaseAge` is deliberately left at pnpm's default, unlike the + * directories the managers are installed into. The hold is pnpm's default + * behavior and it shapes what resolution costs, which is the thing the rows + * without a lockfile measure; the client sends the setting along with its + * resolve request, so the server applies the same cutoff and both pnpm + * columns still install the same graph. + * + * `trustLockfile` turns off the other half of that policy: the supply-chain + * pass that re-checks every entry of a lockfile it is about to install. That + * pass is not free and it is not comparable — it fetches a packument per + * package, measured at 1158 requests on top of the 1346 tarball fetches this + * fixture needs, and none of npm, Yarn or Bun does anything of the sort, so + * the rows with an up-to-date lockfile were timing pnpm doing work no other + * column was asked to do. The rows say `trusted lockfile` for that reason. + * pnpm 11 honors the setting too, so all three pnpm columns skip the same + * pass and the version comparison stays a comparison. + * + * It costs the accelerated column its lockfile rows, which is the trade + * being made knowingly: pnpr answers that pass in one request where the + * client spends a round trip per package, and with nothing to verify the two + * pnpm 12 columns now really do measure the same install there — which is + * what the page has always claimed of them. */ export function pnpmWorkspaceYaml (opts = {}) { - let yaml = "packages:\n - '.'\n" + let yaml = "packages:\n - '.'\ntrustLockfile: true\n" if (opts.pnprServer) { yaml += `pnprServer: ${opts.pnprServer}\n` } diff --git a/index.js b/index.js index d621fb0..ed73954 100644 --- a/index.js +++ b/index.js @@ -89,12 +89,12 @@ const tests = [ const testDescriptions = { firstInstall: ['clean'], withWarmModules: ['node_modules'], - withLockfile: ['lockfile'], + withLockfile: ['trusted lockfile'], withWarmCacheAndModules: ['cache', 'node_modules'], withWarmCache: ['cache'], - withWarmCacheAndLockfile: ['cache', 'lockfile'], - withWarmModulesAndLockfile: ['lockfile', 'node_modules'], - repeatInstall: ['cache', 'lockfile', 'node_modules'], + withWarmCacheAndLockfile: ['cache', 'trusted lockfile'], + withWarmModulesAndLockfile: ['trusted lockfile', 'node_modules'], + repeatInstall: ['cache', 'trusted lockfile', 'node_modules'], updatedDependencies: ['update'], } @@ -113,12 +113,12 @@ const tableRows = [ const explanationByTest = { firstInstall: '`clean`: a brand-new clone — nothing cached, no lockfile, no `node_modules`.', withWarmModules: '`node_modules`: the cache and lockfile are deleted and install is run again.', - withLockfile: '`lockfile`: a CI server doing its first install.', + withLockfile: '`trusted lockfile`: a CI server doing its first install.', withWarmCacheAndModules: '`cache+node_modules`: the lockfile is deleted and install is run again.', withWarmCache: '`cache`: a developer reinstalling without a lockfile.', - withWarmCacheAndLockfile: '`cache+lockfile`: a developer reinstalling a known project.', - withWarmModulesAndLockfile: '`lockfile+node_modules`: the cache is deleted and install is run again.', - repeatInstall: '`cache+lockfile+node_modules`: re-running install when nothing has changed.', + withWarmCacheAndLockfile: '`cache+trusted lockfile`: a developer reinstalling a known project.', + withWarmModulesAndLockfile: '`trusted lockfile+node_modules`: the cache is deleted and install is run again.', + repeatInstall: '`cache+trusted lockfile+node_modules`: re-running install when nothing has changed.', updatedDependencies: '`update`: dependency versions are bumped in `package.json` and install is run again.', } @@ -383,13 +383,14 @@ async function writePage ({ formattedNow, registryVersion, sections, svgs, sorte - **Every manager crosses the same link.** The round trip is applied to all of them, and to pnpm's resolution requests as well, so no client gets a cheaper connection than another. The bandwidth cap is the link's, shared across all of a manager's connections — opening more connections in parallel spreads the latency, as on a real network, but cannot multiply the ${BANDWIDTH_MBPS} Mbit/s. - **pnpr's cache is warmed before anything is timed** — with the fixture's dependency graph and with the one the update row installs — so no manager pays to pull either into the registry on behalf of the ones measured after it. - **Server-side resolution pays off when there is a graph to resolve.** Resolving one means walking it level by level, and each level costs a round trip, so the cost is roughly the depth of the graph times the latency. pnpr does that walk next to the registry — its own metadata access stays on loopback, the co-located shape the [pnpm monorepo's integrated benchmark](https://github.com/pnpm/pnpm) measures — and answers with the whole resolved lockfile at once, which is why the rows without a lockfile, and the row that changes dependencies, are the ones where it pulls ahead of plain pnpm. + - **The lockfile is trusted, so every manager is asked for the same work.** pnpm verifies a lockfile against the registry before installing it — a supply-chain pass that costs a packument per package, and one no other manager here performs. The rows with a lockfile run every pnpm column with [\`trustLockfile\`](https://pnpm.io/settings#trustlockfile), so what they compare is the install rather than a safety check only one participant was asked for. It is on by default outside this benchmark, and pnpm's own resolution still applies its release-age policy on the rows that resolve. - **With an up-to-date lockfile there is nothing to resolve.** pnpm doesn't ask the server then, so those rows measure the same install in both pnpm 12 columns. - Tarballs are still fetched by the client, in parallel and directly, on every row. ` const explanationItems = sortedTests.map(t => `- ${explanationByTest[t]}`).join('\n ') const explanation = stripIndents` - Each row's label lists which of \`cache\`, \`lockfile\`, and \`node_modules\` are warm/present before install runs. Quick mapping to the real world (ordered from slowest to fastest scenario): + Each row's label lists which of \`cache\`, \`trusted lockfile\`, and \`node_modules\` are warm/present before install runs. Quick mapping to the real world (ordered from slowest to fastest scenario): ${explanationItems} ` @@ -470,7 +471,7 @@ async function benchmarkFixtures ({ pmCommands, formattedNow, nodeVersion, runFi sections.push(stripIndents` ${fixture.mdDesc} - | action | cache | lockfile | node_modules| ${headerLegends} | + | action | cache | trusted lockfile | node_modules| ${headerLegends} | | --- | --- | --- | --- | ${headerSep} | ${rows} @@ -516,7 +517,7 @@ async function benchmarkFixtures ({ pmCommands, formattedNow, nodeVersion, runFi pnpm v12 will use a new installation engine for fetching and linking written in Rust. See [pacquet](https://github.com/pnpm/pacquet). - | action | cache | lockfile | node_modules| ${pnpmHeaderLegends} | + | action | cache | trusted lockfile | node_modules| ${pnpmHeaderLegends} | | --- | --- | --- | --- | ${pnpmHeaderSep} | ${pnpmRows} diff --git a/regenerate-svgs.mjs b/regenerate-svgs.mjs index 1a170d2..1c648e1 100644 --- a/regenerate-svgs.mjs +++ b/regenerate-svgs.mjs @@ -39,12 +39,12 @@ const tests = [ const testDescriptions = { firstInstall: ['clean'], withWarmModules: ['node_modules'], - withLockfile: ['lockfile'], + withLockfile: ['trusted lockfile'], withWarmCacheAndModules: ['cache', 'node_modules'], withWarmCache: ['cache'], - withWarmCacheAndLockfile: ['cache', 'lockfile'], - withWarmModulesAndLockfile: ['lockfile', 'node_modules'], - repeatInstall: ['cache', 'lockfile', 'node_modules'], + withWarmCacheAndLockfile: ['cache', 'trusted lockfile'], + withWarmModulesAndLockfile: ['trusted lockfile', 'node_modules'], + repeatInstall: ['cache', 'trusted lockfile', 'node_modules'], updatedDependencies: ['update'], } diff --git a/results/pnpm11/11.22.0/alotta-files-pnpr.yaml b/results/pnpm11/11.22.0/alotta-files-pnpr.yaml deleted file mode 100644 index 511c3a4..0000000 --- a/results/pnpm11/11.22.0/alotta-files-pnpr.yaml +++ /dev/null @@ -1,150 +0,0 @@ -- firstInstall: 8771 - repeatInstall: 629 - withWarmCacheAndLockfile: 2741 - withWarmCache: 4967 - withLockfile: 7310 - withWarmCacheAndModules: 802 - withWarmModulesAndLockfile: 783 - withWarmModules: 833 - updatedDependencies: 8601 - size: 307853191 -- firstInstall: 8838 - repeatInstall: 642 - withWarmCacheAndLockfile: 2747 - withWarmCache: 4987 - withLockfile: 7174 - withWarmCacheAndModules: 837 - withWarmModulesAndLockfile: 800 - withWarmModules: 833 - updatedDependencies: 8711 - size: 307853191 -- firstInstall: 8853 - repeatInstall: 643 - withWarmCacheAndLockfile: 2846 - withWarmCache: 5098 - withLockfile: 7268 - withWarmCacheAndModules: 825 - withWarmModulesAndLockfile: 795 - withWarmModules: 819 - updatedDependencies: 8742 - size: 307853191 -- firstInstall: 8229 - repeatInstall: 608 - withWarmCacheAndLockfile: 2707 - withWarmCache: 4764 - withLockfile: 6961 - withWarmCacheAndModules: 753 - withWarmModulesAndLockfile: 766 - withWarmModules: 780 - updatedDependencies: 8087 - size: 307860851 -- firstInstall: 8424 - repeatInstall: 606 - withWarmCacheAndLockfile: 2682 - withWarmCache: 4802 - withLockfile: 6849 - withWarmCacheAndModules: 749 - withWarmModulesAndLockfile: 738 - withWarmModules: 761 - updatedDependencies: 7915 - size: 307860851 -- firstInstall: 8443 - repeatInstall: 608 - withWarmCacheAndLockfile: 2661 - withWarmCache: 4789 - withLockfile: 7738 - withWarmCacheAndModules: 764 - withWarmModulesAndLockfile: 794 - withWarmModules: 758 - updatedDependencies: 7992 - size: 307860851 -- firstInstall: 8525 - repeatInstall: 615 - withWarmCacheAndLockfile: 2771 - withWarmCache: 4827 - withLockfile: 6998 - withWarmCacheAndModules: 767 - withWarmModulesAndLockfile: 746 - withWarmModules: 790 - updatedDependencies: 8327 - size: 307865167 -- firstInstall: 8587 - repeatInstall: 613 - withWarmCacheAndLockfile: 2768 - withWarmCache: 4893 - withLockfile: 6944 - withWarmCacheAndModules: 785 - withWarmModulesAndLockfile: 754 - withWarmModules: 754 - updatedDependencies: 8222 - size: 307865167 -- firstInstall: 8527 - repeatInstall: 610 - withWarmCacheAndLockfile: 2791 - withWarmCache: 4878 - withLockfile: 7000 - withWarmCacheAndModules: 789 - withWarmModulesAndLockfile: 783 - withWarmModules: 793 - updatedDependencies: 8216 - size: 307869263 -- firstInstall: 8450 - repeatInstall: 603 - withWarmCacheAndLockfile: 2553 - withWarmCache: 4682 - withLockfile: 6940 - withWarmCacheAndModules: 778 - withWarmModulesAndLockfile: 759 - withWarmModules: 767 - updatedDependencies: 8178 - size: 307865227 -- firstInstall: 8833 - repeatInstall: 619 - withWarmCacheAndLockfile: 2637 - withWarmCache: 5034 - withLockfile: 7142 - withWarmCacheAndModules: 850 - withWarmModulesAndLockfile: 811 - withWarmModules: 807 - updatedDependencies: 8630 - size: 307865227 -- firstInstall: 8240 - repeatInstall: 593 - withWarmCacheAndLockfile: 2538 - withWarmCache: 4672 - withLockfile: 6876 - withWarmCacheAndModules: 758 - withWarmModulesAndLockfile: 733 - withWarmModules: 761 - updatedDependencies: 8021 - size: 307865227 -- firstInstall: 8155 - repeatInstall: 593 - withWarmCacheAndLockfile: 2523 - withWarmCache: 4655 - withLockfile: 6787 - withWarmCacheAndModules: 765 - withWarmModulesAndLockfile: 778 - withWarmModules: 740 - updatedDependencies: 8131 - size: 307865227 -- firstInstall: 8180 - repeatInstall: 584 - withWarmCacheAndLockfile: 2474 - withWarmCache: 4599 - withLockfile: 6769 - withWarmCacheAndModules: 731 - withWarmModulesAndLockfile: 769 - withWarmModules: 741 - updatedDependencies: 8066 - size: 307869323 -- firstInstall: 8298 - repeatInstall: 603 - withWarmCacheAndLockfile: 2714 - withWarmCache: 4724 - withLockfile: 7035 - withWarmCacheAndModules: 764 - withWarmModulesAndLockfile: 736 - withWarmModules: 781 - updatedDependencies: 8123 - size: 307865227 diff --git a/results/pnpm12/12.0.0-rc.6/alotta-files-pnpr.yaml b/results/pnpm12/12.0.0-rc.6/alotta-files-pnpr.yaml deleted file mode 100644 index 4f88c77..0000000 --- a/results/pnpm12/12.0.0-rc.6/alotta-files-pnpr.yaml +++ /dev/null @@ -1,60 +0,0 @@ -- firstInstall: 5233 - repeatInstall: 19 - withWarmCacheAndLockfile: 699 - withWarmCache: 1211 - withLockfile: 4932 - withWarmCacheAndModules: 61 - withWarmModulesAndLockfile: 77 - withWarmModules: 68 - updatedDependencies: 3572 - size: 325826390 -- firstInstall: 5208 - repeatInstall: 21 - withWarmCacheAndLockfile: 680 - withWarmCache: 1207 - withLockfile: 5174 - withWarmCacheAndModules: 62 - withWarmModulesAndLockfile: 79 - withWarmModules: 71 - updatedDependencies: 3612 - size: 325810006 -- firstInstall: 5258 - repeatInstall: 20 - withWarmCacheAndLockfile: 694 - withWarmCache: 1198 - withLockfile: 5213 - withWarmCacheAndModules: 60 - withWarmModulesAndLockfile: 76 - withWarmModules: 68 - updatedDependencies: 3562 - size: 325818198 -- firstInstall: 5073 - repeatInstall: 18 - withWarmCacheAndLockfile: 659 - withWarmCache: 1227 - withLockfile: 4915 - withWarmCacheAndModules: 59 - withWarmModulesAndLockfile: 73 - withWarmModules: 65 - updatedDependencies: 3500 - size: 325825858 -- firstInstall: 5106 - repeatInstall: 19 - withWarmCacheAndLockfile: 717 - withWarmCache: 1105 - withLockfile: 5097 - withWarmCacheAndModules: 58 - withWarmModulesAndLockfile: 73 - withWarmModules: 68 - updatedDependencies: 3421 - size: 325825858 -- firstInstall: 5073 - repeatInstall: 18 - withWarmCacheAndLockfile: 695 - withWarmCache: 1115 - withLockfile: 4999 - withWarmCacheAndModules: 58 - withWarmModulesAndLockfile: 73 - withWarmModules: 65 - updatedDependencies: 3474 - size: 325821762 diff --git a/results/pnpm12/12.0.0-rc.7/alotta-files-pnpr.yaml b/results/pnpm12/12.0.0-rc.7/alotta-files-pnpr.yaml deleted file mode 100644 index 5040037..0000000 --- a/results/pnpm12/12.0.0-rc.7/alotta-files-pnpr.yaml +++ /dev/null @@ -1,90 +0,0 @@ -- firstInstall: 5083 - repeatInstall: 19 - withWarmCacheAndLockfile: 641 - withWarmCache: 1130 - withLockfile: 4681 - withWarmCacheAndModules: 61 - withWarmModulesAndLockfile: 76 - withWarmModules: 75 - updatedDependencies: 3232 - size: 325836200 -- firstInstall: 5139 - repeatInstall: 19 - withWarmCacheAndLockfile: 639 - withWarmCache: 1148 - withLockfile: 4314 - withWarmCacheAndModules: 59 - withWarmModulesAndLockfile: 76 - withWarmModules: 67 - updatedDependencies: 3259 - size: 325840296 -- firstInstall: 5186 - repeatInstall: 20 - withWarmCacheAndLockfile: 658 - withWarmCache: 1150 - withLockfile: 4441 - withWarmCacheAndModules: 59 - withWarmModulesAndLockfile: 74 - withWarmModules: 67 - updatedDependencies: 3130 - size: 325848488 -- firstInstall: 5378 - repeatInstall: 20 - withWarmCacheAndLockfile: 698 - withWarmCache: 1189 - withLockfile: 4445 - withWarmCacheAndModules: 62 - withWarmModulesAndLockfile: 81 - withWarmModules: 71 - updatedDependencies: 3397 - size: 325832164 -- firstInstall: 5570 - repeatInstall: 23 - withWarmCacheAndLockfile: 787 - withWarmCache: 1272 - withLockfile: 4278 - withWarmCacheAndModules: 66 - withWarmModulesAndLockfile: 81 - withWarmModules: 75 - updatedDependencies: 3490 - size: 325836260 -- firstInstall: 5166 - repeatInstall: 20 - withWarmCacheAndLockfile: 698 - withWarmCache: 1277 - withLockfile: 4427 - withWarmCacheAndModules: 61 - withWarmModulesAndLockfile: 77 - withWarmModules: 69 - updatedDependencies: 3335 - size: 325844452 -- firstInstall: 5223 - repeatInstall: 20 - withWarmCacheAndLockfile: 700 - withWarmCache: 1129 - withLockfile: 4537 - withWarmCacheAndModules: 63 - withWarmModulesAndLockfile: 76 - withWarmModules: 68 - updatedDependencies: 3309 - size: 325836260 -- firstInstall: 5083 - repeatInstall: 20 - withWarmCacheAndLockfile: 663 - withWarmCache: 1131 - withLockfile: 4582 - withWarmCacheAndModules: 61 - withWarmModulesAndLockfile: 76 - withWarmModules: 69 - updatedDependencies: 3311 - size: 325844452 -- firstInstall: 5147 - repeatInstall: 19 - withWarmCacheAndLockfile: 643 - withWarmCache: 1121 - withLockfile: 4326 - withWarmCacheAndModules: 61 - withWarmModulesAndLockfile: 72 - withWarmModules: 66 - updatedDependencies: 3254 - size: 325836260 diff --git a/results/pnpm_pnpr/12.0.0-rc.6/alotta-files-pnpr.yaml b/results/pnpm_pnpr/12.0.0-rc.6/alotta-files-pnpr.yaml deleted file mode 100644 index 8dfca17..0000000 --- a/results/pnpm_pnpr/12.0.0-rc.6/alotta-files-pnpr.yaml +++ /dev/null @@ -1,60 +0,0 @@ -- firstInstall: 3785 - repeatInstall: 19 - withWarmCacheAndLockfile: 689 - withWarmCache: 812 - withLockfile: 3793 - withWarmCacheAndModules: 66 - withWarmModulesAndLockfile: 85 - withWarmModules: 74 - updatedDependencies: 1157 - size: 325912687 -- firstInstall: 3809 - repeatInstall: 19 - withWarmCacheAndLockfile: 666 - withWarmCache: 765 - withLockfile: 3900 - withWarmCacheAndModules: 66 - withWarmModulesAndLockfile: 84 - withWarmModules: 74 - updatedDependencies: 1209 - size: 325933167 -- firstInstall: 3909 - repeatInstall: 19 - withWarmCacheAndLockfile: 765 - withWarmCache: 807 - withLockfile: 3840 - withWarmCacheAndModules: 66 - withWarmModulesAndLockfile: 83 - withWarmModules: 75 - updatedDependencies: 1210 - size: 325933167 -- firstInstall: 3754 - repeatInstall: 18 - withWarmCacheAndLockfile: 650 - withWarmCache: 775 - withLockfile: 3792 - withWarmCacheAndModules: 63 - withWarmModulesAndLockfile: 79 - withWarmModules: 70 - updatedDependencies: 1142 - size: 325928539 -- firstInstall: 3865 - repeatInstall: 18 - withWarmCacheAndLockfile: 648 - withWarmCache: 826 - withLockfile: 3774 - withWarmCacheAndModules: 62 - withWarmModulesAndLockfile: 79 - withWarmModules: 69 - updatedDependencies: 1184 - size: 325924443 -- firstInstall: 3841 - repeatInstall: 18 - withWarmCacheAndLockfile: 658 - withWarmCache: 796 - withLockfile: 3803 - withWarmCacheAndModules: 62 - withWarmModulesAndLockfile: 78 - withWarmModules: 69 - updatedDependencies: 1138 - size: 325932635 diff --git a/results/pnpm_pnpr/12.0.0-rc.7/alotta-files-pnpr.yaml b/results/pnpm_pnpr/12.0.0-rc.7/alotta-files-pnpr.yaml deleted file mode 100644 index f9712bc..0000000 --- a/results/pnpm_pnpr/12.0.0-rc.7/alotta-files-pnpr.yaml +++ /dev/null @@ -1,90 +0,0 @@ -- firstInstall: 3525 - repeatInstall: 18 - withWarmCacheAndLockfile: 678 - withWarmCache: 786 - withLockfile: 3549 - withWarmCacheAndModules: 68 - withWarmModulesAndLockfile: 82 - withWarmModules: 72 - updatedDependencies: 1132 - size: 325950769 -- firstInstall: 3486 - repeatInstall: 19 - withWarmCacheAndLockfile: 681 - withWarmCache: 769 - withLockfile: 3497 - withWarmCacheAndModules: 63 - withWarmModulesAndLockfile: 81 - withWarmModules: 71 - updatedDependencies: 1131 - size: 325946673 -- firstInstall: 3484 - repeatInstall: 19 - withWarmCacheAndLockfile: 743 - withWarmCache: 758 - withLockfile: 3542 - withWarmCacheAndModules: 63 - withWarmModulesAndLockfile: 80 - withWarmModules: 71 - updatedDependencies: 1126 - size: 325950769 -- firstInstall: 3586 - repeatInstall: 19 - withWarmCacheAndLockfile: 788 - withWarmCache: 813 - withLockfile: 3488 - withWarmCacheAndModules: 66 - withWarmModulesAndLockfile: 83 - withWarmModules: 75 - updatedDependencies: 1147 - size: 325954925 -- firstInstall: 3601 - repeatInstall: 22 - withWarmCacheAndLockfile: 742 - withWarmCache: 814 - withLockfile: 3584 - withWarmCacheAndModules: 71 - withWarmModulesAndLockfile: 91 - withWarmModules: 80 - updatedDependencies: 1191 - size: 325938541 -- firstInstall: 3584 - repeatInstall: 21 - withWarmCacheAndLockfile: 737 - withWarmCache: 812 - withLockfile: 3476 - withWarmCacheAndModules: 66 - withWarmModulesAndLockfile: 83 - withWarmModules: 74 - updatedDependencies: 1145 - size: 325938541 -- firstInstall: 3535 - repeatInstall: 20 - withWarmCacheAndLockfile: 662 - withWarmCache: 765 - withLockfile: 3477 - withWarmCacheAndModules: 66 - withWarmModulesAndLockfile: 82 - withWarmModules: 74 - updatedDependencies: 1141 - size: 325934445 -- firstInstall: 3683 - repeatInstall: 20 - withWarmCacheAndLockfile: 669 - withWarmCache: 760 - withLockfile: 3523 - withWarmCacheAndModules: 65 - withWarmModulesAndLockfile: 82 - withWarmModules: 73 - updatedDependencies: 1135 - size: 325942637 -- firstInstall: 3599 - repeatInstall: 19 - withWarmCacheAndLockfile: 683 - withWarmCache: 773 - withLockfile: 3463 - withWarmCacheAndModules: 64 - withWarmModulesAndLockfile: 80 - withWarmModules: 72 - updatedDependencies: 1126 - size: 325954925