From 764955136d8d457ce126c9c8fb1d575531759747 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 21 Jun 2026 02:10:35 +0200 Subject: [PATCH 1/5] ci: add pull request benchmarks --- .github/workflows/benchmark.yml | 194 +++++++++ .gitignore | 1 + benchmarks/README.md | 76 ++++ package.json | 7 + pnpm-lock.yaml | 28 +- scripts/bench-builds.mjs | 700 ++++++++++++++++++++++++++++++++ scripts/benchmark/fixture.mjs | 454 +++++++++++++++++++++ scripts/compare-benchmarks.mjs | 128 ++++++ scripts/report-benchmark-ci.mjs | 167 ++++++++ 9 files changed, 1754 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/benchmark.yml create mode 100644 benchmarks/README.md create mode 100644 scripts/bench-builds.mjs create mode 100644 scripts/benchmark/fixture.mjs create mode 100644 scripts/compare-benchmarks.mjs create mode 100644 scripts/report-benchmark-ci.mjs diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 0000000..0b79f33 --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,194 @@ +name: Benchmark + +on: + pull_request: + branches: [main] + +permissions: + contents: write + issues: write + pull-requests: write + +concurrency: + group: benchmark-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + benchmark: + name: Compare build benchmarks + runs-on: ubuntu-latest + + env: + BENCHMARK_PROFILE: default + BENCHMARK_ITERATIONS: '5' + BENCHMARK_WARMUP: '1' + BENCHMARK_HISTORY_BRANCH: benchmark-results + + steps: + - name: Checkout PR head + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 + with: + path: head + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + + - name: Checkout PR base + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 + with: + path: base + ref: ${{ github.event.pull_request.base.sha }} + persist-credentials: false + + - name: Setup Node.js + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 + with: + node-version-file: head/.nvmrc + + - name: Setup PNPM + uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 + + - name: Get pnpm store directory + id: pnpm-cache + shell: bash + run: | + echo "STORE_PATH=$(pnpm store path)" >> "$GITHUB_OUTPUT" + + - name: Setup pnpm cache + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 + with: + path: ${{ steps.pnpm-cache.outputs.STORE_PATH }} + key: ${{ runner.os }}-benchmark-pnpm-${{ hashFiles('head/pnpm-lock.yaml', 'base/pnpm-lock.yaml') }} + restore-keys: | + ${{ runner.os }}-benchmark-pnpm- + ${{ runner.os }}-pnpm-store- + + - name: Install head dependencies + working-directory: head + run: pnpm install --frozen-lockfile + + - name: Install base dependencies + working-directory: base + run: pnpm install --frozen-lockfile + + - name: Resolve benchmark-only React plugin + id: benchmark-imports + shell: bash + run: | + node -e "const { pathToFileURL } = require('node:url'); console.log('PLUGIN_REACT_IMPORT=' + pathToFileURL(process.argv[1]).href)" \ + "$GITHUB_WORKSPACE/head/node_modules/@rsbuild/plugin-react/dist/index.js" >> "$GITHUB_OUTPUT" + + - name: Benchmark base + working-directory: base + env: + REACT_ROUTER_BENCHMARK_PLUGIN_REACT_IMPORT: ${{ steps.benchmark-imports.outputs.PLUGIN_REACT_IMPORT }} + run: | + node "$GITHUB_WORKSPACE/head/scripts/bench-builds.mjs" \ + --profile "$BENCHMARK_PROFILE" \ + --iterations "$BENCHMARK_ITERATIONS" \ + --warmup "$BENCHMARK_WARMUP" \ + --clean build \ + --format both \ + --out "$GITHUB_WORKSPACE/benchmark-output/base" + + - name: Benchmark head + working-directory: head + run: | + node scripts/bench-builds.mjs \ + --profile "$BENCHMARK_PROFILE" \ + --iterations "$BENCHMARK_ITERATIONS" \ + --warmup "$BENCHMARK_WARMUP" \ + --clean build \ + --format both \ + --out "$GITHUB_WORKSPACE/benchmark-output/head" + + - name: Generate benchmark report + working-directory: head + env: + PR_NUMBER: ${{ github.event.pull_request.number }} + BASE_REF: ${{ github.event.pull_request.base.ref }} + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_REF: ${{ github.event.pull_request.head.ref }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + node scripts/report-benchmark-ci.mjs \ + --base "$GITHUB_WORKSPACE/benchmark-output/base/baseline.json" \ + --head "$GITHUB_WORKSPACE/benchmark-output/head/baseline.json" \ + --out "$GITHUB_WORKSPACE/benchmark-output/report" \ + --pr "$PR_NUMBER" \ + --base-ref "$BASE_REF" \ + --base-sha "$BASE_SHA" \ + --head-ref "$HEAD_REF" \ + --head-sha "$HEAD_SHA" \ + --run-url "$RUN_URL" + + - name: Upload benchmark artifacts + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: benchmark-pr-${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }} + path: benchmark-output + if-no-files-found: error + + - name: Comment benchmark results + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + REPOSITORY: ${{ github.repository }} + run: | + set -euo pipefail + marker='' + comment_body="$GITHUB_WORKSPACE/benchmark-output/report/comment.md" + body="$(cat "$comment_body")" + comment_id="$(gh api "repos/$REPOSITORY/issues/$PR_NUMBER/comments" --paginate \ + --jq ".[] | select(.body | contains(\"$marker\")) | .id" | tail -n 1)" + + if [ -n "$comment_id" ]; then + gh api --method PATCH "repos/$REPOSITORY/issues/comments/$comment_id" \ + --field body="$body" + else + gh api --method POST "repos/$REPOSITORY/issues/$PR_NUMBER/comments" \ + --field body="$body" + fi + + - name: Persist benchmark history + if: github.event.pull_request.head.repo.full_name == github.repository + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + REPOSITORY: ${{ github.repository }} + shell: bash + run: | + set -euo pipefail + history_dir="$GITHUB_WORKSPACE/benchmark-history" + git clone --depth 1 --branch "$BENCHMARK_HISTORY_BRANCH" \ + "https://x-access-token:${GH_TOKEN}@github.com/${REPOSITORY}.git" "$history_dir" \ + || { + git clone --depth 1 "https://x-access-token:${GH_TOKEN}@github.com/${REPOSITORY}.git" "$history_dir" + cd "$history_dir" + git checkout --orphan "$BENCHMARK_HISTORY_BRANCH" + git rm -rf . + } + + cd "$history_dir" + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + + result_dir="pull-requests/${PR_NUMBER}/${HEAD_SHA}" + latest_dir="pull-requests/${PR_NUMBER}/latest" + mkdir -p "$result_dir" "$latest_dir" + cp "$GITHUB_WORKSPACE/benchmark-output/base/baseline.json" "$result_dir/base.json" + cp "$GITHUB_WORKSPACE/benchmark-output/head/baseline.json" "$result_dir/head.json" + cp "$GITHUB_WORKSPACE/benchmark-output/report/report.json" "$result_dir/report.json" + cp "$GITHUB_WORKSPACE/benchmark-output/report/comment.md" "$result_dir/comment.md" + cp "$result_dir/report.json" "$latest_dir/report.json" + cp "$result_dir/comment.md" "$latest_dir/comment.md" + + git add pull-requests + if git diff --cached --quiet; then + echo "Benchmark history is already up to date." + exit 0 + fi + + git commit -m "bench: record PR ${PR_NUMBER} ${HEAD_SHA:0:7}" + git push origin "$BENCHMARK_HISTORY_BRANCH" diff --git a/.gitignore b/.gitignore index 59214f8..18b9153 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ build .unpack-cache/ .codex/ .tracedecay/ +.benchmark/ task/upstream/ task/output/ diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 0000000..8cb14a0 --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,76 @@ +# Rsbuild React Router Performance Baselines + +This directory documents the repeatable benchmark workflow for +`rsbuild-plugin-react-router`. Benchmark artifacts are written to `.benchmark/`, +which is intentionally ignored. + +## Commands + +```sh +pnpm bench:smoke +pnpm bench:baseline +pnpm bench:full +``` + +`bench:smoke` is a one-iteration sanity check. `bench:baseline` is the default +comparison point for plugin performance work. `bench:full` adds larger route +counts for stress testing. + +All benchmark profiles generate deterministic synthetic React Router apps under +`.benchmark/fixtures/`, build the current plugin package once, then run Rsbuild +builds with `pluginReactRouter({ logPerformance: true })`. + +To capture Rspack tracing output for a benchmark, pass `--rspack-profile`: + +```sh +node scripts/bench-builds.mjs --profile=smoke --iterations=1 --warmup=0 --rspack-profile=OVERVIEW +node scripts/bench-builds.mjs --profile=full --filter=synthetic-1024 --iterations=1 --warmup=0 --rspack-profile=ALL +``` + +Trace directories are moved from fixture roots into +`.benchmark/results//rspack-profiles/` and referenced from the JSON +result. `ALL` can produce large traces; use it for targeted runs. +When `--rspack-trace-output` is provided, the benchmark writes one absolute +trace file per run under that directory so Rsbuild does not resolve the path +inside each generated `.rspack-profile-*` directory. + +## Baseline Shape + +The synthetic fixture keeps app behavior simple and scales route count/export +shape deliberately: + +- `ssr-esm`: production SSR build with ESM server output. +- `ssr-esm-split`: same route set with `future.v8_splitRouteModules`. +- `spa`: `ssr: false` route transform path. +- `sourcemaps`: production client sourcemaps enabled. + +Routes include plain components, server data exports, client data exports, +split-route candidates, and `.client` / `.server` imports. This targets the +plugin paths that are expensive in large apps: per-route client entries, +`?react-router-route` transforms, client-only stubbing, split-route detection, +and manifest emission. + +## Outputs + +Each run writes: + +- `.benchmark/results//baseline.json` +- `.benchmark/results//baseline.md` + +The JSON includes wall time, optional GNU `/usr/bin/time -v` user/sys/RSS data, +parsed `[react-router:performance]` reports from the plugin, and an aggregated +`pluginOperations` table per fixture. The markdown report includes the same +operation breakdown so route transforms and manifest work can be compared +without opening the raw JSON. + +## Hygiene + +Start and end with: + +```sh +git status --short +``` + +Benchmark output should stay inside ignored `.benchmark/`. If you need to clean +generated benchmark data, remove `.benchmark/` directly rather than using a broad +`git clean -fdX`, which can also delete `node_modules/` and TraceDecay indexes. diff --git a/package.json b/package.json index 0ff1577..f964c0d 100644 --- a/package.json +++ b/package.json @@ -49,6 +49,12 @@ }, "scripts": { "build": "rslib build", + "bench": "node scripts/bench-builds.mjs", + "bench:ci-report": "node scripts/report-benchmark-ci.mjs", + "bench:compare": "node scripts/compare-benchmarks.mjs", + "bench:smoke": "node scripts/bench-builds.mjs --profile smoke --iterations 1 --warmup 0 --format both --out .benchmark/results/smoke", + "bench:baseline": "node scripts/bench-builds.mjs --profile default --iterations 5 --warmup 1 --clean build --format both --out .benchmark/results/baseline", + "bench:full": "node scripts/bench-builds.mjs --profile full --iterations 5 --warmup 1 --clean build --format both --out .benchmark/results/full", "e2e": "pnpm build && pnpm --filter './examples/{default-template,spa-mode,prerender,custom-node-server,cloudflare,client-only}' test:e2e", "dev": "rslib build --watch", "test": "rstest run", @@ -88,6 +94,7 @@ "@react-router/dev": "^7.13.0", "@rsbuild/config": "workspace:*", "@rsbuild/core": "2.0.15", + "@rsbuild/plugin-react": "2.0.1", "@rslib/core": "^0.22.1", "@rspack/core": "2.0.8", "@swc/helpers": "^0.5.23", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 633631e..994cb95 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -78,6 +78,9 @@ importers: '@rsbuild/core': specifier: 2.0.15 version: 2.0.15(@module-federation/runtime-tools@2.5.1(node-fetch@2.7.0(encoding@0.1.13)))(core-js@3.47.0) + '@rsbuild/plugin-react': + specifier: 2.0.1 + version: 2.0.1(@rsbuild/core@2.0.15(@module-federation/runtime-tools@2.5.1(node-fetch@2.7.0(encoding@0.1.13)))(core-js@3.47.0))(@rspack/core@2.0.8(@module-federation/runtime-tools@2.5.1(node-fetch@2.7.0(encoding@0.1.13)))(@swc/helpers@0.5.23)) '@rslib/core': specifier: ^0.22.1 version: 0.22.1(@module-federation/runtime-tools@2.5.1)(core-js@3.47.0)(typescript@5.9.3) @@ -167,7 +170,7 @@ importers: dependencies: '@react-router/express': specifier: ^7.13.0 - version: 7.13.0(express@4.22.1)(react-router@7.13.0(react-dom@19.2.4(react@19.2.4))(react@19.2.4))(typescript@5.9.3) + version: 7.13.0(express@5.2.1)(react-router@7.13.0(react-dom@19.2.4(react@19.2.4))(react@19.2.4))(typescript@5.9.3) '@react-router/node': specifier: ^7.13.0 version: 7.13.0(react-router@7.13.0(react-dom@19.2.4(react@19.2.4))(react@19.2.4))(typescript@5.9.3) @@ -3667,96 +3670,112 @@ packages: '@react-email/body@0.2.1': resolution: {integrity: sha512-ljDiQiJDu/Fq//vSIIP0z5Nuvt4+DX1RqGasstChDGJB/14ogd4VdNS9aacoede/ZjGy3o3Qb+cxyS+XgM6SwQ==} engines: {node: '>=20.0.0'} + deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info. peerDependencies: react: ^18.0 || ^19.0 || ^19.0.0-rc '@react-email/button@0.2.1': resolution: {integrity: sha512-qXyj7RZLE7POy9BMKSoqQ00tOXThjOZSUnI2Yu9i29IHngPlmrNayIWBoVKtElES7OWwypUcpiajwi1mUWx6/A==} engines: {node: '>=20.0.0'} + deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info. peerDependencies: react: ^18.0 || ^19.0 || ^19.0.0-rc '@react-email/code-block@0.2.1': resolution: {integrity: sha512-M3B7JpVH4ytgn83/ujRR1k1DQHvTeABiDM61OvAbjLRPhC/5KLHU5KkzIbbuGIrjWwxAbL1kSQzU8MhLEtSxyw==} engines: {node: '>=20.0.0'} + deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info. peerDependencies: react: ^18.0 || ^19.0 || ^19.0.0-rc '@react-email/code-inline@0.0.6': resolution: {integrity: sha512-jfhebvv3dVsp3OdPgKXnk8+e2pBiDVZejDOBFzBa/IblrAJ9cQDkN6rBD5IyEg8hTOxwbw3iaI/yZFmDmIguIA==} engines: {node: '>=20.0.0'} + deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info. peerDependencies: react: ^18.0 || ^19.0 || ^19.0.0-rc '@react-email/column@0.0.14': resolution: {integrity: sha512-f+W+Bk2AjNO77zynE33rHuQhyqVICx4RYtGX9NKsGUg0wWjdGP0qAuIkhx9Rnmk4/hFMo1fUrtYNqca9fwJdHg==} engines: {node: '>=20.0.0'} + deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info. peerDependencies: react: ^18.0 || ^19.0 || ^19.0.0-rc '@react-email/components@1.0.6': resolution: {integrity: sha512-3GwOeq+5yyiAcwSf7TnHi/HWKn22lXbwxQmkkAviSwZLlhsRVxvmWqRxvUVfQk/HclDUG+62+sGz9qjfb2Uxjw==} engines: {node: '>=20.0.0'} + deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info. peerDependencies: react: ^18.0 || ^19.0 || ^19.0.0-rc '@react-email/container@0.0.16': resolution: {integrity: sha512-QWBB56RkkU0AJ9h+qy33gfT5iuZknPC7Un/IjZv9B0QmMIK+WWacc0cH6y2SV5Cv/b99hU94fjEMOOO4enpkbQ==} engines: {node: '>=20.0.0'} + deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info. peerDependencies: react: ^18.0 || ^19.0 || ^19.0.0-rc '@react-email/font@0.0.10': resolution: {integrity: sha512-0urVSgCmQIfx5r7Xc586miBnQUVnGp3OTYUm8m5pwtQRdTRO5XrTtEfNJ3JhYhSOruV0nD8fd+dXtKXobum6tA==} engines: {node: '>=20.0.0'} + deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info. peerDependencies: react: ^18.0 || ^19.0 || ^19.0.0-rc '@react-email/head@0.0.13': resolution: {integrity: sha512-AJg6le/08Gz4tm+6MtKXqtNNyKHzmooOCdmtqmWxD7FxoAdU1eVcizhtQ0gcnVaY6ethEyE/hnEzQxt1zu5Kog==} engines: {node: '>=20.0.0'} + deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info. peerDependencies: react: ^18.0 || ^19.0 || ^19.0.0-rc '@react-email/heading@0.0.16': resolution: {integrity: sha512-jmsKnQm1ykpBzw4hCYHwBkt5pW2jScXffPeEH5ZRF5tZeF5b1pvlFTO9han7C0pCkZYo1kEvWiRtx69yfCIwuw==} engines: {node: '>=20.0.0'} + deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info. peerDependencies: react: ^18.0 || ^19.0 || ^19.0.0-rc '@react-email/hr@0.0.12': resolution: {integrity: sha512-TwmOmBDibavUQpXBxpmZYi2Iks/yeZOzFYh+di9EltMSnEabH8dMZXrl+pxNXzCgZ2XE8HY7VmUL65Lenfu5PA==} engines: {node: '>=20.0.0'} + deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info. peerDependencies: react: ^18.0 || ^19.0 || ^19.0.0-rc '@react-email/html@0.0.12': resolution: {integrity: sha512-KTShZesan+UsreU7PDUV90afrZwU5TLwYlALuCSU0OT+/U8lULNNbAUekg+tGwCnOfIKYtpDPKkAMRdYlqUznw==} engines: {node: '>=20.0.0'} + deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info. peerDependencies: react: ^18.0 || ^19.0 || ^19.0.0-rc '@react-email/img@0.0.12': resolution: {integrity: sha512-sRCpEARNVTf3FQhZOC+JTvu5r6ubiYWkT0ucYXg8ctkyi4G8QG+jgYPiNUqVeTLA2STOfmPM/nrk1nb84y6CPQ==} engines: {node: '>=20.0.0'} + deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info. peerDependencies: react: ^18.0 || ^19.0 || ^19.0.0-rc '@react-email/link@0.0.13': resolution: {integrity: sha512-lkWc/NjOcefRZMkQoSDDbuKBEBDES9aXnFEOuPH845wD3TxPwh+QTf0fStuzjoRLUZWpHnio4z7qGGRYusn/sw==} engines: {node: '>=20.0.0'} + deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info. peerDependencies: react: ^18.0 || ^19.0 || ^19.0.0-rc '@react-email/markdown@0.0.18': resolution: {integrity: sha512-gSuYK5fsMbGk87jDebqQ6fa2fKcWlkf2Dkva8kMONqLgGCq8/0d+ZQYMEJsdidIeBo3kmsnHZPrwdFB4HgjUXg==} engines: {node: '>=20.0.0'} + deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info. peerDependencies: react: ^18.0 || ^19.0 || ^19.0.0-rc '@react-email/preview@0.0.14': resolution: {integrity: sha512-aYK8q0IPkBXyMsbpMXgxazwHxYJxTrXrV95GFuu2HbEiIToMwSyUgb8HDFYwPqqfV03/jbwqlsXmFxsOd+VNaw==} engines: {node: '>=20.0.0'} + deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info. peerDependencies: react: ^18.0 || ^19.0 || ^19.0.0-rc @@ -3770,18 +3789,21 @@ packages: '@react-email/row@0.0.13': resolution: {integrity: sha512-bYnOac40vIKCId7IkwuLAAsa3fKfSfqCvv6epJKmPE0JBuu5qI4FHFCl9o9dVpIIS08s/ub+Y/txoMt0dYziGw==} engines: {node: '>=20.0.0'} + deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info. peerDependencies: react: ^18.0 || ^19.0 || ^19.0.0-rc '@react-email/section@0.0.17': resolution: {integrity: sha512-qNl65ye3W0Rd5udhdORzTV9ezjb+GFqQQSae03NDzXtmJq6sqVXNWNiVolAjvJNypim+zGXmv6J9TcV5aNtE/w==} engines: {node: '>=20.0.0'} + deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info. peerDependencies: react: ^18.0 || ^19.0 || ^19.0.0-rc '@react-email/tailwind@2.0.3': resolution: {integrity: sha512-URXb/T2WS4RlNGM5QwekYnivuiVUcU87H0y5sqLl6/Oi3bMmgL0Bmw/W9GeJylC+876Vw+E6NkE0uRiUFIQwGg==} engines: {node: '>=20.0.0'} + deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info. peerDependencies: '@react-email/body': 0.2.1 '@react-email/button': 0.2.1 @@ -3820,6 +3842,7 @@ packages: '@react-email/text@0.1.6': resolution: {integrity: sha512-TYqkioRS45wTR5il3dYk/SbUjjEdhSwh9BtRNB99qNH1pXAwA45H7rAuxehiu8iJQJH0IyIr+6n62gBz9ezmsw==} engines: {node: '>=20.0.0'} + deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info. peerDependencies: react: ^18.0 || ^19.0 || ^19.0.0-rc @@ -6493,6 +6516,7 @@ packages: glob@10.5.0: resolution: {integrity: sha512-DfXN8DfhJ7NH3Oe7cFmu3NCu1wKbkReJ8TorzSAFbSKrlNaQSKfIzqYqVY8zlbs2NLBbWpRiU52GX2PbaBVNkg==} + deprecated: Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me hasBin: true glob@13.0.0: @@ -7778,6 +7802,7 @@ packages: prebuild-install@7.1.3: resolution: {integrity: sha512-8Mf2cbV7x1cXPUILADGI3wuhfqWvtiLA1iclTDbFRZkgRQS0NqsPZphna9V+HyTEadheuPmjaJMsbzKQFOzLug==} engines: {node: '>=10'} + deprecated: No longer maintained. Please contact the author of the relevant native addon; alternatives are available. hasBin: true prelude-ls@1.2.1: @@ -9130,6 +9155,7 @@ packages: uuid@8.3.2: resolution: {integrity: sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==} + deprecated: uuid@10 and below is no longer supported. For ESM codebases, update to uuid@latest. For CommonJS codebases, use uuid@11 (but be aware this version will likely be deprecated in 2028). hasBin: true valibot@1.2.0: diff --git a/scripts/bench-builds.mjs b/scripts/bench-builds.mjs new file mode 100644 index 0000000..062ba11 --- /dev/null +++ b/scripts/bench-builds.mjs @@ -0,0 +1,700 @@ +#!/usr/bin/env node +import { + access, + cp, + mkdir, + readdir, + rename, + rm, + writeFile, +} from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { performance } from 'node:perf_hooks'; +import { pathToFileURL } from 'node:url'; +import { parseArgs as parseCliArgs } from 'node:util'; +import { execa } from 'execa'; +import { generateSyntheticFixture } from './benchmark/fixture.mjs'; + +const rootDir = process.cwd(); +const benchmarkRoot = path.join(rootDir, '.benchmark'); +const rsbuildBin = path.join( + rootDir, + 'node_modules', + '@rsbuild', + 'core', + 'bin', + 'rsbuild.js' +); + +const profiles = { + smoke: [{ id: 'synthetic-48-ssr-esm', routeCount: 48, variant: 'ssr-esm' }], + default: [ + { id: 'synthetic-256-ssr-esm', routeCount: 256, variant: 'ssr-esm' }, + { + id: 'synthetic-256-ssr-esm-split', + routeCount: 256, + variant: 'ssr-esm-split', + }, + { id: 'synthetic-256-spa', routeCount: 256, variant: 'spa' }, + { + id: 'synthetic-256-sourcemaps', + routeCount: 256, + variant: 'ssr-esm', + sourceMap: true, + }, + ], + full: [ + { id: 'synthetic-48-ssr-esm', routeCount: 48, variant: 'ssr-esm' }, + { id: 'synthetic-256-ssr-esm', routeCount: 256, variant: 'ssr-esm' }, + { id: 'synthetic-1024-ssr-esm', routeCount: 1024, variant: 'ssr-esm' }, + { + id: 'synthetic-256-ssr-esm-split', + routeCount: 256, + variant: 'ssr-esm-split', + }, + { + id: 'synthetic-1024-ssr-esm-split', + routeCount: 1024, + variant: 'ssr-esm-split', + }, + { + id: 'synthetic-256-sourcemaps', + routeCount: 256, + variant: 'ssr-esm', + sourceMap: true, + }, + ], +}; + +const parseArgs = argv => { + const { values } = parseCliArgs({ + args: argv, + allowPositionals: false, + strict: true, + options: { + profile: { type: 'string', default: 'default' }, + iterations: { type: 'string', default: '5' }, + warmup: { type: 'string', default: '1' }, + format: { type: 'string', default: 'both' }, + out: { + type: 'string', + default: path.join('.benchmark', 'results', 'baseline'), + }, + clean: { type: 'string', default: 'build' }, + filter: { type: 'string' }, + 'parallel-transforms': { type: 'string' }, + 'rspack-profile': { type: 'string' }, + 'rspack-trace-output': { type: 'string' }, + 'fail-fast': { type: 'boolean', default: false }, + 'skip-root-build': { type: 'boolean', default: false }, + }, + }); + + const parseParallelTransforms = value => { + if (value === undefined) { + return undefined; + } + if (value === 'false' || value === '0') { + return false; + } + if (value === 'true' || value === '1' || value === 'auto') { + return true; + } + const maxWorkers = Number(value); + if (!Number.isInteger(maxWorkers) || maxWorkers < 1) { + throw new Error( + '--parallel-transforms must be true, false, auto, or a positive integer.' + ); + } + return { maxWorkers }; + }; + + const args = { + profile: values.profile, + iterations: Number(values.iterations), + warmup: Number(values.warmup), + format: values.format, + out: values.out, + clean: values.clean, + filter: values.filter ?? null, + parallelTransforms: parseParallelTransforms(values['parallel-transforms']), + rspackProfile: values['rspack-profile'] ?? null, + rspackTraceOutput: values['rspack-trace-output'] ?? null, + failFast: values['fail-fast'], + skipRootBuild: values['skip-root-build'], + }; + + if (!profiles[args.profile]) { + throw new Error( + `Unknown profile "${args.profile}". Use smoke, default, or full.` + ); + } + if (!Number.isInteger(args.iterations) || args.iterations < 1) { + throw new Error('--iterations must be a positive integer.'); + } + if (!Number.isInteger(args.warmup) || args.warmup < 0) { + throw new Error('--warmup must be a non-negative integer.'); + } + if (!['json', 'md', 'markdown', 'both'].includes(args.format)) { + throw new Error('--format must be json, md, markdown, or both.'); + } + if (!['none', 'build', 'cold'].includes(args.clean)) { + throw new Error('--clean must be none, build, or cold.'); + } + if (args.rspackProfile !== null && args.rspackProfile.trim() === '') { + throw new Error('--rspack-profile must not be empty.'); + } + if (args.rspackTraceOutput !== null && args.rspackTraceOutput.trim() === '') { + throw new Error('--rspack-trace-output must not be empty.'); + } + + return args; +}; + +const hasGnuTime = async () => { + try { + await access('/usr/bin/time'); + return true; + } catch { + return false; + } +}; + +const runCommand = async ({ + command, + args, + cwd, + env = {}, + useTime = false, +}) => { + const startedAt = performance.now(); + const childCommand = useTime ? '/usr/bin/time' : command; + const childArgs = useTime ? ['-v', command, ...args] : args; + + const child = execa(childCommand, childArgs, { + cwd, + env, + reject: false, + }); + + child.stdout?.pipe(process.stdout); + child.stderr?.pipe(process.stderr); + + const result = await child; + return { + status: result.exitCode, + stdout: result.stdout, + stderr: result.stderr, + wallMs: performance.now() - startedAt, + }; +}; + +const parseTimeStats = stderr => { + const user = stderr.match(/User time \(seconds\):\s*([\d.]+)/); + const sys = stderr.match(/System time \(seconds\):\s*([\d.]+)/); + const rss = stderr.match(/Maximum resident set size \(kbytes\):\s*(\d+)/); + return { + userMs: user ? Number(user[1]) * 1000 : null, + sysMs: sys ? Number(sys[1]) * 1000 : null, + maxRssKb: rss ? Number(rss[1]) : null, + }; +}; + +const parsePluginReports = output => { + const reports = []; + for (const line of output.split(/\r?\n/)) { + const markerIndex = line.indexOf('[react-router:performance]'); + if (markerIndex === -1) { + continue; + } + const jsonStart = line.indexOf('{', markerIndex); + if (jsonStart === -1) { + continue; + } + try { + reports.push(JSON.parse(line.slice(jsonStart))); + } catch { + // Keep raw build output useful even if one line is malformed. + } + } + return reports; +}; + +const summarizeMetric = values => { + const sorted = values + .filter(value => typeof value === 'number') + .sort((a, b) => a - b); + if (sorted.length === 0) { + return { min: null, median: null, mean: null, p95: null, stdev: null }; + } + const mean = sorted.reduce((sum, value) => sum + value, 0) / sorted.length; + const variance = + sorted.reduce((sum, value) => sum + (value - mean) ** 2, 0) / sorted.length; + const percentileIndex = Math.min( + sorted.length - 1, + Math.ceil(sorted.length * 0.95) - 1 + ); + return { + min: sorted[0], + median: sorted[Math.floor(sorted.length / 2)], + mean, + p95: sorted[percentileIndex], + stdev: Math.sqrt(variance), + }; +}; + +const summarizeRuns = runs => ({ + wallMs: summarizeMetric(runs.map(run => run.wallMs)), + userMs: summarizeMetric(runs.map(run => run.userMs)), + sysMs: summarizeMetric(runs.map(run => run.sysMs)), + maxRssKb: summarizeMetric(runs.map(run => run.maxRssKb)), +}); + +const summarizePluginOperations = runs => { + const operations = new Map(); + + for (const run of runs) { + for (const report of run.pluginReports) { + for (const [operation, metrics] of Object.entries( + report.operations ?? {} + )) { + const key = `${report.environment}:${operation}`; + const current = operations.get(key) ?? { + environment: report.environment, + operation, + count: 0, + totalMs: 0, + wallMs: null, + maxMs: 0, + reports: 0, + }; + current.count += metrics.count ?? 0; + current.totalMs += metrics.totalMs ?? 0; + if (typeof metrics.wallMs === 'number') { + current.wallMs = (current.wallMs ?? 0) + metrics.wallMs; + } + current.maxMs = Math.max(current.maxMs, metrics.maxMs ?? 0); + current.reports += 1; + operations.set(key, current); + } + } + } + + return [...operations.values()].sort((a, b) => { + if (b.totalMs !== a.totalMs) { + return b.totalMs - a.totalMs; + } + return `${a.environment}:${a.operation}`.localeCompare( + `${b.environment}:${b.operation}` + ); + }); +}; + +const formatMs = value => + value == null ? '-' : `${(value / 1000).toFixed(2)}s`; +const formatReportMs = value => (value == null ? '-' : `${value.toFixed(1)}ms`); +const formatRss = value => + value == null ? '-' : `${Math.round(value / 1024)} MB`; + +const renderMarkdown = result => { + const lines = [ + '# Rsbuild React Router Benchmark Baseline', + '', + `- Date: ${result.date}`, + `- Commit: ${result.commit}`, + `- Node: ${result.node}`, + `- pnpm: ${result.pnpm}`, + `- Platform: ${result.platform}`, + `- Profile: ${result.profile}`, + `- Iterations: ${result.iterations}`, + `- Warmup: ${result.warmup}`, + `- Parallel transforms: ${formatParallelTransforms(result.parallelTransforms)}`, + `- Rspack profile: ${result.rspackProfile ?? 'false'}`, + ...(result.rspackTraceOutput + ? [`- Rspack trace output: ${result.rspackTraceOutput}`] + : []), + '', + '| Benchmark | Routes | Variant | Median wall | Mean wall | p95 wall | Max RSS | Plugin reports |', + '|---|---:|---|---:|---:|---:|---:|---:|', + ]; + + for (const benchmark of result.benchmarks) { + lines.push( + [ + benchmark.id, + benchmark.routeCount, + benchmark.variant, + formatMs(benchmark.summary.wallMs.median), + formatMs(benchmark.summary.wallMs.mean), + formatMs(benchmark.summary.wallMs.p95), + formatRss(benchmark.summary.maxRssKb.p95), + benchmark.runs.reduce((sum, run) => sum + run.pluginReports.length, 0), + ] + .join(' | ') + .replace(/^/, '| ') + .replace(/$/, ' |') + ); + } + + for (const benchmark of result.benchmarks) { + if (benchmark.pluginOperations.length === 0) { + continue; + } + lines.push( + '', + `## ${benchmark.id} Plugin Operations`, + '', + 'Total is the sum of all measured operation durations. Wall merges overlapping intervals to approximate elapsed plugin time. Max is the slowest single operation call.', + '', + '| Environment | Operation | Count | Total | Wall | Max | Reports |', + '|---|---|---:|---:|---:|---:|---:|' + ); + for (const operation of benchmark.pluginOperations.slice(0, 12)) { + lines.push( + [ + operation.environment, + operation.operation, + operation.count, + formatReportMs(operation.totalMs), + formatReportMs(operation.wallMs), + formatReportMs(operation.maxMs), + operation.reports, + ] + .join(' | ') + .replace(/^/, '| ') + .replace(/$/, ' |') + ); + } + } + + lines.push(''); + return `${lines.join('\n')}\n`; +}; + +const resolveOutputPaths = args => { + const outPath = path.resolve(rootDir, args.out); + const format = args.format === 'markdown' ? 'md' : args.format; + const writeJson = format === 'json' || format === 'both'; + const writeMd = format === 'md' || format === 'both'; + + if (writeJson && writeMd) { + return { + artifactRoot: outPath, + jsonPath: path.join(outPath, 'baseline.json'), + mdPath: path.join(outPath, 'baseline.md'), + outPath, + writeJson, + writeMd, + }; + } + + const artifactRoot = path.extname(outPath) + ? path.join(path.dirname(outPath), `${path.basename(outPath)}.artifacts`) + : `${outPath}.artifacts`; + + return { + artifactRoot, + jsonPath: writeJson ? outPath : null, + mdPath: writeMd ? outPath : null, + outPath, + writeJson, + writeMd, + }; +}; + +const writeOutputs = async (result, outputPaths) => { + const { jsonPath, mdPath, outPath, writeJson, writeMd } = outputPaths; + + if (writeJson && writeMd) { + await mkdir(outPath, { recursive: true }); + await writeFile(jsonPath, `${JSON.stringify(result, null, 2)}\n`); + await writeFile(mdPath, renderMarkdown(result)); + return; + } + + await mkdir(path.dirname(outPath), { recursive: true }); + if (writeJson) { + await writeFile(jsonPath, `${JSON.stringify(result, null, 2)}\n`); + } else { + await writeFile(mdPath, renderMarkdown(result)); + } +}; + +const formatParallelTransforms = parallelTransforms => { + if (parallelTransforms === undefined) { + return 'default'; + } + if (!parallelTransforms) { + return 'false'; + } + if (parallelTransforms === true) { + return 'true'; + } + return `maxWorkers=${parallelTransforms.maxWorkers}`; +}; + +const git = async args => { + const result = await runCommand({ + command: 'git', + args, + cwd: rootDir, + useTime: false, + }); + return result.status === 0 ? result.stdout.trim() : null; +}; + +const pnpmVersion = async () => { + const result = await runCommand({ + command: 'pnpm', + args: ['--version'], + cwd: rootDir, + useTime: false, + }); + return result.status === 0 ? result.stdout.trim() : null; +}; + +const cleanBuildOutputs = async fixtureRoot => { + await Promise.all([ + rm(path.join(fixtureRoot, 'build'), { recursive: true, force: true }), + rm(path.join(fixtureRoot, '.react-router'), { + recursive: true, + force: true, + }), + ]); +}; + +const listRspackProfileDirs = async cwd => { + const entries = await readdir(cwd, { withFileTypes: true }); + return entries + .filter( + entry => entry.isDirectory() && entry.name.startsWith('.rspack-profile-') + ) + .map(entry => entry.name) + .sort(); +}; + +const moveDirectory = async (source, destination) => { + await rm(destination, { recursive: true, force: true }); + await mkdir(path.dirname(destination), { recursive: true }); + try { + await rename(source, destination); + } catch (error) { + if (error?.code !== 'EXDEV') { + throw error; + } + await cp(source, destination, { recursive: true }); + await rm(source, { recursive: true, force: true }); + } +}; + +const collectRspackProfiles = async ({ + fixtureRoot, + beforeProfiles, + destinationRoot, +}) => { + const before = new Set(beforeProfiles); + const afterProfiles = await listRspackProfileDirs(fixtureRoot); + const createdProfiles = afterProfiles.filter(profile => !before.has(profile)); + const collected = []; + + for (const profile of createdProfiles) { + const source = path.join(fixtureRoot, profile); + const destination = path.join(destinationRoot, profile.slice(1)); + await moveDirectory(source, destination); + collected.push(path.relative(rootDir, destination)); + } + + return collected; +}; + +const isTraceOutputStream = value => value === 'stdout' || value === 'stderr'; + +const resolveRspackTraceOutput = async ({ + traceOutput, + benchmarkId, + runLabel, +}) => { + if (!traceOutput || isTraceOutputStream(traceOutput)) { + return traceOutput; + } + + const tracePath = path.resolve( + rootDir, + traceOutput, + benchmarkId, + `${runLabel}.log` + ); + await mkdir(path.dirname(tracePath), { recursive: true }); + return tracePath; +}; + +const main = async () => { + const args = parseArgs(process.argv.slice(2)); + const useTime = await hasGnuTime(); + const outputPaths = resolveOutputPaths(args); + const pluginImportPath = pathToFileURL( + path.join(rootDir, 'dist/index.js') + ).href; + const pluginReactImportPath = + process.env.REACT_ROUTER_BENCHMARK_PLUGIN_REACT_IMPORT ?? + '@rsbuild/plugin-react'; + const selectedBenchmarks = profiles[args.profile].filter(benchmark => + args.filter ? benchmark.id.includes(args.filter) : true + ); + + if (selectedBenchmarks.length === 0) { + throw new Error(`No benchmarks matched filter "${args.filter}".`); + } + + if (!args.skipRootBuild) { + console.log('Building plugin package before benchmarks...'); + const buildResult = await runCommand({ + command: 'pnpm', + args: ['build'], + cwd: rootDir, + }); + if (buildResult.status !== 0) { + process.exit(buildResult.status ?? 1); + } + } + + const benchmarks = []; + for (const benchmark of selectedBenchmarks) { + const fixtureRoot = path.join(benchmarkRoot, 'fixtures', benchmark.id); + await generateSyntheticFixture({ + root: fixtureRoot, + routeCount: benchmark.routeCount, + variant: benchmark.variant, + sourceMap: benchmark.sourceMap ?? false, + pluginImportPath, + pluginReactImportPath, + parallelTransforms: args.parallelTransforms, + }); + + const runs = []; + const totalRuns = args.warmup + args.iterations; + for (let index = 0; index < totalRuns; index += 1) { + const measured = index >= args.warmup; + if (args.clean !== 'none') { + await cleanBuildOutputs(fixtureRoot); + } + if (args.clean === 'cold') { + await rm(path.join(fixtureRoot, 'node_modules'), { + recursive: true, + force: true, + }); + } + + console.log( + `${measured ? 'Measuring' : 'Warming'} ${benchmark.id} (${index + 1}/${totalRuns})` + ); + const rspackProfileEnabled = Boolean(args.rspackProfile); + const beforeRspackProfiles = rspackProfileEnabled + ? await listRspackProfileDirs(fixtureRoot) + : []; + const runLabel = `${measured ? 'run' : 'warmup'}-${ + measured ? index - args.warmup + 1 : index + 1 + }`; + const rspackTraceOutput = await resolveRspackTraceOutput({ + traceOutput: args.rspackTraceOutput, + benchmarkId: benchmark.id, + runLabel, + }); + const commandResult = await runCommand({ + command: process.execPath, + args: [rsbuildBin, 'build', '--config', 'rsbuild.config.mjs'], + cwd: fixtureRoot, + env: { + NODE_ENV: 'production', + REACT_ROUTER_BENCHMARK_LOG_PERFORMANCE: '1', + ...(args.rspackProfile ? { RSPACK_PROFILE: args.rspackProfile } : {}), + ...(rspackTraceOutput + ? { RSPACK_TRACE_OUTPUT: rspackTraceOutput } + : {}), + }, + useTime, + }); + const rspackProfiles = rspackProfileEnabled + ? await collectRspackProfiles({ + fixtureRoot, + beforeProfiles: beforeRspackProfiles, + destinationRoot: path.join( + outputPaths.artifactRoot, + 'rspack-profiles', + benchmark.id, + runLabel + ), + }) + : []; + const timeStats = useTime ? parseTimeStats(commandResult.stderr) : {}; + const pluginReports = parsePluginReports( + `${commandResult.stdout}\n${commandResult.stderr}` + ); + + if (commandResult.status !== 0 && args.failFast) { + process.exit(commandResult.status ?? 1); + } + + if (measured) { + runs.push({ + status: commandResult.status, + wallMs: commandResult.wallMs, + userMs: timeStats.userMs ?? null, + sysMs: timeStats.sysMs ?? null, + maxRssKb: timeStats.maxRssKb ?? null, + pluginReports, + rspackProfiles, + rspackTraceOutput: + rspackTraceOutput && !isTraceOutputStream(rspackTraceOutput) + ? path.relative(rootDir, rspackTraceOutput) + : rspackTraceOutput, + }); + } + } + + benchmarks.push({ + ...benchmark, + parallelTransforms: args.parallelTransforms, + cwd: path.relative(rootDir, fixtureRoot), + command: + 'node /node_modules/@rsbuild/core/bin/rsbuild.js build --config rsbuild.config.mjs', + runs, + summary: summarizeRuns(runs), + pluginOperations: summarizePluginOperations(runs), + }); + } + + const failed = benchmarks.some(benchmark => + benchmark.runs.some(run => run.status !== 0) + ); + const result = { + repo: 'rsbuild-plugin-react-router', + commit: await git(['rev-parse', 'HEAD']), + date: new Date().toISOString(), + node: process.version, + pnpm: await pnpmVersion(), + platform: `${os.platform()} ${os.release()} ${os.arch()}`, + profile: args.profile, + iterations: args.iterations, + warmup: args.warmup, + parallelTransforms: args.parallelTransforms, + rspackProfile: args.rspackProfile, + rspackTraceOutput: args.rspackTraceOutput, + failed, + benchmarks, + }; + + await writeOutputs(result, outputPaths); + console.log(`Benchmark results written to ${outputPaths.outPath}`); + + if (failed) { + console.error('One or more measured benchmark builds failed.'); + process.exitCode = 1; + } +}; + +main().catch(error => { + console.error(error); + process.exit(1); +}); diff --git a/scripts/benchmark/fixture.mjs b/scripts/benchmark/fixture.mjs new file mode 100644 index 0000000..5869b22 --- /dev/null +++ b/scripts/benchmark/fixture.mjs @@ -0,0 +1,454 @@ +import { mkdir, rm, writeFile } from 'node:fs/promises'; +import path from 'node:path'; + +const routeExportProfiles = [ + 'plain', + 'ssr-data', + 'split-client', + 'split-client', + 'ssr-data', + 'client-server-imports', +]; + +export const benchmarkFixtureNames = [ + 'default', + 'export-heavy', + 'reexports', + 'import-fanout', + 'chunk-saturated', +]; + +const stressFixtureNames = new Set(benchmarkFixtureNames); + +export const padRoute = number => String(number).padStart(4, '0'); + +export const routeFile = index => `routes/route-${padRoute(index)}.tsx`; + +export const routeId = index => `route-${padRoute(index)}`; + +const routeComponentName = index => `Route${padRoute(index)}`; + +const createSharedRouteExports = (index, { includeHeaders = false } = {}) => { + const name = routeComponentName(index); + return [ + `export const handle = { label: '${routeId(index)}' };`, + `export function meta() { return [{ title: '${routeId(index)}' }]; }`, + `export default function ${name}() { return null; }`, + ...(includeHeaders + ? [ + `export function headers() { return { 'x-route': '${routeId(index)}' }; }`, + ] + : []), + ]; +}; + +const createDefaultRouteModule = (index, profile, { isSpa }) => { + const shared = createSharedRouteExports(index); + + if (profile === 'ssr-data') { + if (isSpa) { + return [ + ...shared, + `export function shouldRevalidate() { return false; }`, + ].join('\n'); + } + + return [ + `import { serverValue } from '../server-data.server';`, + ...shared, + `export async function loader() { return { id: '${routeId(index)}', serverValue }; }`, + `export async function action() { return { ok: true }; }`, + `export function headers() { return { 'x-route': '${routeId(index)}' }; }`, + `export function shouldRevalidate() { return false; }`, + ].join('\n'); + } + + if (profile === 'split-client') { + return [ + `import { clientValue } from '../client-data.client';`, + ...shared, + `export async function clientLoader() { return { id: '${routeId(index)}', clientValue }; }`, + `export async function clientAction() { return { ok: true }; }`, + `export async function clientMiddleware() { return undefined; }`, + ...(isSpa ? [] : [`export function HydrateFallback() { return null; }`]), + ].join('\n'); + } + + if (profile === 'client-server-imports') { + if (isSpa) { + return [ + `import { clientValue } from '../client-data.client';`, + ...shared, + `export async function clientLoader() { return { id: '${routeId(index)}', clientValue }; }`, + ].join('\n'); + } + + return [ + `import { clientValue } from '../client-data.client';`, + `import { serverValue } from '../server-data.server';`, + ...shared, + `export async function loader() { return { id: '${routeId(index)}', serverValue }; }`, + `export async function clientLoader() { return { id: '${routeId(index)}', clientValue }; }`, + ].join('\n'); + } + + return shared.join('\n'); +}; + +const createExportHeavyRouteModule = (index, { isSpa }) => { + const extraExports = Array.from({ length: 32 }, (_, extraIndex) => { + const exportName = `unusedExport${padRoute(index)}_${String(extraIndex).padStart(2, '0')}`; + return `export const ${exportName} = '${routeId(index)}-${extraIndex}';`; + }); + + return [ + `import { clientValue } from '../client-data.client';`, + ...(isSpa ? [] : [`import { serverValue } from '../server-data.server';`]), + ...createSharedRouteExports(index, { includeHeaders: !isSpa }), + `export function links() { return []; }`, + `export function shouldRevalidate() { return false; }`, + `export async function clientLoader() { return { id: '${routeId(index)}', clientValue }; }`, + `export async function clientAction() { return { ok: true }; }`, + `export async function clientMiddleware() { return undefined; }`, + ...(isSpa + ? [] + : [ + `export async function loader() { return { id: '${routeId(index)}', serverValue }; }`, + `export async function action() { return { ok: true }; }`, + `export function HydrateFallback() { return null; }`, + ]), + ...extraExports, + ].join('\n'); +}; + +const createReexportsRouteModule = (index, { isSpa }) => + [ + `export { default, handle, meta, shouldRevalidate } from '../route-reexports/reexport-${padRoute(index)}';`, + `export { clientLoader, clientAction, clientMiddleware${isSpa ? '' : ', HydrateFallback'} } from '../route-reexports/reexport-${padRoute(index)}';`, + ...(isSpa + ? [] + : [ + `export { loader, action, headers } from '../route-reexports/reexport-${padRoute(index)}';`, + ]), + `export * from '../route-reexports/reexport-all-${padRoute(index)}';`, + ].join('\n'); + +const createImportFanoutRouteModule = (index, { isSpa }) => { + const imports = Array.from({ length: 16 }, (_, fanoutIndex) => { + const suffix = String(fanoutIndex).padStart(2, '0'); + return `import { fanoutValue${suffix} } from '../fanout/fanout-${suffix}';`; + }); + const values = Array.from( + { length: 16 }, + (_, fanoutIndex) => `fanoutValue${String(fanoutIndex).padStart(2, '0')}` + ).join(', '); + + return [ + ...imports, + ...createSharedRouteExports(index, { includeHeaders: !isSpa }), + `const fanoutValues = [${values}];`, + `export function shouldRevalidate() { return fanoutValues.length > ${index % 7}; }`, + `export async function clientLoader() { return { values: fanoutValues }; }`, + ...(isSpa + ? [] + : [ + `export async function loader() { return { values: fanoutValues }; }`, + ]), + ].join('\n'); +}; + +const createChunkSaturatedRouteModule = (index, { isSpa }) => + [ + `import { clientValue } from '../client-data.client';`, + ...(isSpa ? [] : [`import { serverValue } from '../server-data.server';`]), + ...createSharedRouteExports(index, { includeHeaders: !isSpa }), + `const routeLabel = '${routeId(index)}';`, + `export function shouldRevalidate() { return routeLabel.length > 0; }`, + `export async function clientLoader() { return { routeLabel, clientValue }; }`, + `export async function clientAction() { return { ok: routeLabel, clientValue }; }`, + `export async function clientMiddleware() { return undefined; }`, + ...(isSpa + ? [] + : [ + `export function HydrateFallback() { return null; }`, + `export async function loader() { return { routeLabel, serverValue }; }`, + `export async function action() { return { ok: true, serverValue }; }`, + ]), + ].join('\n'); + +const createRouteModule = (index, profile, { isSpa, fixture }) => { + if (fixture === 'export-heavy') { + return createExportHeavyRouteModule(index, { isSpa }); + } + if (fixture === 'reexports') { + return createReexportsRouteModule(index, { isSpa }); + } + if (fixture === 'import-fanout') { + return createImportFanoutRouteModule(index, { isSpa }); + } + if (fixture === 'chunk-saturated') { + return createChunkSaturatedRouteModule(index, { isSpa }); + } + return createDefaultRouteModule(index, profile, { isSpa }); +}; + +const createRoutesConfig = routeCount => { + const routes = []; + for (let index = 1; index <= routeCount; index += 1) { + const id = routeId(index); + const isIndex = index === 1; + routes.push( + [ + ' {', + ` id: '${id}',`, + ` file: '${routeFile(index)}',`, + isIndex ? ' index: true,' : ` path: '${id}',`, + ' },', + ].join('\n') + ); + } + + return [ + `import type { RouteConfigEntry } from '@react-router/dev/routes';`, + '', + 'export default [', + ...routes, + '] satisfies RouteConfigEntry[];', + '', + ].join('\n'); +}; + +const renderParallelTransformsOption = parallelTransforms => { + if (parallelTransforms === undefined) { + return []; + } + if (parallelTransforms === false) { + return [` parallelTransforms: false,`]; + } + if (parallelTransforms === true) { + return [` parallelTransforms: true,`]; + } + return [ + ` parallelTransforms: { maxWorkers: ${parallelTransforms.maxWorkers} },`, + ]; +}; + +const createRsbuildConfig = ({ + variant, + sourceMap, + pluginImportPath, + pluginReactImportPath, + parallelTransforms, +}) => { + const ssr = variant !== 'spa'; + const lazyCompilationOption = + ` ...(process.env.REACT_ROUTER_BENCHMARK_LAZY_COMPILATION === '0'` + + ` ? { lazyCompilation: false }` + + ` : process.env.REACT_ROUTER_BENCHMARK_LAZY_COMPILATION === '1'` + + ` ? { lazyCompilation: true } : {}),`; + + return [ + `import { defineConfig } from '@rsbuild/core';`, + `import { pluginReact } from '${pluginReactImportPath}';`, + `import { pluginReactRouter } from '${pluginImportPath}';`, + '', + 'export default defineConfig({', + ' plugins: [', + ' pluginReact(),', + ' pluginReactRouter({', + ...(ssr ? [` serverOutput: 'module',`] : []), + ...renderParallelTransformsOption(parallelTransforms), + lazyCompilationOption, + ` logPerformance: process.env.REACT_ROUTER_BENCHMARK_LOG_PERFORMANCE === '1',`, + ' }),', + ' ],', + ' output: {', + ` sourceMap: ${ + sourceMap ? `{ js: 'cheap-module-source-map', css: false }` : 'false' + },`, + ' },', + '});', + '', + ].join('\n'); +}; + +const createReactRouterConfig = variant => { + const ssr = variant !== 'spa'; + const splitRouteModules = variant.includes('split'); + + return [ + `import type { Config } from '@react-router/dev/config';`, + '', + 'export default {', + ` ssr: ${ssr ? 'true' : 'false'},`, + ` future: {`, + ` v8_splitRouteModules: ${splitRouteModules ? 'true' : 'false'},`, + ' },', + '} satisfies Config;', + '', + ].join('\n'); +}; + +const writeReexportFixtures = async (root, routeCount, { isSpa }) => { + await mkdir(path.join(root, 'app/route-reexports'), { recursive: true }); + const batchSize = 64; + for (let batchStart = 0; batchStart < routeCount; batchStart += batchSize) { + await Promise.all( + Array.from( + { length: Math.min(batchSize, routeCount - batchStart) }, + (_, batchIndex) => { + const routeIndex = batchStart + batchIndex; + const index = routeIndex + 1; + const module = [ + `import { clientValue } from '../client-data.client';`, + ...(isSpa + ? [] + : [`import { serverValue } from '../server-data.server';`]), + ...createSharedRouteExports(index, { includeHeaders: !isSpa }), + `export function shouldRevalidate() { return false; }`, + `export async function clientLoader() { return { id: '${routeId(index)}', clientValue }; }`, + `export async function clientAction() { return { ok: true }; }`, + `export async function clientMiddleware() { return undefined; }`, + ...(isSpa + ? [] + : [ + `export async function loader() { return { id: '${routeId(index)}', serverValue }; }`, + `export async function action() { return { ok: true }; }`, + `export function HydrateFallback() { return null; }`, + ]), + ].join('\n'); + const exportAllModule = [ + `export const reexportedValue${padRoute(index)} = '${routeId(index)}';`, + `export function reexportedHelper${padRoute(index)}() { return reexportedValue${padRoute(index)}; }`, + ].join('\n'); + return Promise.all([ + writeFile( + path.join( + root, + `app/route-reexports/reexport-${padRoute(index)}.ts` + ), + `${module}\n` + ), + writeFile( + path.join( + root, + `app/route-reexports/reexport-all-${padRoute(index)}.ts` + ), + `${exportAllModule}\n` + ), + ]); + } + ) + ); + } +}; + +const writeFanoutFixtures = async root => { + await mkdir(path.join(root, 'app/fanout'), { recursive: true }); + await Promise.all( + Array.from({ length: 16 }, (_, fanoutIndex) => { + const suffix = String(fanoutIndex).padStart(2, '0'); + return writeFile( + path.join(root, `app/fanout/fanout-${suffix}.ts`), + `export const fanoutValue${suffix} = '${suffix}';\n` + ); + }) + ); +}; + +export async function generateSyntheticFixture({ + root, + routeCount, + variant, + sourceMap = false, + pluginImportPath = 'rsbuild-plugin-react-router', + pluginReactImportPath = '@rsbuild/plugin-react', + fixture = 'default', + parallelTransforms, +}) { + if (!stressFixtureNames.has(fixture)) { + throw new Error( + `Unknown benchmark fixture "${fixture}". Use ${benchmarkFixtureNames.join(', ')}.` + ); + } + + const isSpa = variant === 'spa'; + + await rm(root, { recursive: true, force: true }); + await mkdir(path.join(root, 'app/routes'), { recursive: true }); + + await writeFile( + path.join(root, 'package.json'), + JSON.stringify({ type: 'module', private: true }, null, 2) + ); + await writeFile( + path.join(root, 'rsbuild.config.mjs'), + createRsbuildConfig({ + variant, + sourceMap, + pluginImportPath, + pluginReactImportPath, + parallelTransforms, + }) + ); + await writeFile( + path.join(root, 'react-router.config.ts'), + createReactRouterConfig(variant) + ); + await writeFile( + path.join(root, 'app/routes.ts'), + createRoutesConfig(routeCount) + ); + await writeFile( + path.join(root, 'app/root.tsx'), + [ + `import { createElement } from 'react';`, + `import { Outlet, Scripts } from 'react-router';`, + `export function Layout({ children }) {`, + ` return createElement('html', null, createElement('head'), createElement('body', null, children, createElement(Scripts)));`, + `}`, + `export default function Root() { return createElement(Outlet); }`, + `export function ErrorBoundary() { return null; }`, + '', + ].join('\n') + ); + await writeFile( + path.join(root, 'app/client-data.client.ts'), + `export const clientValue = 'client';\nexport * from './client-extra.client';\n` + ); + await writeFile( + path.join(root, 'app/client-extra.client.ts'), + `export const extraClientValue = 'extra-client';\n` + ); + await writeFile( + path.join(root, 'app/server-data.server.ts'), + `export const serverValue = 'server';\n` + ); + + if (fixture === 'reexports') { + await writeReexportFixtures(root, routeCount, { isSpa }); + } + if (fixture === 'import-fanout') { + await writeFanoutFixtures(root); + } + + await Promise.all( + Array.from({ length: routeCount }, (_, routeIndex) => { + const index = routeIndex + 1; + const profile = routeExportProfiles[index % routeExportProfiles.length]; + return writeFile( + path.join(root, 'app', routeFile(index)), + `${createRouteModule(index, profile, { isSpa, fixture })}\n` + ); + }) + ); + + return { + root, + routeCount, + variant, + sourceMap, + fixture, + parallelTransforms, + }; +} diff --git a/scripts/compare-benchmarks.mjs b/scripts/compare-benchmarks.mjs new file mode 100644 index 0000000..31192ac --- /dev/null +++ b/scripts/compare-benchmarks.mjs @@ -0,0 +1,128 @@ +#!/usr/bin/env node +import { readFile } from 'node:fs/promises'; +import { parseArgs } from 'node:util'; + +const { values } = parseArgs({ + allowPositionals: false, + strict: true, + options: { + before: { type: 'string' }, + after: { type: 'string' }, + benchmark: { type: 'string', default: 'synthetic-256-ssr-esm-split' }, + operations: { + type: 'string', + default: 'route:chunk,route:client-entry,route:split-exports', + }, + }, +}); + +if (!values.before || !values.after) { + throw new Error( + 'Usage: node scripts/compare-benchmarks.mjs --before --after [--benchmark ] [--operations op,op]' + ); +} + +const readJson = async file => JSON.parse(await readFile(file, 'utf8')); +const before = await readJson(values.before); +const after = await readJson(values.after); +const operations = new Set(values.operations.split(',').filter(Boolean)); + +const findBenchmark = (result, id) => { + const benchmark = result.benchmarks?.find(item => item.id === id); + if (!benchmark) { + throw new Error( + `Benchmark "${id}" not found in ${result.date ?? 'input'}.` + ); + } + return benchmark; +}; + +const metric = (benchmark, path) => + path.split('.').reduce((value, key) => value?.[key], benchmark); + +const operationMetric = (benchmark, operation, key) => { + const matches = + benchmark.pluginOperations?.filter(item => item.operation === operation) ?? + []; + const values = matches + .map(item => item[key]) + .filter(value => typeof value === 'number'); + if (values.length === 0) { + return null; + } + return values.reduce((sum, value) => sum + value, 0); +}; + +const percentDelta = (beforeValue, afterValue) => { + if (beforeValue == null || afterValue == null || beforeValue === 0) { + return '-'; + } + return `${(((afterValue - beforeValue) / beforeValue) * 100).toFixed(1)}%`; +}; + +const formatNumber = value => (value == null ? '-' : value.toFixed(1)); +const formatMs = value => + value == null ? '-' : `${(value / 1000).toFixed(2)}s`; +const formatKb = value => + value == null ? '-' : `${Math.round(value / 1024)} MB`; + +const beforeBenchmark = findBenchmark(before, values.benchmark); +const afterBenchmark = findBenchmark(after, values.benchmark); + +const rows = [ + { + label: 'Wall median', + before: metric(beforeBenchmark, 'summary.wallMs.median'), + after: metric(afterBenchmark, 'summary.wallMs.median'), + format: formatMs, + }, + { + label: 'CPU median (user+sys)', + before: + metric(beforeBenchmark, 'summary.userMs.median') == null || + metric(beforeBenchmark, 'summary.sysMs.median') == null + ? null + : metric(beforeBenchmark, 'summary.userMs.median') + + metric(beforeBenchmark, 'summary.sysMs.median'), + after: + metric(afterBenchmark, 'summary.userMs.median') == null || + metric(afterBenchmark, 'summary.sysMs.median') == null + ? null + : metric(afterBenchmark, 'summary.userMs.median') + + metric(afterBenchmark, 'summary.sysMs.median'), + format: formatMs, + }, + { + label: 'Peak RSS p95', + before: metric(beforeBenchmark, 'summary.maxRssKb.p95'), + after: metric(afterBenchmark, 'summary.maxRssKb.p95'), + format: formatKb, + }, +]; + +for (const operation of operations) { + rows.push( + { + label: `${operation} totalMs`, + before: operationMetric(beforeBenchmark, operation, 'totalMs'), + after: operationMetric(afterBenchmark, operation, 'totalMs'), + format: formatNumber, + }, + { + label: `${operation} wallMs`, + before: operationMetric(beforeBenchmark, operation, 'wallMs'), + after: operationMetric(afterBenchmark, operation, 'wallMs'), + format: formatNumber, + } + ); +} + +console.log(`Benchmark comparison: ${values.benchmark}`); +console.log(''); +console.log('| Metric | Before | After | Delta |'); +console.log('|---|---:|---:|---:|'); +for (const row of rows) { + console.log( + `| ${row.label} | ${row.format(row.before)} | ${row.format(row.after)} | ${percentDelta(row.before, row.after)} |` + ); +} diff --git a/scripts/report-benchmark-ci.mjs b/scripts/report-benchmark-ci.mjs new file mode 100644 index 0000000..2bd3154 --- /dev/null +++ b/scripts/report-benchmark-ci.mjs @@ -0,0 +1,167 @@ +#!/usr/bin/env node +import { mkdir, readFile, writeFile } from 'node:fs/promises'; +import path from 'node:path'; +import { parseArgs } from 'node:util'; + +const { values } = parseArgs({ + allowPositionals: false, + strict: true, + options: { + base: { type: 'string' }, + head: { type: 'string' }, + out: { type: 'string', default: '.benchmark/ci-report' }, + pr: { type: 'string' }, + 'base-ref': { type: 'string' }, + 'base-sha': { type: 'string' }, + 'head-ref': { type: 'string' }, + 'head-sha': { type: 'string' }, + 'run-url': { type: 'string' }, + }, +}); + +if (!values.base || !values.head) { + throw new Error( + 'Usage: node scripts/report-benchmark-ci.mjs --base --head [--out ]' + ); +} + +const readJson = async file => JSON.parse(await readFile(file, 'utf8')); +const formatSeconds = value => + typeof value === 'number' ? `${(value / 1000).toFixed(2)}s` : '-'; +const formatPercent = value => + typeof value === 'number' + ? `${value > 0 ? '+' : ''}${value.toFixed(1)}%` + : '-'; +const formatSpeedup = value => + typeof value === 'number' ? `${value.toFixed(2)}x` : '-'; +const formatRss = value => + typeof value === 'number' ? `${Math.round(value / 1024)} MB` : '-'; + +const percentDelta = (base, head) => + typeof base === 'number' && typeof head === 'number' && base !== 0 + ? ((head - base) / base) * 100 + : null; + +const speedup = (base, head) => + typeof base === 'number' && typeof head === 'number' && head !== 0 + ? base / head + : null; + +const medianWall = benchmark => benchmark?.summary?.wallMs?.median ?? null; +const p95Rss = benchmark => benchmark?.summary?.maxRssKb?.p95 ?? null; +const cpuMedian = benchmark => { + const user = benchmark?.summary?.userMs?.median; + const sys = benchmark?.summary?.sysMs?.median; + return typeof user === 'number' && typeof sys === 'number' + ? user + sys + : null; +}; + +const indexBenchmarks = result => + new Map( + (result.benchmarks ?? []).map(benchmark => [benchmark.id, benchmark]) + ); + +const base = await readJson(values.base); +const head = await readJson(values.head); +const baseBenchmarks = indexBenchmarks(base); +const headBenchmarks = indexBenchmarks(head); +const benchmarkIds = [ + ...new Set([...baseBenchmarks.keys(), ...headBenchmarks.keys()]), +].sort(); + +const benchmarks = benchmarkIds.map(id => { + const baseBenchmark = baseBenchmarks.get(id); + const headBenchmark = headBenchmarks.get(id); + const baseWallMs = medianWall(baseBenchmark); + const headWallMs = medianWall(headBenchmark); + return { + id, + routeCount: headBenchmark?.routeCount ?? baseBenchmark?.routeCount ?? null, + variant: headBenchmark?.variant ?? baseBenchmark?.variant ?? null, + baseWallMs, + headWallMs, + wallDeltaPercent: percentDelta(baseWallMs, headWallMs), + wallSpeedup: speedup(baseWallMs, headWallMs), + baseCpuMs: cpuMedian(baseBenchmark), + headCpuMs: cpuMedian(headBenchmark), + baseRssKb: p95Rss(baseBenchmark), + headRssKb: p95Rss(headBenchmark), + }; +}); + +const totalBaseWallMs = benchmarks.reduce( + (sum, benchmark) => sum + (benchmark.baseWallMs ?? 0), + 0 +); +const totalHeadWallMs = benchmarks.reduce( + (sum, benchmark) => sum + (benchmark.headWallMs ?? 0), + 0 +); + +const summary = { + baseWallMs: totalBaseWallMs, + headWallMs: totalHeadWallMs, + wallDeltaPercent: percentDelta(totalBaseWallMs, totalHeadWallMs), + wallSpeedup: speedup(totalBaseWallMs, totalHeadWallMs), +}; + +const report = { + generatedAt: new Date().toISOString(), + pullRequest: values.pr ?? null, + base: { + ref: values['base-ref'] ?? null, + sha: values['base-sha'] ?? base.commit ?? null, + benchmarkCommit: base.commit ?? null, + }, + head: { + ref: values['head-ref'] ?? null, + sha: values['head-sha'] ?? head.commit ?? null, + benchmarkCommit: head.commit ?? null, + }, + runUrl: values['run-url'] ?? null, + profile: head.profile ?? base.profile ?? null, + iterations: head.iterations ?? base.iterations ?? null, + warmup: head.warmup ?? base.warmup ?? null, + summary, + benchmarks, +}; + +const renderComment = () => { + const lines = [ + '', + '## Benchmark Results', + '', + `Compared PR head \`${report.head.sha?.slice(0, 7) ?? 'unknown'}\` against base \`${report.base.sha?.slice(0, 7) ?? 'unknown'}\`.`, + '', + `**Total median wall time:** ${formatSeconds(summary.baseWallMs)} -> ${formatSeconds(summary.headWallMs)} (${formatPercent(summary.wallDeltaPercent)}, ${formatSpeedup(summary.wallSpeedup)} speedup)`, + '', + '| Benchmark | Base | Head | Delta | Speedup | Head RSS p95 |', + '|---|---:|---:|---:|---:|---:|', + ]; + + for (const benchmark of benchmarks) { + lines.push( + `| \`${benchmark.id}\` | ${formatSeconds(benchmark.baseWallMs)} | ${formatSeconds(benchmark.headWallMs)} | ${formatPercent(benchmark.wallDeltaPercent)} | ${formatSpeedup(benchmark.wallSpeedup)} | ${formatRss(benchmark.headRssKb)} |` + ); + } + + lines.push( + '', + `Profile: \`${report.profile ?? 'unknown'}\`; iterations: \`${report.iterations ?? 'unknown'}\`; warmup: \`${report.warmup ?? 'unknown'}\`.`, + ...(report.runUrl ? [`[Workflow run](${report.runUrl})`] : []), + '' + ); + + return `${lines.join('\n')}\n`; +}; + +const outDir = path.resolve(values.out); +await mkdir(outDir, { recursive: true }); +await writeFile( + path.join(outDir, 'report.json'), + `${JSON.stringify(report, null, 2)}\n` +); +await writeFile(path.join(outDir, 'comment.md'), renderComment()); + +console.log(`Benchmark CI report written to ${outDir}`); From 000435904e4ce41a670832cc03957e548145543c Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 21 Jun 2026 02:12:31 +0200 Subject: [PATCH 2/5] ci: fix benchmark pnpm setup --- .github/workflows/benchmark.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 0b79f33..5278739 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -46,6 +46,8 @@ jobs: - name: Setup PNPM uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 + with: + package_json_file: head/package.json - name: Get pnpm store directory id: pnpm-cache From 23a886792baa0f15b7572fa9833f6f911cb7d6ac Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 21 Jun 2026 02:20:56 +0200 Subject: [PATCH 3/5] ci: address benchmark review feedback --- .github/workflows/benchmark.yml | 1 + benchmarks/README.md | 7 +-- scripts/bench-builds.mjs | 104 +------------------------------- scripts/benchmark/fixture.mjs | 1 - scripts/compare-benchmarks.mjs | 38 +----------- 5 files changed, 7 insertions(+), 144 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 5278739..7efd1c7 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -132,6 +132,7 @@ jobs: if-no-files-found: error - name: Comment benchmark results + if: github.event.pull_request.head.repo.full_name == github.repository env: GH_TOKEN: ${{ github.token }} PR_NUMBER: ${{ github.event.pull_request.number }} diff --git a/benchmarks/README.md b/benchmarks/README.md index 8cb14a0..f0bee00 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -18,7 +18,7 @@ counts for stress testing. All benchmark profiles generate deterministic synthetic React Router apps under `.benchmark/fixtures/`, build the current plugin package once, then run Rsbuild -builds with `pluginReactRouter({ logPerformance: true })`. +production builds against those fixtures. To capture Rspack tracing output for a benchmark, pass `--rspack-profile`: @@ -58,9 +58,8 @@ Each run writes: - `.benchmark/results//baseline.md` The JSON includes wall time, optional GNU `/usr/bin/time -v` user/sys/RSS data, -parsed `[react-router:performance]` reports from the plugin, and an aggregated -`pluginOperations` table per fixture. The markdown report includes the same -operation breakdown so route transforms and manifest work can be compared +per-run exit status, and Rspack trace artifact paths when tracing is enabled. +The markdown report summarizes the same benchmark-level timing and memory data without opening the raw JSON. ## Hygiene diff --git a/scripts/bench-builds.mjs b/scripts/bench-builds.mjs index 062ba11..173e40a 100644 --- a/scripts/bench-builds.mjs +++ b/scripts/bench-builds.mjs @@ -201,26 +201,6 @@ const parseTimeStats = stderr => { }; }; -const parsePluginReports = output => { - const reports = []; - for (const line of output.split(/\r?\n/)) { - const markerIndex = line.indexOf('[react-router:performance]'); - if (markerIndex === -1) { - continue; - } - const jsonStart = line.indexOf('{', markerIndex); - if (jsonStart === -1) { - continue; - } - try { - reports.push(JSON.parse(line.slice(jsonStart))); - } catch { - // Keep raw build output useful even if one line is malformed. - } - } - return reports; -}; - const summarizeMetric = values => { const sorted = values .filter(value => typeof value === 'number') @@ -251,49 +231,8 @@ const summarizeRuns = runs => ({ maxRssKb: summarizeMetric(runs.map(run => run.maxRssKb)), }); -const summarizePluginOperations = runs => { - const operations = new Map(); - - for (const run of runs) { - for (const report of run.pluginReports) { - for (const [operation, metrics] of Object.entries( - report.operations ?? {} - )) { - const key = `${report.environment}:${operation}`; - const current = operations.get(key) ?? { - environment: report.environment, - operation, - count: 0, - totalMs: 0, - wallMs: null, - maxMs: 0, - reports: 0, - }; - current.count += metrics.count ?? 0; - current.totalMs += metrics.totalMs ?? 0; - if (typeof metrics.wallMs === 'number') { - current.wallMs = (current.wallMs ?? 0) + metrics.wallMs; - } - current.maxMs = Math.max(current.maxMs, metrics.maxMs ?? 0); - current.reports += 1; - operations.set(key, current); - } - } - } - - return [...operations.values()].sort((a, b) => { - if (b.totalMs !== a.totalMs) { - return b.totalMs - a.totalMs; - } - return `${a.environment}:${a.operation}`.localeCompare( - `${b.environment}:${b.operation}` - ); - }); -}; - const formatMs = value => value == null ? '-' : `${(value / 1000).toFixed(2)}s`; -const formatReportMs = value => (value == null ? '-' : `${value.toFixed(1)}ms`); const formatRss = value => value == null ? '-' : `${Math.round(value / 1024)} MB`; @@ -315,8 +254,8 @@ const renderMarkdown = result => { ? [`- Rspack trace output: ${result.rspackTraceOutput}`] : []), '', - '| Benchmark | Routes | Variant | Median wall | Mean wall | p95 wall | Max RSS | Plugin reports |', - '|---|---:|---|---:|---:|---:|---:|---:|', + '| Benchmark | Routes | Variant | Median wall | Mean wall | p95 wall | Max RSS |', + '|---|---:|---|---:|---:|---:|---:|', ]; for (const benchmark of result.benchmarks) { @@ -329,7 +268,6 @@ const renderMarkdown = result => { formatMs(benchmark.summary.wallMs.mean), formatMs(benchmark.summary.wallMs.p95), formatRss(benchmark.summary.maxRssKb.p95), - benchmark.runs.reduce((sum, run) => sum + run.pluginReports.length, 0), ] .join(' | ') .replace(/^/, '| ') @@ -337,37 +275,6 @@ const renderMarkdown = result => { ); } - for (const benchmark of result.benchmarks) { - if (benchmark.pluginOperations.length === 0) { - continue; - } - lines.push( - '', - `## ${benchmark.id} Plugin Operations`, - '', - 'Total is the sum of all measured operation durations. Wall merges overlapping intervals to approximate elapsed plugin time. Max is the slowest single operation call.', - '', - '| Environment | Operation | Count | Total | Wall | Max | Reports |', - '|---|---|---:|---:|---:|---:|---:|' - ); - for (const operation of benchmark.pluginOperations.slice(0, 12)) { - lines.push( - [ - operation.environment, - operation.operation, - operation.count, - formatReportMs(operation.totalMs), - formatReportMs(operation.wallMs), - formatReportMs(operation.maxMs), - operation.reports, - ] - .join(' | ') - .replace(/^/, '| ') - .replace(/$/, ' |') - ); - } - } - lines.push(''); return `${lines.join('\n')}\n`; }; @@ -607,7 +514,6 @@ const main = async () => { cwd: fixtureRoot, env: { NODE_ENV: 'production', - REACT_ROUTER_BENCHMARK_LOG_PERFORMANCE: '1', ...(args.rspackProfile ? { RSPACK_PROFILE: args.rspackProfile } : {}), ...(rspackTraceOutput ? { RSPACK_TRACE_OUTPUT: rspackTraceOutput } @@ -628,10 +534,6 @@ const main = async () => { }) : []; const timeStats = useTime ? parseTimeStats(commandResult.stderr) : {}; - const pluginReports = parsePluginReports( - `${commandResult.stdout}\n${commandResult.stderr}` - ); - if (commandResult.status !== 0 && args.failFast) { process.exit(commandResult.status ?? 1); } @@ -643,7 +545,6 @@ const main = async () => { userMs: timeStats.userMs ?? null, sysMs: timeStats.sysMs ?? null, maxRssKb: timeStats.maxRssKb ?? null, - pluginReports, rspackProfiles, rspackTraceOutput: rspackTraceOutput && !isTraceOutputStream(rspackTraceOutput) @@ -661,7 +562,6 @@ const main = async () => { 'node /node_modules/@rsbuild/core/bin/rsbuild.js build --config rsbuild.config.mjs', runs, summary: summarizeRuns(runs), - pluginOperations: summarizePluginOperations(runs), }); } diff --git a/scripts/benchmark/fixture.mjs b/scripts/benchmark/fixture.mjs index 5869b22..b132449 100644 --- a/scripts/benchmark/fixture.mjs +++ b/scripts/benchmark/fixture.mjs @@ -259,7 +259,6 @@ const createRsbuildConfig = ({ ...(ssr ? [` serverOutput: 'module',`] : []), ...renderParallelTransformsOption(parallelTransforms), lazyCompilationOption, - ` logPerformance: process.env.REACT_ROUTER_BENCHMARK_LOG_PERFORMANCE === '1',`, ' }),', ' ],', ' output: {', diff --git a/scripts/compare-benchmarks.mjs b/scripts/compare-benchmarks.mjs index 31192ac..780b58b 100644 --- a/scripts/compare-benchmarks.mjs +++ b/scripts/compare-benchmarks.mjs @@ -9,23 +9,18 @@ const { values } = parseArgs({ before: { type: 'string' }, after: { type: 'string' }, benchmark: { type: 'string', default: 'synthetic-256-ssr-esm-split' }, - operations: { - type: 'string', - default: 'route:chunk,route:client-entry,route:split-exports', - }, }, }); if (!values.before || !values.after) { throw new Error( - 'Usage: node scripts/compare-benchmarks.mjs --before --after [--benchmark ] [--operations op,op]' + 'Usage: node scripts/compare-benchmarks.mjs --before --after [--benchmark ]' ); } const readJson = async file => JSON.parse(await readFile(file, 'utf8')); const before = await readJson(values.before); const after = await readJson(values.after); -const operations = new Set(values.operations.split(',').filter(Boolean)); const findBenchmark = (result, id) => { const benchmark = result.benchmarks?.find(item => item.id === id); @@ -40,19 +35,6 @@ const findBenchmark = (result, id) => { const metric = (benchmark, path) => path.split('.').reduce((value, key) => value?.[key], benchmark); -const operationMetric = (benchmark, operation, key) => { - const matches = - benchmark.pluginOperations?.filter(item => item.operation === operation) ?? - []; - const values = matches - .map(item => item[key]) - .filter(value => typeof value === 'number'); - if (values.length === 0) { - return null; - } - return values.reduce((sum, value) => sum + value, 0); -}; - const percentDelta = (beforeValue, afterValue) => { if (beforeValue == null || afterValue == null || beforeValue === 0) { return '-'; @@ -60,7 +42,6 @@ const percentDelta = (beforeValue, afterValue) => { return `${(((afterValue - beforeValue) / beforeValue) * 100).toFixed(1)}%`; }; -const formatNumber = value => (value == null ? '-' : value.toFixed(1)); const formatMs = value => value == null ? '-' : `${(value / 1000).toFixed(2)}s`; const formatKb = value => @@ -100,23 +81,6 @@ const rows = [ }, ]; -for (const operation of operations) { - rows.push( - { - label: `${operation} totalMs`, - before: operationMetric(beforeBenchmark, operation, 'totalMs'), - after: operationMetric(afterBenchmark, operation, 'totalMs'), - format: formatNumber, - }, - { - label: `${operation} wallMs`, - before: operationMetric(beforeBenchmark, operation, 'wallMs'), - after: operationMetric(afterBenchmark, operation, 'wallMs'), - format: formatNumber, - } - ); -} - console.log(`Benchmark comparison: ${values.benchmark}`); console.log(''); console.log('| Metric | Before | After | Delta |'); From ee77122662129571330673ef5c205739b1f6b01b Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 21 Jun 2026 02:21:29 +0200 Subject: [PATCH 4/5] chore: clarify benchmark script usage --- scripts/compare-benchmarks.mjs | 2 +- scripts/report-benchmark-ci.mjs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/compare-benchmarks.mjs b/scripts/compare-benchmarks.mjs index 780b58b..89e663f 100644 --- a/scripts/compare-benchmarks.mjs +++ b/scripts/compare-benchmarks.mjs @@ -14,7 +14,7 @@ const { values } = parseArgs({ if (!values.before || !values.after) { throw new Error( - 'Usage: node scripts/compare-benchmarks.mjs --before --after [--benchmark ]' + 'Usage: node scripts/compare-benchmarks.mjs --before --after [--benchmark ]' ); } diff --git a/scripts/report-benchmark-ci.mjs b/scripts/report-benchmark-ci.mjs index 2bd3154..5e0d167 100644 --- a/scripts/report-benchmark-ci.mjs +++ b/scripts/report-benchmark-ci.mjs @@ -21,7 +21,7 @@ const { values } = parseArgs({ if (!values.base || !values.head) { throw new Error( - 'Usage: node scripts/report-benchmark-ci.mjs --base --head [--out ]' + 'Usage: node scripts/report-benchmark-ci.mjs --base --head [--out ]' ); } From e8dbc7e5b61743f7e56a2b0939dda7c5720ff8aa Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 21 Jun 2026 02:27:32 +0200 Subject: [PATCH 5/5] ci: harden benchmark history persistence --- .github/workflows/benchmark.yml | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 7efd1c7..892081e 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -17,6 +17,7 @@ jobs: benchmark: name: Compare build benchmarks runs-on: ubuntu-latest + timeout-minutes: 30 env: BENCHMARK_PROFILE: default @@ -164,14 +165,23 @@ jobs: run: | set -euo pipefail history_dir="$GITHUB_WORKSPACE/benchmark-history" - git clone --depth 1 --branch "$BENCHMARK_HISTORY_BRANCH" \ - "https://x-access-token:${GH_TOKEN}@github.com/${REPOSITORY}.git" "$history_dir" \ - || { - git clone --depth 1 "https://x-access-token:${GH_TOKEN}@github.com/${REPOSITORY}.git" "$history_dir" - cd "$history_dir" - git checkout --orphan "$BENCHMARK_HISTORY_BRANCH" - git rm -rf . - } + remote_url="https://x-access-token:${GH_TOKEN}@github.com/${REPOSITORY}.git" + + set +e + git ls-remote --exit-code --heads "$remote_url" "$BENCHMARK_HISTORY_BRANCH" > /dev/null 2>&1 + branch_status=$? + set -e + + if [ "$branch_status" -eq 0 ]; then + git clone --depth 1 --branch "$BENCHMARK_HISTORY_BRANCH" "$remote_url" "$history_dir" + elif [ "$branch_status" -eq 2 ]; then + git clone --depth 1 "$remote_url" "$history_dir" + cd "$history_dir" + git checkout --orphan "$BENCHMARK_HISTORY_BRANCH" + git rm -rf . + else + exit "$branch_status" + fi cd "$history_dir" git config user.name "github-actions[bot]"