diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 0000000..892081e --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,207 @@ +name: Benchmark + +on: + pull_request: + branches: [main] + +permissions: + contents: write + issues: write + pull-requests: write + +concurrency: + group: benchmark-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + benchmark: + name: Compare build benchmarks + runs-on: ubuntu-latest + timeout-minutes: 30 + + env: + BENCHMARK_PROFILE: default + BENCHMARK_ITERATIONS: '5' + BENCHMARK_WARMUP: '1' + BENCHMARK_HISTORY_BRANCH: benchmark-results + + steps: + - name: Checkout PR head + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 + with: + path: head + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + + - name: Checkout PR base + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 + with: + path: base + ref: ${{ github.event.pull_request.base.sha }} + persist-credentials: false + + - name: Setup Node.js + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 + with: + node-version-file: head/.nvmrc + + - name: Setup PNPM + uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 + with: + package_json_file: head/package.json + + - name: Get pnpm store directory + id: pnpm-cache + shell: bash + run: | + echo "STORE_PATH=$(pnpm store path)" >> "$GITHUB_OUTPUT" + + - name: Setup pnpm cache + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 + with: + path: ${{ steps.pnpm-cache.outputs.STORE_PATH }} + key: ${{ runner.os }}-benchmark-pnpm-${{ hashFiles('head/pnpm-lock.yaml', 'base/pnpm-lock.yaml') }} + restore-keys: | + ${{ runner.os }}-benchmark-pnpm- + ${{ runner.os }}-pnpm-store- + + - name: Install head dependencies + working-directory: head + run: pnpm install --frozen-lockfile + + - name: Install base dependencies + working-directory: base + run: pnpm install --frozen-lockfile + + - name: Resolve benchmark-only React plugin + id: benchmark-imports + shell: bash + run: | + node -e "const { pathToFileURL } = require('node:url'); console.log('PLUGIN_REACT_IMPORT=' + pathToFileURL(process.argv[1]).href)" \ + "$GITHUB_WORKSPACE/head/node_modules/@rsbuild/plugin-react/dist/index.js" >> "$GITHUB_OUTPUT" + + - name: Benchmark base + working-directory: base + env: + REACT_ROUTER_BENCHMARK_PLUGIN_REACT_IMPORT: ${{ steps.benchmark-imports.outputs.PLUGIN_REACT_IMPORT }} + run: | + node "$GITHUB_WORKSPACE/head/scripts/bench-builds.mjs" \ + --profile "$BENCHMARK_PROFILE" \ + --iterations "$BENCHMARK_ITERATIONS" \ + --warmup "$BENCHMARK_WARMUP" \ + --clean build \ + --format both \ + --out "$GITHUB_WORKSPACE/benchmark-output/base" + + - name: Benchmark head + working-directory: head + run: | + node scripts/bench-builds.mjs \ + --profile "$BENCHMARK_PROFILE" \ + --iterations "$BENCHMARK_ITERATIONS" \ + --warmup "$BENCHMARK_WARMUP" \ + --clean build \ + --format both \ + --out "$GITHUB_WORKSPACE/benchmark-output/head" + + - name: Generate benchmark report + working-directory: head + env: + PR_NUMBER: ${{ github.event.pull_request.number }} + BASE_REF: ${{ github.event.pull_request.base.ref }} + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_REF: ${{ github.event.pull_request.head.ref }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + node scripts/report-benchmark-ci.mjs \ + --base "$GITHUB_WORKSPACE/benchmark-output/base/baseline.json" \ + --head "$GITHUB_WORKSPACE/benchmark-output/head/baseline.json" \ + --out "$GITHUB_WORKSPACE/benchmark-output/report" \ + --pr "$PR_NUMBER" \ + --base-ref "$BASE_REF" \ + --base-sha "$BASE_SHA" \ + --head-ref "$HEAD_REF" \ + --head-sha "$HEAD_SHA" \ + --run-url "$RUN_URL" + + - name: Upload benchmark artifacts + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: benchmark-pr-${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }} + path: benchmark-output + if-no-files-found: error + + - name: Comment benchmark results + if: github.event.pull_request.head.repo.full_name == github.repository + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + REPOSITORY: ${{ github.repository }} + run: | + set -euo pipefail + marker='' + comment_body="$GITHUB_WORKSPACE/benchmark-output/report/comment.md" + body="$(cat "$comment_body")" + comment_id="$(gh api "repos/$REPOSITORY/issues/$PR_NUMBER/comments" --paginate \ + --jq ".[] | select(.body | contains(\"$marker\")) | .id" | tail -n 1)" + + if [ -n "$comment_id" ]; then + gh api --method PATCH "repos/$REPOSITORY/issues/comments/$comment_id" \ + --field body="$body" + else + gh api --method POST "repos/$REPOSITORY/issues/$PR_NUMBER/comments" \ + --field body="$body" + fi + + - name: Persist benchmark history + if: github.event.pull_request.head.repo.full_name == github.repository + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + REPOSITORY: ${{ github.repository }} + shell: bash + run: | + set -euo pipefail + history_dir="$GITHUB_WORKSPACE/benchmark-history" + remote_url="https://x-access-token:${GH_TOKEN}@github.com/${REPOSITORY}.git" + + set +e + git ls-remote --exit-code --heads "$remote_url" "$BENCHMARK_HISTORY_BRANCH" > /dev/null 2>&1 + branch_status=$? + set -e + + if [ "$branch_status" -eq 0 ]; then + git clone --depth 1 --branch "$BENCHMARK_HISTORY_BRANCH" "$remote_url" "$history_dir" + elif [ "$branch_status" -eq 2 ]; then + git clone --depth 1 "$remote_url" "$history_dir" + cd "$history_dir" + git checkout --orphan "$BENCHMARK_HISTORY_BRANCH" + git rm -rf . + else + exit "$branch_status" + fi + + cd "$history_dir" + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + + result_dir="pull-requests/${PR_NUMBER}/${HEAD_SHA}" + latest_dir="pull-requests/${PR_NUMBER}/latest" + mkdir -p "$result_dir" "$latest_dir" + cp "$GITHUB_WORKSPACE/benchmark-output/base/baseline.json" "$result_dir/base.json" + cp "$GITHUB_WORKSPACE/benchmark-output/head/baseline.json" "$result_dir/head.json" + cp "$GITHUB_WORKSPACE/benchmark-output/report/report.json" "$result_dir/report.json" + cp "$GITHUB_WORKSPACE/benchmark-output/report/comment.md" "$result_dir/comment.md" + cp "$result_dir/report.json" "$latest_dir/report.json" + cp "$result_dir/comment.md" "$latest_dir/comment.md" + + git add pull-requests + if git diff --cached --quiet; then + echo "Benchmark history is already up to date." + exit 0 + fi + + git commit -m "bench: record PR ${PR_NUMBER} ${HEAD_SHA:0:7}" + git push origin "$BENCHMARK_HISTORY_BRANCH" diff --git a/.gitignore b/.gitignore index 59214f8..18b9153 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ build .unpack-cache/ .codex/ .tracedecay/ +.benchmark/ task/upstream/ task/output/ diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 0000000..f0bee00 --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,75 @@ +# Rsbuild React Router Performance Baselines + +This directory documents the repeatable benchmark workflow for +`rsbuild-plugin-react-router`. Benchmark artifacts are written to `.benchmark/`, +which is intentionally ignored. + +## Commands + +```sh +pnpm bench:smoke +pnpm bench:baseline +pnpm bench:full +``` + +`bench:smoke` is a one-iteration sanity check. `bench:baseline` is the default +comparison point for plugin performance work. `bench:full` adds larger route +counts for stress testing. + +All benchmark profiles generate deterministic synthetic React Router apps under +`.benchmark/fixtures/`, build the current plugin package once, then run Rsbuild +production builds against those fixtures. + +To capture Rspack tracing output for a benchmark, pass `--rspack-profile`: + +```sh +node scripts/bench-builds.mjs --profile=smoke --iterations=1 --warmup=0 --rspack-profile=OVERVIEW +node scripts/bench-builds.mjs --profile=full --filter=synthetic-1024 --iterations=1 --warmup=0 --rspack-profile=ALL +``` + +Trace directories are moved from fixture roots into +`.benchmark/results//rspack-profiles/` and referenced from the JSON +result. `ALL` can produce large traces; use it for targeted runs. +When `--rspack-trace-output` is provided, the benchmark writes one absolute +trace file per run under that directory so Rsbuild does not resolve the path +inside each generated `.rspack-profile-*` directory. + +## Baseline Shape + +The synthetic fixture keeps app behavior simple and scales route count/export +shape deliberately: + +- `ssr-esm`: production SSR build with ESM server output. +- `ssr-esm-split`: same route set with `future.v8_splitRouteModules`. +- `spa`: `ssr: false` route transform path. +- `sourcemaps`: production client sourcemaps enabled. + +Routes include plain components, server data exports, client data exports, +split-route candidates, and `.client` / `.server` imports. This targets the +plugin paths that are expensive in large apps: per-route client entries, +`?react-router-route` transforms, client-only stubbing, split-route detection, +and manifest emission. + +## Outputs + +Each run writes: + +- `.benchmark/results//baseline.json` +- `.benchmark/results//baseline.md` + +The JSON includes wall time, optional GNU `/usr/bin/time -v` user/sys/RSS data, +per-run exit status, and Rspack trace artifact paths when tracing is enabled. +The markdown report summarizes the same benchmark-level timing and memory data +without opening the raw JSON. + +## Hygiene + +Start and end with: + +```sh +git status --short +``` + +Benchmark output should stay inside ignored `.benchmark/`. If you need to clean +generated benchmark data, remove `.benchmark/` directly rather than using a broad +`git clean -fdX`, which can also delete `node_modules/` and TraceDecay indexes. diff --git a/package.json b/package.json index 0ff1577..f964c0d 100644 --- a/package.json +++ b/package.json @@ -49,6 +49,12 @@ }, "scripts": { "build": "rslib build", + "bench": "node scripts/bench-builds.mjs", + "bench:ci-report": "node scripts/report-benchmark-ci.mjs", + "bench:compare": "node scripts/compare-benchmarks.mjs", + "bench:smoke": "node scripts/bench-builds.mjs --profile smoke --iterations 1 --warmup 0 --format both --out .benchmark/results/smoke", + "bench:baseline": "node scripts/bench-builds.mjs --profile default --iterations 5 --warmup 1 --clean build --format both --out .benchmark/results/baseline", + "bench:full": "node scripts/bench-builds.mjs --profile full --iterations 5 --warmup 1 --clean build --format both --out .benchmark/results/full", "e2e": "pnpm build && pnpm --filter './examples/{default-template,spa-mode,prerender,custom-node-server,cloudflare,client-only}' test:e2e", "dev": "rslib build --watch", "test": "rstest run", @@ -88,6 +94,7 @@ "@react-router/dev": "^7.13.0", "@rsbuild/config": "workspace:*", "@rsbuild/core": "2.0.15", + "@rsbuild/plugin-react": "2.0.1", "@rslib/core": "^0.22.1", "@rspack/core": "2.0.8", "@swc/helpers": "^0.5.23", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 633631e..994cb95 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -78,6 +78,9 @@ importers: '@rsbuild/core': specifier: 2.0.15 version: 2.0.15(@module-federation/runtime-tools@2.5.1(node-fetch@2.7.0(encoding@0.1.13)))(core-js@3.47.0) + '@rsbuild/plugin-react': + specifier: 2.0.1 + version: 2.0.1(@rsbuild/core@2.0.15(@module-federation/runtime-tools@2.5.1(node-fetch@2.7.0(encoding@0.1.13)))(core-js@3.47.0))(@rspack/core@2.0.8(@module-federation/runtime-tools@2.5.1(node-fetch@2.7.0(encoding@0.1.13)))(@swc/helpers@0.5.23)) '@rslib/core': specifier: ^0.22.1 version: 0.22.1(@module-federation/runtime-tools@2.5.1)(core-js@3.47.0)(typescript@5.9.3) @@ -167,7 +170,7 @@ importers: dependencies: '@react-router/express': specifier: ^7.13.0 - version: 7.13.0(express@4.22.1)(react-router@7.13.0(react-dom@19.2.4(react@19.2.4))(react@19.2.4))(typescript@5.9.3) + version: 7.13.0(express@5.2.1)(react-router@7.13.0(react-dom@19.2.4(react@19.2.4))(react@19.2.4))(typescript@5.9.3) '@react-router/node': specifier: ^7.13.0 version: 7.13.0(react-router@7.13.0(react-dom@19.2.4(react@19.2.4))(react@19.2.4))(typescript@5.9.3) @@ -3667,96 +3670,112 @@ packages: '@react-email/body@0.2.1': resolution: {integrity: sha512-ljDiQiJDu/Fq//vSIIP0z5Nuvt4+DX1RqGasstChDGJB/14ogd4VdNS9aacoede/ZjGy3o3Qb+cxyS+XgM6SwQ==} engines: {node: '>=20.0.0'} + deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info. peerDependencies: react: ^18.0 || ^19.0 || ^19.0.0-rc '@react-email/button@0.2.1': resolution: {integrity: sha512-qXyj7RZLE7POy9BMKSoqQ00tOXThjOZSUnI2Yu9i29IHngPlmrNayIWBoVKtElES7OWwypUcpiajwi1mUWx6/A==} engines: {node: '>=20.0.0'} + deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info. peerDependencies: react: ^18.0 || ^19.0 || ^19.0.0-rc '@react-email/code-block@0.2.1': resolution: {integrity: sha512-M3B7JpVH4ytgn83/ujRR1k1DQHvTeABiDM61OvAbjLRPhC/5KLHU5KkzIbbuGIrjWwxAbL1kSQzU8MhLEtSxyw==} engines: {node: '>=20.0.0'} + deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info. peerDependencies: react: ^18.0 || ^19.0 || ^19.0.0-rc '@react-email/code-inline@0.0.6': resolution: {integrity: sha512-jfhebvv3dVsp3OdPgKXnk8+e2pBiDVZejDOBFzBa/IblrAJ9cQDkN6rBD5IyEg8hTOxwbw3iaI/yZFmDmIguIA==} engines: {node: '>=20.0.0'} + deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info. peerDependencies: react: ^18.0 || ^19.0 || ^19.0.0-rc '@react-email/column@0.0.14': resolution: {integrity: sha512-f+W+Bk2AjNO77zynE33rHuQhyqVICx4RYtGX9NKsGUg0wWjdGP0qAuIkhx9Rnmk4/hFMo1fUrtYNqca9fwJdHg==} engines: {node: '>=20.0.0'} + deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info. peerDependencies: react: ^18.0 || ^19.0 || ^19.0.0-rc '@react-email/components@1.0.6': resolution: {integrity: sha512-3GwOeq+5yyiAcwSf7TnHi/HWKn22lXbwxQmkkAviSwZLlhsRVxvmWqRxvUVfQk/HclDUG+62+sGz9qjfb2Uxjw==} engines: {node: '>=20.0.0'} + deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info. peerDependencies: react: ^18.0 || ^19.0 || ^19.0.0-rc '@react-email/container@0.0.16': resolution: {integrity: sha512-QWBB56RkkU0AJ9h+qy33gfT5iuZknPC7Un/IjZv9B0QmMIK+WWacc0cH6y2SV5Cv/b99hU94fjEMOOO4enpkbQ==} engines: {node: '>=20.0.0'} + deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info. peerDependencies: react: ^18.0 || ^19.0 || ^19.0.0-rc '@react-email/font@0.0.10': resolution: {integrity: sha512-0urVSgCmQIfx5r7Xc586miBnQUVnGp3OTYUm8m5pwtQRdTRO5XrTtEfNJ3JhYhSOruV0nD8fd+dXtKXobum6tA==} engines: {node: '>=20.0.0'} + deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info. peerDependencies: react: ^18.0 || ^19.0 || ^19.0.0-rc '@react-email/head@0.0.13': resolution: {integrity: sha512-AJg6le/08Gz4tm+6MtKXqtNNyKHzmooOCdmtqmWxD7FxoAdU1eVcizhtQ0gcnVaY6ethEyE/hnEzQxt1zu5Kog==} engines: {node: '>=20.0.0'} + deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info. peerDependencies: react: ^18.0 || ^19.0 || ^19.0.0-rc '@react-email/heading@0.0.16': resolution: {integrity: sha512-jmsKnQm1ykpBzw4hCYHwBkt5pW2jScXffPeEH5ZRF5tZeF5b1pvlFTO9han7C0pCkZYo1kEvWiRtx69yfCIwuw==} engines: {node: '>=20.0.0'} + deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info. peerDependencies: react: ^18.0 || ^19.0 || ^19.0.0-rc '@react-email/hr@0.0.12': resolution: {integrity: sha512-TwmOmBDibavUQpXBxpmZYi2Iks/yeZOzFYh+di9EltMSnEabH8dMZXrl+pxNXzCgZ2XE8HY7VmUL65Lenfu5PA==} engines: {node: '>=20.0.0'} + deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info. peerDependencies: react: ^18.0 || ^19.0 || ^19.0.0-rc '@react-email/html@0.0.12': resolution: {integrity: sha512-KTShZesan+UsreU7PDUV90afrZwU5TLwYlALuCSU0OT+/U8lULNNbAUekg+tGwCnOfIKYtpDPKkAMRdYlqUznw==} engines: {node: '>=20.0.0'} + deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info. peerDependencies: react: ^18.0 || ^19.0 || ^19.0.0-rc '@react-email/img@0.0.12': resolution: {integrity: sha512-sRCpEARNVTf3FQhZOC+JTvu5r6ubiYWkT0ucYXg8ctkyi4G8QG+jgYPiNUqVeTLA2STOfmPM/nrk1nb84y6CPQ==} engines: {node: '>=20.0.0'} + deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info. peerDependencies: react: ^18.0 || ^19.0 || ^19.0.0-rc '@react-email/link@0.0.13': resolution: {integrity: sha512-lkWc/NjOcefRZMkQoSDDbuKBEBDES9aXnFEOuPH845wD3TxPwh+QTf0fStuzjoRLUZWpHnio4z7qGGRYusn/sw==} engines: {node: '>=20.0.0'} + deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info. peerDependencies: react: ^18.0 || ^19.0 || ^19.0.0-rc '@react-email/markdown@0.0.18': resolution: {integrity: sha512-gSuYK5fsMbGk87jDebqQ6fa2fKcWlkf2Dkva8kMONqLgGCq8/0d+ZQYMEJsdidIeBo3kmsnHZPrwdFB4HgjUXg==} engines: {node: '>=20.0.0'} + deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info. peerDependencies: react: ^18.0 || ^19.0 || ^19.0.0-rc '@react-email/preview@0.0.14': resolution: {integrity: sha512-aYK8q0IPkBXyMsbpMXgxazwHxYJxTrXrV95GFuu2HbEiIToMwSyUgb8HDFYwPqqfV03/jbwqlsXmFxsOd+VNaw==} engines: {node: '>=20.0.0'} + deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info. peerDependencies: react: ^18.0 || ^19.0 || ^19.0.0-rc @@ -3770,18 +3789,21 @@ packages: '@react-email/row@0.0.13': resolution: {integrity: sha512-bYnOac40vIKCId7IkwuLAAsa3fKfSfqCvv6epJKmPE0JBuu5qI4FHFCl9o9dVpIIS08s/ub+Y/txoMt0dYziGw==} engines: {node: '>=20.0.0'} + deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info. peerDependencies: react: ^18.0 || ^19.0 || ^19.0.0-rc '@react-email/section@0.0.17': resolution: {integrity: sha512-qNl65ye3W0Rd5udhdORzTV9ezjb+GFqQQSae03NDzXtmJq6sqVXNWNiVolAjvJNypim+zGXmv6J9TcV5aNtE/w==} engines: {node: '>=20.0.0'} + deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info. peerDependencies: react: ^18.0 || ^19.0 || ^19.0.0-rc '@react-email/tailwind@2.0.3': resolution: {integrity: sha512-URXb/T2WS4RlNGM5QwekYnivuiVUcU87H0y5sqLl6/Oi3bMmgL0Bmw/W9GeJylC+876Vw+E6NkE0uRiUFIQwGg==} engines: {node: '>=20.0.0'} + deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info. peerDependencies: '@react-email/body': 0.2.1 '@react-email/button': 0.2.1 @@ -3820,6 +3842,7 @@ packages: '@react-email/text@0.1.6': resolution: {integrity: sha512-TYqkioRS45wTR5il3dYk/SbUjjEdhSwh9BtRNB99qNH1pXAwA45H7rAuxehiu8iJQJH0IyIr+6n62gBz9ezmsw==} engines: {node: '>=20.0.0'} + deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info. peerDependencies: react: ^18.0 || ^19.0 || ^19.0.0-rc @@ -6493,6 +6516,7 @@ packages: glob@10.5.0: resolution: {integrity: sha512-DfXN8DfhJ7NH3Oe7cFmu3NCu1wKbkReJ8TorzSAFbSKrlNaQSKfIzqYqVY8zlbs2NLBbWpRiU52GX2PbaBVNkg==} + deprecated: Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me hasBin: true glob@13.0.0: @@ -7778,6 +7802,7 @@ packages: prebuild-install@7.1.3: resolution: {integrity: sha512-8Mf2cbV7x1cXPUILADGI3wuhfqWvtiLA1iclTDbFRZkgRQS0NqsPZphna9V+HyTEadheuPmjaJMsbzKQFOzLug==} engines: {node: '>=10'} + deprecated: No longer maintained. Please contact the author of the relevant native addon; alternatives are available. hasBin: true prelude-ls@1.2.1: @@ -9130,6 +9155,7 @@ packages: uuid@8.3.2: resolution: {integrity: sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==} + deprecated: uuid@10 and below is no longer supported. For ESM codebases, update to uuid@latest. For CommonJS codebases, use uuid@11 (but be aware this version will likely be deprecated in 2028). hasBin: true valibot@1.2.0: diff --git a/scripts/bench-builds.mjs b/scripts/bench-builds.mjs new file mode 100644 index 0000000..173e40a --- /dev/null +++ b/scripts/bench-builds.mjs @@ -0,0 +1,600 @@ +#!/usr/bin/env node +import { + access, + cp, + mkdir, + readdir, + rename, + rm, + writeFile, +} from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { performance } from 'node:perf_hooks'; +import { pathToFileURL } from 'node:url'; +import { parseArgs as parseCliArgs } from 'node:util'; +import { execa } from 'execa'; +import { generateSyntheticFixture } from './benchmark/fixture.mjs'; + +const rootDir = process.cwd(); +const benchmarkRoot = path.join(rootDir, '.benchmark'); +const rsbuildBin = path.join( + rootDir, + 'node_modules', + '@rsbuild', + 'core', + 'bin', + 'rsbuild.js' +); + +const profiles = { + smoke: [{ id: 'synthetic-48-ssr-esm', routeCount: 48, variant: 'ssr-esm' }], + default: [ + { id: 'synthetic-256-ssr-esm', routeCount: 256, variant: 'ssr-esm' }, + { + id: 'synthetic-256-ssr-esm-split', + routeCount: 256, + variant: 'ssr-esm-split', + }, + { id: 'synthetic-256-spa', routeCount: 256, variant: 'spa' }, + { + id: 'synthetic-256-sourcemaps', + routeCount: 256, + variant: 'ssr-esm', + sourceMap: true, + }, + ], + full: [ + { id: 'synthetic-48-ssr-esm', routeCount: 48, variant: 'ssr-esm' }, + { id: 'synthetic-256-ssr-esm', routeCount: 256, variant: 'ssr-esm' }, + { id: 'synthetic-1024-ssr-esm', routeCount: 1024, variant: 'ssr-esm' }, + { + id: 'synthetic-256-ssr-esm-split', + routeCount: 256, + variant: 'ssr-esm-split', + }, + { + id: 'synthetic-1024-ssr-esm-split', + routeCount: 1024, + variant: 'ssr-esm-split', + }, + { + id: 'synthetic-256-sourcemaps', + routeCount: 256, + variant: 'ssr-esm', + sourceMap: true, + }, + ], +}; + +const parseArgs = argv => { + const { values } = parseCliArgs({ + args: argv, + allowPositionals: false, + strict: true, + options: { + profile: { type: 'string', default: 'default' }, + iterations: { type: 'string', default: '5' }, + warmup: { type: 'string', default: '1' }, + format: { type: 'string', default: 'both' }, + out: { + type: 'string', + default: path.join('.benchmark', 'results', 'baseline'), + }, + clean: { type: 'string', default: 'build' }, + filter: { type: 'string' }, + 'parallel-transforms': { type: 'string' }, + 'rspack-profile': { type: 'string' }, + 'rspack-trace-output': { type: 'string' }, + 'fail-fast': { type: 'boolean', default: false }, + 'skip-root-build': { type: 'boolean', default: false }, + }, + }); + + const parseParallelTransforms = value => { + if (value === undefined) { + return undefined; + } + if (value === 'false' || value === '0') { + return false; + } + if (value === 'true' || value === '1' || value === 'auto') { + return true; + } + const maxWorkers = Number(value); + if (!Number.isInteger(maxWorkers) || maxWorkers < 1) { + throw new Error( + '--parallel-transforms must be true, false, auto, or a positive integer.' + ); + } + return { maxWorkers }; + }; + + const args = { + profile: values.profile, + iterations: Number(values.iterations), + warmup: Number(values.warmup), + format: values.format, + out: values.out, + clean: values.clean, + filter: values.filter ?? null, + parallelTransforms: parseParallelTransforms(values['parallel-transforms']), + rspackProfile: values['rspack-profile'] ?? null, + rspackTraceOutput: values['rspack-trace-output'] ?? null, + failFast: values['fail-fast'], + skipRootBuild: values['skip-root-build'], + }; + + if (!profiles[args.profile]) { + throw new Error( + `Unknown profile "${args.profile}". Use smoke, default, or full.` + ); + } + if (!Number.isInteger(args.iterations) || args.iterations < 1) { + throw new Error('--iterations must be a positive integer.'); + } + if (!Number.isInteger(args.warmup) || args.warmup < 0) { + throw new Error('--warmup must be a non-negative integer.'); + } + if (!['json', 'md', 'markdown', 'both'].includes(args.format)) { + throw new Error('--format must be json, md, markdown, or both.'); + } + if (!['none', 'build', 'cold'].includes(args.clean)) { + throw new Error('--clean must be none, build, or cold.'); + } + if (args.rspackProfile !== null && args.rspackProfile.trim() === '') { + throw new Error('--rspack-profile must not be empty.'); + } + if (args.rspackTraceOutput !== null && args.rspackTraceOutput.trim() === '') { + throw new Error('--rspack-trace-output must not be empty.'); + } + + return args; +}; + +const hasGnuTime = async () => { + try { + await access('/usr/bin/time'); + return true; + } catch { + return false; + } +}; + +const runCommand = async ({ + command, + args, + cwd, + env = {}, + useTime = false, +}) => { + const startedAt = performance.now(); + const childCommand = useTime ? '/usr/bin/time' : command; + const childArgs = useTime ? ['-v', command, ...args] : args; + + const child = execa(childCommand, childArgs, { + cwd, + env, + reject: false, + }); + + child.stdout?.pipe(process.stdout); + child.stderr?.pipe(process.stderr); + + const result = await child; + return { + status: result.exitCode, + stdout: result.stdout, + stderr: result.stderr, + wallMs: performance.now() - startedAt, + }; +}; + +const parseTimeStats = stderr => { + const user = stderr.match(/User time \(seconds\):\s*([\d.]+)/); + const sys = stderr.match(/System time \(seconds\):\s*([\d.]+)/); + const rss = stderr.match(/Maximum resident set size \(kbytes\):\s*(\d+)/); + return { + userMs: user ? Number(user[1]) * 1000 : null, + sysMs: sys ? Number(sys[1]) * 1000 : null, + maxRssKb: rss ? Number(rss[1]) : null, + }; +}; + +const summarizeMetric = values => { + const sorted = values + .filter(value => typeof value === 'number') + .sort((a, b) => a - b); + if (sorted.length === 0) { + return { min: null, median: null, mean: null, p95: null, stdev: null }; + } + const mean = sorted.reduce((sum, value) => sum + value, 0) / sorted.length; + const variance = + sorted.reduce((sum, value) => sum + (value - mean) ** 2, 0) / sorted.length; + const percentileIndex = Math.min( + sorted.length - 1, + Math.ceil(sorted.length * 0.95) - 1 + ); + return { + min: sorted[0], + median: sorted[Math.floor(sorted.length / 2)], + mean, + p95: sorted[percentileIndex], + stdev: Math.sqrt(variance), + }; +}; + +const summarizeRuns = runs => ({ + wallMs: summarizeMetric(runs.map(run => run.wallMs)), + userMs: summarizeMetric(runs.map(run => run.userMs)), + sysMs: summarizeMetric(runs.map(run => run.sysMs)), + maxRssKb: summarizeMetric(runs.map(run => run.maxRssKb)), +}); + +const formatMs = value => + value == null ? '-' : `${(value / 1000).toFixed(2)}s`; +const formatRss = value => + value == null ? '-' : `${Math.round(value / 1024)} MB`; + +const renderMarkdown = result => { + const lines = [ + '# Rsbuild React Router Benchmark Baseline', + '', + `- Date: ${result.date}`, + `- Commit: ${result.commit}`, + `- Node: ${result.node}`, + `- pnpm: ${result.pnpm}`, + `- Platform: ${result.platform}`, + `- Profile: ${result.profile}`, + `- Iterations: ${result.iterations}`, + `- Warmup: ${result.warmup}`, + `- Parallel transforms: ${formatParallelTransforms(result.parallelTransforms)}`, + `- Rspack profile: ${result.rspackProfile ?? 'false'}`, + ...(result.rspackTraceOutput + ? [`- Rspack trace output: ${result.rspackTraceOutput}`] + : []), + '', + '| Benchmark | Routes | Variant | Median wall | Mean wall | p95 wall | Max RSS |', + '|---|---:|---|---:|---:|---:|---:|', + ]; + + for (const benchmark of result.benchmarks) { + lines.push( + [ + benchmark.id, + benchmark.routeCount, + benchmark.variant, + formatMs(benchmark.summary.wallMs.median), + formatMs(benchmark.summary.wallMs.mean), + formatMs(benchmark.summary.wallMs.p95), + formatRss(benchmark.summary.maxRssKb.p95), + ] + .join(' | ') + .replace(/^/, '| ') + .replace(/$/, ' |') + ); + } + + lines.push(''); + return `${lines.join('\n')}\n`; +}; + +const resolveOutputPaths = args => { + const outPath = path.resolve(rootDir, args.out); + const format = args.format === 'markdown' ? 'md' : args.format; + const writeJson = format === 'json' || format === 'both'; + const writeMd = format === 'md' || format === 'both'; + + if (writeJson && writeMd) { + return { + artifactRoot: outPath, + jsonPath: path.join(outPath, 'baseline.json'), + mdPath: path.join(outPath, 'baseline.md'), + outPath, + writeJson, + writeMd, + }; + } + + const artifactRoot = path.extname(outPath) + ? path.join(path.dirname(outPath), `${path.basename(outPath)}.artifacts`) + : `${outPath}.artifacts`; + + return { + artifactRoot, + jsonPath: writeJson ? outPath : null, + mdPath: writeMd ? outPath : null, + outPath, + writeJson, + writeMd, + }; +}; + +const writeOutputs = async (result, outputPaths) => { + const { jsonPath, mdPath, outPath, writeJson, writeMd } = outputPaths; + + if (writeJson && writeMd) { + await mkdir(outPath, { recursive: true }); + await writeFile(jsonPath, `${JSON.stringify(result, null, 2)}\n`); + await writeFile(mdPath, renderMarkdown(result)); + return; + } + + await mkdir(path.dirname(outPath), { recursive: true }); + if (writeJson) { + await writeFile(jsonPath, `${JSON.stringify(result, null, 2)}\n`); + } else { + await writeFile(mdPath, renderMarkdown(result)); + } +}; + +const formatParallelTransforms = parallelTransforms => { + if (parallelTransforms === undefined) { + return 'default'; + } + if (!parallelTransforms) { + return 'false'; + } + if (parallelTransforms === true) { + return 'true'; + } + return `maxWorkers=${parallelTransforms.maxWorkers}`; +}; + +const git = async args => { + const result = await runCommand({ + command: 'git', + args, + cwd: rootDir, + useTime: false, + }); + return result.status === 0 ? result.stdout.trim() : null; +}; + +const pnpmVersion = async () => { + const result = await runCommand({ + command: 'pnpm', + args: ['--version'], + cwd: rootDir, + useTime: false, + }); + return result.status === 0 ? result.stdout.trim() : null; +}; + +const cleanBuildOutputs = async fixtureRoot => { + await Promise.all([ + rm(path.join(fixtureRoot, 'build'), { recursive: true, force: true }), + rm(path.join(fixtureRoot, '.react-router'), { + recursive: true, + force: true, + }), + ]); +}; + +const listRspackProfileDirs = async cwd => { + const entries = await readdir(cwd, { withFileTypes: true }); + return entries + .filter( + entry => entry.isDirectory() && entry.name.startsWith('.rspack-profile-') + ) + .map(entry => entry.name) + .sort(); +}; + +const moveDirectory = async (source, destination) => { + await rm(destination, { recursive: true, force: true }); + await mkdir(path.dirname(destination), { recursive: true }); + try { + await rename(source, destination); + } catch (error) { + if (error?.code !== 'EXDEV') { + throw error; + } + await cp(source, destination, { recursive: true }); + await rm(source, { recursive: true, force: true }); + } +}; + +const collectRspackProfiles = async ({ + fixtureRoot, + beforeProfiles, + destinationRoot, +}) => { + const before = new Set(beforeProfiles); + const afterProfiles = await listRspackProfileDirs(fixtureRoot); + const createdProfiles = afterProfiles.filter(profile => !before.has(profile)); + const collected = []; + + for (const profile of createdProfiles) { + const source = path.join(fixtureRoot, profile); + const destination = path.join(destinationRoot, profile.slice(1)); + await moveDirectory(source, destination); + collected.push(path.relative(rootDir, destination)); + } + + return collected; +}; + +const isTraceOutputStream = value => value === 'stdout' || value === 'stderr'; + +const resolveRspackTraceOutput = async ({ + traceOutput, + benchmarkId, + runLabel, +}) => { + if (!traceOutput || isTraceOutputStream(traceOutput)) { + return traceOutput; + } + + const tracePath = path.resolve( + rootDir, + traceOutput, + benchmarkId, + `${runLabel}.log` + ); + await mkdir(path.dirname(tracePath), { recursive: true }); + return tracePath; +}; + +const main = async () => { + const args = parseArgs(process.argv.slice(2)); + const useTime = await hasGnuTime(); + const outputPaths = resolveOutputPaths(args); + const pluginImportPath = pathToFileURL( + path.join(rootDir, 'dist/index.js') + ).href; + const pluginReactImportPath = + process.env.REACT_ROUTER_BENCHMARK_PLUGIN_REACT_IMPORT ?? + '@rsbuild/plugin-react'; + const selectedBenchmarks = profiles[args.profile].filter(benchmark => + args.filter ? benchmark.id.includes(args.filter) : true + ); + + if (selectedBenchmarks.length === 0) { + throw new Error(`No benchmarks matched filter "${args.filter}".`); + } + + if (!args.skipRootBuild) { + console.log('Building plugin package before benchmarks...'); + const buildResult = await runCommand({ + command: 'pnpm', + args: ['build'], + cwd: rootDir, + }); + if (buildResult.status !== 0) { + process.exit(buildResult.status ?? 1); + } + } + + const benchmarks = []; + for (const benchmark of selectedBenchmarks) { + const fixtureRoot = path.join(benchmarkRoot, 'fixtures', benchmark.id); + await generateSyntheticFixture({ + root: fixtureRoot, + routeCount: benchmark.routeCount, + variant: benchmark.variant, + sourceMap: benchmark.sourceMap ?? false, + pluginImportPath, + pluginReactImportPath, + parallelTransforms: args.parallelTransforms, + }); + + const runs = []; + const totalRuns = args.warmup + args.iterations; + for (let index = 0; index < totalRuns; index += 1) { + const measured = index >= args.warmup; + if (args.clean !== 'none') { + await cleanBuildOutputs(fixtureRoot); + } + if (args.clean === 'cold') { + await rm(path.join(fixtureRoot, 'node_modules'), { + recursive: true, + force: true, + }); + } + + console.log( + `${measured ? 'Measuring' : 'Warming'} ${benchmark.id} (${index + 1}/${totalRuns})` + ); + const rspackProfileEnabled = Boolean(args.rspackProfile); + const beforeRspackProfiles = rspackProfileEnabled + ? await listRspackProfileDirs(fixtureRoot) + : []; + const runLabel = `${measured ? 'run' : 'warmup'}-${ + measured ? index - args.warmup + 1 : index + 1 + }`; + const rspackTraceOutput = await resolveRspackTraceOutput({ + traceOutput: args.rspackTraceOutput, + benchmarkId: benchmark.id, + runLabel, + }); + const commandResult = await runCommand({ + command: process.execPath, + args: [rsbuildBin, 'build', '--config', 'rsbuild.config.mjs'], + cwd: fixtureRoot, + env: { + NODE_ENV: 'production', + ...(args.rspackProfile ? { RSPACK_PROFILE: args.rspackProfile } : {}), + ...(rspackTraceOutput + ? { RSPACK_TRACE_OUTPUT: rspackTraceOutput } + : {}), + }, + useTime, + }); + const rspackProfiles = rspackProfileEnabled + ? await collectRspackProfiles({ + fixtureRoot, + beforeProfiles: beforeRspackProfiles, + destinationRoot: path.join( + outputPaths.artifactRoot, + 'rspack-profiles', + benchmark.id, + runLabel + ), + }) + : []; + const timeStats = useTime ? parseTimeStats(commandResult.stderr) : {}; + if (commandResult.status !== 0 && args.failFast) { + process.exit(commandResult.status ?? 1); + } + + if (measured) { + runs.push({ + status: commandResult.status, + wallMs: commandResult.wallMs, + userMs: timeStats.userMs ?? null, + sysMs: timeStats.sysMs ?? null, + maxRssKb: timeStats.maxRssKb ?? null, + rspackProfiles, + rspackTraceOutput: + rspackTraceOutput && !isTraceOutputStream(rspackTraceOutput) + ? path.relative(rootDir, rspackTraceOutput) + : rspackTraceOutput, + }); + } + } + + benchmarks.push({ + ...benchmark, + parallelTransforms: args.parallelTransforms, + cwd: path.relative(rootDir, fixtureRoot), + command: + 'node /node_modules/@rsbuild/core/bin/rsbuild.js build --config rsbuild.config.mjs', + runs, + summary: summarizeRuns(runs), + }); + } + + const failed = benchmarks.some(benchmark => + benchmark.runs.some(run => run.status !== 0) + ); + const result = { + repo: 'rsbuild-plugin-react-router', + commit: await git(['rev-parse', 'HEAD']), + date: new Date().toISOString(), + node: process.version, + pnpm: await pnpmVersion(), + platform: `${os.platform()} ${os.release()} ${os.arch()}`, + profile: args.profile, + iterations: args.iterations, + warmup: args.warmup, + parallelTransforms: args.parallelTransforms, + rspackProfile: args.rspackProfile, + rspackTraceOutput: args.rspackTraceOutput, + failed, + benchmarks, + }; + + await writeOutputs(result, outputPaths); + console.log(`Benchmark results written to ${outputPaths.outPath}`); + + if (failed) { + console.error('One or more measured benchmark builds failed.'); + process.exitCode = 1; + } +}; + +main().catch(error => { + console.error(error); + process.exit(1); +}); diff --git a/scripts/benchmark/fixture.mjs b/scripts/benchmark/fixture.mjs new file mode 100644 index 0000000..b132449 --- /dev/null +++ b/scripts/benchmark/fixture.mjs @@ -0,0 +1,453 @@ +import { mkdir, rm, writeFile } from 'node:fs/promises'; +import path from 'node:path'; + +const routeExportProfiles = [ + 'plain', + 'ssr-data', + 'split-client', + 'split-client', + 'ssr-data', + 'client-server-imports', +]; + +export const benchmarkFixtureNames = [ + 'default', + 'export-heavy', + 'reexports', + 'import-fanout', + 'chunk-saturated', +]; + +const stressFixtureNames = new Set(benchmarkFixtureNames); + +export const padRoute = number => String(number).padStart(4, '0'); + +export const routeFile = index => `routes/route-${padRoute(index)}.tsx`; + +export const routeId = index => `route-${padRoute(index)}`; + +const routeComponentName = index => `Route${padRoute(index)}`; + +const createSharedRouteExports = (index, { includeHeaders = false } = {}) => { + const name = routeComponentName(index); + return [ + `export const handle = { label: '${routeId(index)}' };`, + `export function meta() { return [{ title: '${routeId(index)}' }]; }`, + `export default function ${name}() { return null; }`, + ...(includeHeaders + ? [ + `export function headers() { return { 'x-route': '${routeId(index)}' }; }`, + ] + : []), + ]; +}; + +const createDefaultRouteModule = (index, profile, { isSpa }) => { + const shared = createSharedRouteExports(index); + + if (profile === 'ssr-data') { + if (isSpa) { + return [ + ...shared, + `export function shouldRevalidate() { return false; }`, + ].join('\n'); + } + + return [ + `import { serverValue } from '../server-data.server';`, + ...shared, + `export async function loader() { return { id: '${routeId(index)}', serverValue }; }`, + `export async function action() { return { ok: true }; }`, + `export function headers() { return { 'x-route': '${routeId(index)}' }; }`, + `export function shouldRevalidate() { return false; }`, + ].join('\n'); + } + + if (profile === 'split-client') { + return [ + `import { clientValue } from '../client-data.client';`, + ...shared, + `export async function clientLoader() { return { id: '${routeId(index)}', clientValue }; }`, + `export async function clientAction() { return { ok: true }; }`, + `export async function clientMiddleware() { return undefined; }`, + ...(isSpa ? [] : [`export function HydrateFallback() { return null; }`]), + ].join('\n'); + } + + if (profile === 'client-server-imports') { + if (isSpa) { + return [ + `import { clientValue } from '../client-data.client';`, + ...shared, + `export async function clientLoader() { return { id: '${routeId(index)}', clientValue }; }`, + ].join('\n'); + } + + return [ + `import { clientValue } from '../client-data.client';`, + `import { serverValue } from '../server-data.server';`, + ...shared, + `export async function loader() { return { id: '${routeId(index)}', serverValue }; }`, + `export async function clientLoader() { return { id: '${routeId(index)}', clientValue }; }`, + ].join('\n'); + } + + return shared.join('\n'); +}; + +const createExportHeavyRouteModule = (index, { isSpa }) => { + const extraExports = Array.from({ length: 32 }, (_, extraIndex) => { + const exportName = `unusedExport${padRoute(index)}_${String(extraIndex).padStart(2, '0')}`; + return `export const ${exportName} = '${routeId(index)}-${extraIndex}';`; + }); + + return [ + `import { clientValue } from '../client-data.client';`, + ...(isSpa ? [] : [`import { serverValue } from '../server-data.server';`]), + ...createSharedRouteExports(index, { includeHeaders: !isSpa }), + `export function links() { return []; }`, + `export function shouldRevalidate() { return false; }`, + `export async function clientLoader() { return { id: '${routeId(index)}', clientValue }; }`, + `export async function clientAction() { return { ok: true }; }`, + `export async function clientMiddleware() { return undefined; }`, + ...(isSpa + ? [] + : [ + `export async function loader() { return { id: '${routeId(index)}', serverValue }; }`, + `export async function action() { return { ok: true }; }`, + `export function HydrateFallback() { return null; }`, + ]), + ...extraExports, + ].join('\n'); +}; + +const createReexportsRouteModule = (index, { isSpa }) => + [ + `export { default, handle, meta, shouldRevalidate } from '../route-reexports/reexport-${padRoute(index)}';`, + `export { clientLoader, clientAction, clientMiddleware${isSpa ? '' : ', HydrateFallback'} } from '../route-reexports/reexport-${padRoute(index)}';`, + ...(isSpa + ? [] + : [ + `export { loader, action, headers } from '../route-reexports/reexport-${padRoute(index)}';`, + ]), + `export * from '../route-reexports/reexport-all-${padRoute(index)}';`, + ].join('\n'); + +const createImportFanoutRouteModule = (index, { isSpa }) => { + const imports = Array.from({ length: 16 }, (_, fanoutIndex) => { + const suffix = String(fanoutIndex).padStart(2, '0'); + return `import { fanoutValue${suffix} } from '../fanout/fanout-${suffix}';`; + }); + const values = Array.from( + { length: 16 }, + (_, fanoutIndex) => `fanoutValue${String(fanoutIndex).padStart(2, '0')}` + ).join(', '); + + return [ + ...imports, + ...createSharedRouteExports(index, { includeHeaders: !isSpa }), + `const fanoutValues = [${values}];`, + `export function shouldRevalidate() { return fanoutValues.length > ${index % 7}; }`, + `export async function clientLoader() { return { values: fanoutValues }; }`, + ...(isSpa + ? [] + : [ + `export async function loader() { return { values: fanoutValues }; }`, + ]), + ].join('\n'); +}; + +const createChunkSaturatedRouteModule = (index, { isSpa }) => + [ + `import { clientValue } from '../client-data.client';`, + ...(isSpa ? [] : [`import { serverValue } from '../server-data.server';`]), + ...createSharedRouteExports(index, { includeHeaders: !isSpa }), + `const routeLabel = '${routeId(index)}';`, + `export function shouldRevalidate() { return routeLabel.length > 0; }`, + `export async function clientLoader() { return { routeLabel, clientValue }; }`, + `export async function clientAction() { return { ok: routeLabel, clientValue }; }`, + `export async function clientMiddleware() { return undefined; }`, + ...(isSpa + ? [] + : [ + `export function HydrateFallback() { return null; }`, + `export async function loader() { return { routeLabel, serverValue }; }`, + `export async function action() { return { ok: true, serverValue }; }`, + ]), + ].join('\n'); + +const createRouteModule = (index, profile, { isSpa, fixture }) => { + if (fixture === 'export-heavy') { + return createExportHeavyRouteModule(index, { isSpa }); + } + if (fixture === 'reexports') { + return createReexportsRouteModule(index, { isSpa }); + } + if (fixture === 'import-fanout') { + return createImportFanoutRouteModule(index, { isSpa }); + } + if (fixture === 'chunk-saturated') { + return createChunkSaturatedRouteModule(index, { isSpa }); + } + return createDefaultRouteModule(index, profile, { isSpa }); +}; + +const createRoutesConfig = routeCount => { + const routes = []; + for (let index = 1; index <= routeCount; index += 1) { + const id = routeId(index); + const isIndex = index === 1; + routes.push( + [ + ' {', + ` id: '${id}',`, + ` file: '${routeFile(index)}',`, + isIndex ? ' index: true,' : ` path: '${id}',`, + ' },', + ].join('\n') + ); + } + + return [ + `import type { RouteConfigEntry } from '@react-router/dev/routes';`, + '', + 'export default [', + ...routes, + '] satisfies RouteConfigEntry[];', + '', + ].join('\n'); +}; + +const renderParallelTransformsOption = parallelTransforms => { + if (parallelTransforms === undefined) { + return []; + } + if (parallelTransforms === false) { + return [` parallelTransforms: false,`]; + } + if (parallelTransforms === true) { + return [` parallelTransforms: true,`]; + } + return [ + ` parallelTransforms: { maxWorkers: ${parallelTransforms.maxWorkers} },`, + ]; +}; + +const createRsbuildConfig = ({ + variant, + sourceMap, + pluginImportPath, + pluginReactImportPath, + parallelTransforms, +}) => { + const ssr = variant !== 'spa'; + const lazyCompilationOption = + ` ...(process.env.REACT_ROUTER_BENCHMARK_LAZY_COMPILATION === '0'` + + ` ? { lazyCompilation: false }` + + ` : process.env.REACT_ROUTER_BENCHMARK_LAZY_COMPILATION === '1'` + + ` ? { lazyCompilation: true } : {}),`; + + return [ + `import { defineConfig } from '@rsbuild/core';`, + `import { pluginReact } from '${pluginReactImportPath}';`, + `import { pluginReactRouter } from '${pluginImportPath}';`, + '', + 'export default defineConfig({', + ' plugins: [', + ' pluginReact(),', + ' pluginReactRouter({', + ...(ssr ? [` serverOutput: 'module',`] : []), + ...renderParallelTransformsOption(parallelTransforms), + lazyCompilationOption, + ' }),', + ' ],', + ' output: {', + ` sourceMap: ${ + sourceMap ? `{ js: 'cheap-module-source-map', css: false }` : 'false' + },`, + ' },', + '});', + '', + ].join('\n'); +}; + +const createReactRouterConfig = variant => { + const ssr = variant !== 'spa'; + const splitRouteModules = variant.includes('split'); + + return [ + `import type { Config } from '@react-router/dev/config';`, + '', + 'export default {', + ` ssr: ${ssr ? 'true' : 'false'},`, + ` future: {`, + ` v8_splitRouteModules: ${splitRouteModules ? 'true' : 'false'},`, + ' },', + '} satisfies Config;', + '', + ].join('\n'); +}; + +const writeReexportFixtures = async (root, routeCount, { isSpa }) => { + await mkdir(path.join(root, 'app/route-reexports'), { recursive: true }); + const batchSize = 64; + for (let batchStart = 0; batchStart < routeCount; batchStart += batchSize) { + await Promise.all( + Array.from( + { length: Math.min(batchSize, routeCount - batchStart) }, + (_, batchIndex) => { + const routeIndex = batchStart + batchIndex; + const index = routeIndex + 1; + const module = [ + `import { clientValue } from '../client-data.client';`, + ...(isSpa + ? [] + : [`import { serverValue } from '../server-data.server';`]), + ...createSharedRouteExports(index, { includeHeaders: !isSpa }), + `export function shouldRevalidate() { return false; }`, + `export async function clientLoader() { return { id: '${routeId(index)}', clientValue }; }`, + `export async function clientAction() { return { ok: true }; }`, + `export async function clientMiddleware() { return undefined; }`, + ...(isSpa + ? [] + : [ + `export async function loader() { return { id: '${routeId(index)}', serverValue }; }`, + `export async function action() { return { ok: true }; }`, + `export function HydrateFallback() { return null; }`, + ]), + ].join('\n'); + const exportAllModule = [ + `export const reexportedValue${padRoute(index)} = '${routeId(index)}';`, + `export function reexportedHelper${padRoute(index)}() { return reexportedValue${padRoute(index)}; }`, + ].join('\n'); + return Promise.all([ + writeFile( + path.join( + root, + `app/route-reexports/reexport-${padRoute(index)}.ts` + ), + `${module}\n` + ), + writeFile( + path.join( + root, + `app/route-reexports/reexport-all-${padRoute(index)}.ts` + ), + `${exportAllModule}\n` + ), + ]); + } + ) + ); + } +}; + +const writeFanoutFixtures = async root => { + await mkdir(path.join(root, 'app/fanout'), { recursive: true }); + await Promise.all( + Array.from({ length: 16 }, (_, fanoutIndex) => { + const suffix = String(fanoutIndex).padStart(2, '0'); + return writeFile( + path.join(root, `app/fanout/fanout-${suffix}.ts`), + `export const fanoutValue${suffix} = '${suffix}';\n` + ); + }) + ); +}; + +export async function generateSyntheticFixture({ + root, + routeCount, + variant, + sourceMap = false, + pluginImportPath = 'rsbuild-plugin-react-router', + pluginReactImportPath = '@rsbuild/plugin-react', + fixture = 'default', + parallelTransforms, +}) { + if (!stressFixtureNames.has(fixture)) { + throw new Error( + `Unknown benchmark fixture "${fixture}". Use ${benchmarkFixtureNames.join(', ')}.` + ); + } + + const isSpa = variant === 'spa'; + + await rm(root, { recursive: true, force: true }); + await mkdir(path.join(root, 'app/routes'), { recursive: true }); + + await writeFile( + path.join(root, 'package.json'), + JSON.stringify({ type: 'module', private: true }, null, 2) + ); + await writeFile( + path.join(root, 'rsbuild.config.mjs'), + createRsbuildConfig({ + variant, + sourceMap, + pluginImportPath, + pluginReactImportPath, + parallelTransforms, + }) + ); + await writeFile( + path.join(root, 'react-router.config.ts'), + createReactRouterConfig(variant) + ); + await writeFile( + path.join(root, 'app/routes.ts'), + createRoutesConfig(routeCount) + ); + await writeFile( + path.join(root, 'app/root.tsx'), + [ + `import { createElement } from 'react';`, + `import { Outlet, Scripts } from 'react-router';`, + `export function Layout({ children }) {`, + ` return createElement('html', null, createElement('head'), createElement('body', null, children, createElement(Scripts)));`, + `}`, + `export default function Root() { return createElement(Outlet); }`, + `export function ErrorBoundary() { return null; }`, + '', + ].join('\n') + ); + await writeFile( + path.join(root, 'app/client-data.client.ts'), + `export const clientValue = 'client';\nexport * from './client-extra.client';\n` + ); + await writeFile( + path.join(root, 'app/client-extra.client.ts'), + `export const extraClientValue = 'extra-client';\n` + ); + await writeFile( + path.join(root, 'app/server-data.server.ts'), + `export const serverValue = 'server';\n` + ); + + if (fixture === 'reexports') { + await writeReexportFixtures(root, routeCount, { isSpa }); + } + if (fixture === 'import-fanout') { + await writeFanoutFixtures(root); + } + + await Promise.all( + Array.from({ length: routeCount }, (_, routeIndex) => { + const index = routeIndex + 1; + const profile = routeExportProfiles[index % routeExportProfiles.length]; + return writeFile( + path.join(root, 'app', routeFile(index)), + `${createRouteModule(index, profile, { isSpa, fixture })}\n` + ); + }) + ); + + return { + root, + routeCount, + variant, + sourceMap, + fixture, + parallelTransforms, + }; +} diff --git a/scripts/compare-benchmarks.mjs b/scripts/compare-benchmarks.mjs new file mode 100644 index 0000000..89e663f --- /dev/null +++ b/scripts/compare-benchmarks.mjs @@ -0,0 +1,92 @@ +#!/usr/bin/env node +import { readFile } from 'node:fs/promises'; +import { parseArgs } from 'node:util'; + +const { values } = parseArgs({ + allowPositionals: false, + strict: true, + options: { + before: { type: 'string' }, + after: { type: 'string' }, + benchmark: { type: 'string', default: 'synthetic-256-ssr-esm-split' }, + }, +}); + +if (!values.before || !values.after) { + throw new Error( + 'Usage: node scripts/compare-benchmarks.mjs --before --after [--benchmark ]' + ); +} + +const readJson = async file => JSON.parse(await readFile(file, 'utf8')); +const before = await readJson(values.before); +const after = await readJson(values.after); + +const findBenchmark = (result, id) => { + const benchmark = result.benchmarks?.find(item => item.id === id); + if (!benchmark) { + throw new Error( + `Benchmark "${id}" not found in ${result.date ?? 'input'}.` + ); + } + return benchmark; +}; + +const metric = (benchmark, path) => + path.split('.').reduce((value, key) => value?.[key], benchmark); + +const percentDelta = (beforeValue, afterValue) => { + if (beforeValue == null || afterValue == null || beforeValue === 0) { + return '-'; + } + return `${(((afterValue - beforeValue) / beforeValue) * 100).toFixed(1)}%`; +}; + +const formatMs = value => + value == null ? '-' : `${(value / 1000).toFixed(2)}s`; +const formatKb = value => + value == null ? '-' : `${Math.round(value / 1024)} MB`; + +const beforeBenchmark = findBenchmark(before, values.benchmark); +const afterBenchmark = findBenchmark(after, values.benchmark); + +const rows = [ + { + label: 'Wall median', + before: metric(beforeBenchmark, 'summary.wallMs.median'), + after: metric(afterBenchmark, 'summary.wallMs.median'), + format: formatMs, + }, + { + label: 'CPU median (user+sys)', + before: + metric(beforeBenchmark, 'summary.userMs.median') == null || + metric(beforeBenchmark, 'summary.sysMs.median') == null + ? null + : metric(beforeBenchmark, 'summary.userMs.median') + + metric(beforeBenchmark, 'summary.sysMs.median'), + after: + metric(afterBenchmark, 'summary.userMs.median') == null || + metric(afterBenchmark, 'summary.sysMs.median') == null + ? null + : metric(afterBenchmark, 'summary.userMs.median') + + metric(afterBenchmark, 'summary.sysMs.median'), + format: formatMs, + }, + { + label: 'Peak RSS p95', + before: metric(beforeBenchmark, 'summary.maxRssKb.p95'), + after: metric(afterBenchmark, 'summary.maxRssKb.p95'), + format: formatKb, + }, +]; + +console.log(`Benchmark comparison: ${values.benchmark}`); +console.log(''); +console.log('| Metric | Before | After | Delta |'); +console.log('|---|---:|---:|---:|'); +for (const row of rows) { + console.log( + `| ${row.label} | ${row.format(row.before)} | ${row.format(row.after)} | ${percentDelta(row.before, row.after)} |` + ); +} diff --git a/scripts/report-benchmark-ci.mjs b/scripts/report-benchmark-ci.mjs new file mode 100644 index 0000000..5e0d167 --- /dev/null +++ b/scripts/report-benchmark-ci.mjs @@ -0,0 +1,167 @@ +#!/usr/bin/env node +import { mkdir, readFile, writeFile } from 'node:fs/promises'; +import path from 'node:path'; +import { parseArgs } from 'node:util'; + +const { values } = parseArgs({ + allowPositionals: false, + strict: true, + options: { + base: { type: 'string' }, + head: { type: 'string' }, + out: { type: 'string', default: '.benchmark/ci-report' }, + pr: { type: 'string' }, + 'base-ref': { type: 'string' }, + 'base-sha': { type: 'string' }, + 'head-ref': { type: 'string' }, + 'head-sha': { type: 'string' }, + 'run-url': { type: 'string' }, + }, +}); + +if (!values.base || !values.head) { + throw new Error( + 'Usage: node scripts/report-benchmark-ci.mjs --base --head [--out ]' + ); +} + +const readJson = async file => JSON.parse(await readFile(file, 'utf8')); +const formatSeconds = value => + typeof value === 'number' ? `${(value / 1000).toFixed(2)}s` : '-'; +const formatPercent = value => + typeof value === 'number' + ? `${value > 0 ? '+' : ''}${value.toFixed(1)}%` + : '-'; +const formatSpeedup = value => + typeof value === 'number' ? `${value.toFixed(2)}x` : '-'; +const formatRss = value => + typeof value === 'number' ? `${Math.round(value / 1024)} MB` : '-'; + +const percentDelta = (base, head) => + typeof base === 'number' && typeof head === 'number' && base !== 0 + ? ((head - base) / base) * 100 + : null; + +const speedup = (base, head) => + typeof base === 'number' && typeof head === 'number' && head !== 0 + ? base / head + : null; + +const medianWall = benchmark => benchmark?.summary?.wallMs?.median ?? null; +const p95Rss = benchmark => benchmark?.summary?.maxRssKb?.p95 ?? null; +const cpuMedian = benchmark => { + const user = benchmark?.summary?.userMs?.median; + const sys = benchmark?.summary?.sysMs?.median; + return typeof user === 'number' && typeof sys === 'number' + ? user + sys + : null; +}; + +const indexBenchmarks = result => + new Map( + (result.benchmarks ?? []).map(benchmark => [benchmark.id, benchmark]) + ); + +const base = await readJson(values.base); +const head = await readJson(values.head); +const baseBenchmarks = indexBenchmarks(base); +const headBenchmarks = indexBenchmarks(head); +const benchmarkIds = [ + ...new Set([...baseBenchmarks.keys(), ...headBenchmarks.keys()]), +].sort(); + +const benchmarks = benchmarkIds.map(id => { + const baseBenchmark = baseBenchmarks.get(id); + const headBenchmark = headBenchmarks.get(id); + const baseWallMs = medianWall(baseBenchmark); + const headWallMs = medianWall(headBenchmark); + return { + id, + routeCount: headBenchmark?.routeCount ?? baseBenchmark?.routeCount ?? null, + variant: headBenchmark?.variant ?? baseBenchmark?.variant ?? null, + baseWallMs, + headWallMs, + wallDeltaPercent: percentDelta(baseWallMs, headWallMs), + wallSpeedup: speedup(baseWallMs, headWallMs), + baseCpuMs: cpuMedian(baseBenchmark), + headCpuMs: cpuMedian(headBenchmark), + baseRssKb: p95Rss(baseBenchmark), + headRssKb: p95Rss(headBenchmark), + }; +}); + +const totalBaseWallMs = benchmarks.reduce( + (sum, benchmark) => sum + (benchmark.baseWallMs ?? 0), + 0 +); +const totalHeadWallMs = benchmarks.reduce( + (sum, benchmark) => sum + (benchmark.headWallMs ?? 0), + 0 +); + +const summary = { + baseWallMs: totalBaseWallMs, + headWallMs: totalHeadWallMs, + wallDeltaPercent: percentDelta(totalBaseWallMs, totalHeadWallMs), + wallSpeedup: speedup(totalBaseWallMs, totalHeadWallMs), +}; + +const report = { + generatedAt: new Date().toISOString(), + pullRequest: values.pr ?? null, + base: { + ref: values['base-ref'] ?? null, + sha: values['base-sha'] ?? base.commit ?? null, + benchmarkCommit: base.commit ?? null, + }, + head: { + ref: values['head-ref'] ?? null, + sha: values['head-sha'] ?? head.commit ?? null, + benchmarkCommit: head.commit ?? null, + }, + runUrl: values['run-url'] ?? null, + profile: head.profile ?? base.profile ?? null, + iterations: head.iterations ?? base.iterations ?? null, + warmup: head.warmup ?? base.warmup ?? null, + summary, + benchmarks, +}; + +const renderComment = () => { + const lines = [ + '', + '## Benchmark Results', + '', + `Compared PR head \`${report.head.sha?.slice(0, 7) ?? 'unknown'}\` against base \`${report.base.sha?.slice(0, 7) ?? 'unknown'}\`.`, + '', + `**Total median wall time:** ${formatSeconds(summary.baseWallMs)} -> ${formatSeconds(summary.headWallMs)} (${formatPercent(summary.wallDeltaPercent)}, ${formatSpeedup(summary.wallSpeedup)} speedup)`, + '', + '| Benchmark | Base | Head | Delta | Speedup | Head RSS p95 |', + '|---|---:|---:|---:|---:|---:|', + ]; + + for (const benchmark of benchmarks) { + lines.push( + `| \`${benchmark.id}\` | ${formatSeconds(benchmark.baseWallMs)} | ${formatSeconds(benchmark.headWallMs)} | ${formatPercent(benchmark.wallDeltaPercent)} | ${formatSpeedup(benchmark.wallSpeedup)} | ${formatRss(benchmark.headRssKb)} |` + ); + } + + lines.push( + '', + `Profile: \`${report.profile ?? 'unknown'}\`; iterations: \`${report.iterations ?? 'unknown'}\`; warmup: \`${report.warmup ?? 'unknown'}\`.`, + ...(report.runUrl ? [`[Workflow run](${report.runUrl})`] : []), + '' + ); + + return `${lines.join('\n')}\n`; +}; + +const outDir = path.resolve(values.out); +await mkdir(outDir, { recursive: true }); +await writeFile( + path.join(outDir, 'report.json'), + `${JSON.stringify(report, null, 2)}\n` +); +await writeFile(path.join(outDir, 'comment.md'), renderComment()); + +console.log(`Benchmark CI report written to ${outDir}`);