From 1728049993f72aeb006bc126507c6ff49b6a3e5b Mon Sep 17 00:00:00 2001 From: Claude Code Date: Mon, 6 Oct 2025 16:42:29 +0000 Subject: [PATCH 1/3] Add performance benchmark suite MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements Phase 6.3 Performance Regression Detection with: - Frame serialization benchmarks (4M+ ops/sec) - Connection operations benchmarks (25-35K ops/sec) - Vitest bench infrastructure with dedicated config - README documentation for benchmark interpretation Benchmarks cover: - Small text frames (17 bytes) - Medium binary frames (1KB) - Large binary frames (64KB) - Connection lifecycle and message sending - Ping/Pong frames ๐Ÿค– Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- .gitignore | 1 + CLAUDE.md | 6 +- package.json | 2 + test/benchmark/README.md | 60 +++++++++++++++++++ .../benchmark/connection-operations.bench.mjs | 53 ++++++++++++++++ test/benchmark/frame-operations.bench.mjs | 35 +++++++++++ vitest.bench.config.mjs | 11 ++++ 7 files changed, 167 insertions(+), 1 deletion(-) create mode 100644 test/benchmark/README.md create mode 100644 test/benchmark/connection-operations.bench.mjs create mode 100644 test/benchmark/frame-operations.bench.mjs create mode 100644 vitest.bench.config.mjs diff --git a/.gitignore b/.gitignore index 4b5838a8..7de598f2 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ test/scripts/heapdump/* test-results.json /coverage .claude/ +test-results/ diff --git a/CLAUDE.md b/CLAUDE.md index b755207e..e3873c19 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -3,7 +3,7 @@ ## Build/Test Commands ### Testing -- `pnpm test` - Run all vitest tests (632 unit + integration tests) +- `pnpm test` - Run all vitest tests (628 unit + integration tests) - `pnpm test:watch` - Run vitest in watch mode for development - `pnpm test:ui` - Run vitest with web UI interface - `pnpm test:coverage` - Run tests with coverage report (target: 85%+) @@ -14,6 +14,10 @@ - `pnpm test:autobahn` - Run Autobahn WebSocket Protocol Compliance Suite (517 tests) - `pnpx vitest run test/unit/[filename].test.mjs` - Run single vitest test file +### Performance Benchmarking +- `pnpm bench` - Run performance benchmarks for critical operations +- `pnpm bench:compare` - Compare current performance with previous baseline + ### Linting - `pnpm lint` - Check code for lint errors - `pnpm lint:fix` - Auto-fix lint errors (always run before commit) diff --git a/package.json b/package.json index b8d239f3..f28eb094 100644 --- a/package.json +++ b/package.json @@ -55,6 +55,8 @@ "test:browser:chromium": "playwright test --project=chromium", "test:browser:ui": "playwright test --ui", "test:autobahn": "cd test/autobahn && ./run-wstest.js", + "bench": "vitest bench --run --config vitest.bench.config.mjs", + "bench:compare": "vitest bench --run --config vitest.bench.config.mjs --compare", "lint": "eslint lib/**/*.js test/**/*.js", "lint:fix": "eslint lib/**/*.js test/**/*.js --fix" }, diff --git a/test/benchmark/README.md b/test/benchmark/README.md new file mode 100644 index 00000000..f20b8361 --- /dev/null +++ b/test/benchmark/README.md @@ -0,0 +1,60 @@ +# WebSocket-Node Performance Benchmarks + +This directory contains performance benchmarks for critical WebSocket operations. + +## Running Benchmarks + +```bash +# Run all benchmarks +pnpm run bench + +# Compare with previous results +pnpm run bench:compare +``` + +## Benchmark Suites + +### Frame Operations (`frame-operations.bench.mjs`) +Tests the performance of WebSocket frame serialization: +- Small text frames (17 bytes) - unmasked and masked +- Medium binary frames (1KB) +- Large binary frames (64KB) + +**Typical Results:** +- Frame serialization: ~4.3M ops/sec (unmasked), ~3M ops/sec (masked) +- Larger frames maintain similar performance due to efficient buffering + +### Connection Operations (`connection-operations.bench.mjs`) +Tests WebSocket connection-level operations: +- Connection instance creation +- Sending UTF-8 messages (small and 1KB) +- Sending binary messages (1KB) +- Ping/Pong frames + +**Typical Results:** +- Connection creation: ~30K ops/sec +- Message sending: ~25-35K ops/sec +- Ping/Pong: ~33-35K ops/sec + +## Interpreting Results + +Benchmarks output operations per second (hz) and timing statistics: +- **hz**: Operations per second (higher is better) +- **mean**: Average time per operation +- **p75/p99**: 75th/99th percentile latencies +- **rme**: Relative margin of error (lower is better) + +## Performance Baselines + +These benchmarks establish baseline performance for regression detection: +1. Frame serialization should maintain 4M+ ops/sec for small frames +2. Connection operations should maintain 25K+ ops/sec +3. Large message handling (64KB) should not degrade significantly + +## Adding New Benchmarks + +When adding benchmarks: +1. Pre-allocate buffers and data outside the benchmark loop +2. Use descriptive test names with size information +3. Focus on operations that directly impact production performance +4. Avoid testing implementation details diff --git a/test/benchmark/connection-operations.bench.mjs b/test/benchmark/connection-operations.bench.mjs new file mode 100644 index 00000000..dc15c13b --- /dev/null +++ b/test/benchmark/connection-operations.bench.mjs @@ -0,0 +1,53 @@ +import { bench, describe } from 'vitest'; +import WebSocketConnection from '../../lib/WebSocketConnection.js'; +import { MockSocket } from '../helpers/mocks.mjs'; + +describe('WebSocketConnection Performance', () => { + bench('create connection instance', () => { + const socket = new MockSocket(); + const connection = new WebSocketConnection(socket, [], 'echo-protocol', false, {}); + connection._addSocketEventListeners(); + }); + + bench('send small UTF-8 message', () => { + const socket = new MockSocket(); + const connection = new WebSocketConnection(socket, [], 'echo-protocol', false, {}); + connection._addSocketEventListeners(); + connection.state = 'open'; + connection.sendUTF('Hello, WebSocket!'); + }); + + bench('send medium UTF-8 message (1KB)', () => { + const socket = new MockSocket(); + const connection = new WebSocketConnection(socket, [], 'echo-protocol', false, {}); + connection._addSocketEventListeners(); + connection.state = 'open'; + const message = 'x'.repeat(1024); + connection.sendUTF(message); + }); + + bench('send binary message (1KB)', () => { + const socket = new MockSocket(); + const connection = new WebSocketConnection(socket, [], 'echo-protocol', false, {}); + connection._addSocketEventListeners(); + connection.state = 'open'; + const buffer = Buffer.alloc(1024); + connection.sendBytes(buffer); + }); + + bench('send ping frame', () => { + const socket = new MockSocket(); + const connection = new WebSocketConnection(socket, [], 'echo-protocol', false, {}); + connection._addSocketEventListeners(); + connection.state = 'open'; + connection.ping(); + }); + + bench('send pong frame', () => { + const socket = new MockSocket(); + const connection = new WebSocketConnection(socket, [], 'echo-protocol', false, {}); + connection._addSocketEventListeners(); + connection.state = 'open'; + connection.pong(); + }); +}); diff --git a/test/benchmark/frame-operations.bench.mjs b/test/benchmark/frame-operations.bench.mjs new file mode 100644 index 00000000..f4fc3acc --- /dev/null +++ b/test/benchmark/frame-operations.bench.mjs @@ -0,0 +1,35 @@ +import { bench, describe } from 'vitest'; +import WebSocketFrame from '../../lib/WebSocketFrame.js'; + +describe('WebSocketFrame Performance', () => { + // Pre-allocate payloads outside benchmark loops + const smallPayload = Buffer.from('Hello, WebSocket!'); + const mediumPayload = Buffer.alloc(1024); + mediumPayload.fill('x'); + const largePayload = Buffer.alloc(64 * 1024); + largePayload.fill('y'); + + // Pre-allocate mask + const mask = Buffer.from([0x12, 0x34, 0x56, 0x78]); + + bench('serialize small text frame (17 bytes, unmasked)', () => { + const frame = new WebSocketFrame(smallPayload, true, 0x01); + frame.toBuffer(); + }); + + bench('serialize small text frame (17 bytes, masked)', () => { + const frame = new WebSocketFrame(smallPayload, true, 0x01); + frame.mask = mask; + frame.toBuffer(); + }); + + bench('serialize medium binary frame (1KB)', () => { + const frame = new WebSocketFrame(mediumPayload, true, 0x02); + frame.toBuffer(); + }); + + bench('serialize large binary frame (64KB)', () => { + const frame = new WebSocketFrame(largePayload, true, 0x02); + frame.toBuffer(); + }); +}); diff --git a/vitest.bench.config.mjs b/vitest.bench.config.mjs new file mode 100644 index 00000000..00d879a8 --- /dev/null +++ b/vitest.bench.config.mjs @@ -0,0 +1,11 @@ +import { defineConfig } from 'vitest/config'; + +export default defineConfig({ + test: { + include: ['test/benchmark/**/*.bench.mjs'], + benchmark: { + include: ['test/benchmark/**/*.bench.mjs'], + exclude: ['node_modules/', 'test/unit/', 'test/integration/'], + }, + }, +}); From f2c1bce686e53d60bde9d2b48b0ca766f2600d97 Mon Sep 17 00:00:00 2001 From: Claude Code Date: Mon, 6 Oct 2025 16:44:13 +0000 Subject: [PATCH 2/3] Add performance baseline tracking and regression detection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Automated baseline save/compare script - 15% regression threshold for CI failures - npm scripts for baseline management - Initial performance baseline saved Commands: - pnpm bench:baseline - Save current perf as baseline - pnpm bench:check - Check for regressions (exits 1 if found) ๐Ÿค– Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- CLAUDE.md | 3 +- package.json | 3 +- test/benchmark/baseline.json | 21 +++++ test/benchmark/track-performance.mjs | 124 +++++++++++++++++++++++++++ 4 files changed, 149 insertions(+), 2 deletions(-) create mode 100644 test/benchmark/baseline.json create mode 100755 test/benchmark/track-performance.mjs diff --git a/CLAUDE.md b/CLAUDE.md index e3873c19..cb53d1e2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -16,7 +16,8 @@ ### Performance Benchmarking - `pnpm bench` - Run performance benchmarks for critical operations -- `pnpm bench:compare` - Compare current performance with previous baseline +- `pnpm bench:baseline` - Save current performance as baseline +- `pnpm bench:check` - Check for performance regressions (CI) ### Linting - `pnpm lint` - Check code for lint errors diff --git a/package.json b/package.json index f28eb094..fcdbd98e 100644 --- a/package.json +++ b/package.json @@ -56,7 +56,8 @@ "test:browser:ui": "playwright test --ui", "test:autobahn": "cd test/autobahn && ./run-wstest.js", "bench": "vitest bench --run --config vitest.bench.config.mjs", - "bench:compare": "vitest bench --run --config vitest.bench.config.mjs --compare", + "bench:baseline": "node test/benchmark/track-performance.mjs save", + "bench:check": "node test/benchmark/track-performance.mjs check", "lint": "eslint lib/**/*.js test/**/*.js", "lint:fix": "eslint lib/**/*.js test/**/*.js --fix" }, diff --git a/test/benchmark/baseline.json b/test/benchmark/baseline.json new file mode 100644 index 00000000..bcbd72d1 --- /dev/null +++ b/test/benchmark/baseline.json @@ -0,0 +1,21 @@ +{ + "timestamp": "2025-10-06T16:43:17.731Z", + "results": { + "WebSocketConnection Performance 3866ms": { + "create connection instance": 28323.39, + "send small UTF-8 message": 28201.35, + "send medium UTF-8 message (1KB)": 24615.97, + "send binary message (1KB)": 24889.31, + "send ping frame": 31562.37, + "send pong frame": 32393.72 + }, + "WebSocketFrame Performance 10001ms": { + "serialize small text frame (17 bytes, unmasked)": 4427042.23, + "serialize small text frame (17 bytes, masked)": 3005215.87, + "serialize medium binary frame (1KB)": 4239270.36, + "serialize large binary frame (64KB)": 4024552.34 + }, + "WebSocketConnection Performance": {}, + "WebSocketFrame Performance": {} + } +} \ No newline at end of file diff --git a/test/benchmark/track-performance.mjs b/test/benchmark/track-performance.mjs new file mode 100755 index 00000000..ba959b19 --- /dev/null +++ b/test/benchmark/track-performance.mjs @@ -0,0 +1,124 @@ +#!/usr/bin/env node + +/** + * Performance Baseline Tracking Script + * + * This script runs benchmarks and tracks performance over time. + * It can save baselines and compare current performance against them. + * + * Usage: + * node test/benchmark/track-performance.mjs save # Save current results as baseline + * node test/benchmark/track-performance.mjs compare # Compare with baseline + * node test/benchmark/track-performance.mjs check # Check for regressions (CI) + */ + +import { exec } from 'child_process'; +import { readFileSync, writeFileSync, existsSync } from 'fs'; +import { promisify } from 'util'; + +const execAsync = promisify(exec); +const BASELINE_FILE = 'test/benchmark/baseline.json'; +const REGRESSION_THRESHOLD = 0.15; // 15% regression threshold + +async function runBenchmarks() { + console.log('Running benchmarks...'); + const { stdout } = await execAsync('pnpm run bench 2>&1'); + return parseBenchmarkOutput(stdout); +} + +function parseBenchmarkOutput(output) { + const results = {}; + const lines = output.split('\n'); + + let currentSuite = null; + for (const line of lines) { + // Detect suite name + if (line.includes('> WebSocket')) { + currentSuite = line.match(/> (.*)/)[1].trim(); + results[currentSuite] = {}; + } + + // Parse benchmark results + const benchMatch = line.match(/^\s*[ยทโ€ข]\s+(.+?)\s+(\d+(?:,\d+)*(?:\.\d+)?)\s/); + if (benchMatch && currentSuite) { + const [, name, hz] = benchMatch; + results[currentSuite][name.trim()] = parseFloat(hz.replace(/,/g, '')); + } + } + + return results; +} + +async function saveBaseline() { + const results = await runBenchmarks(); + writeFileSync(BASELINE_FILE, JSON.stringify({ + timestamp: new Date().toISOString(), + results + }, null, 2)); + console.log(`\nโœ… Baseline saved to ${BASELINE_FILE}`); +} + +async function compareWithBaseline() { + if (!existsSync(BASELINE_FILE)) { + console.error(`โŒ No baseline found at ${BASELINE_FILE}`); + console.log('Run: node test/benchmark/track-performance.mjs save'); + process.exit(1); + } + + const baseline = JSON.parse(readFileSync(BASELINE_FILE, 'utf-8')); + const current = await runBenchmarks(); + + console.log(`\n๐Ÿ“Š Comparing with baseline from ${baseline.timestamp}\n`); + + let hasRegression = false; + + for (const [suite, tests] of Object.entries(current)) { + if (!baseline.results[suite]) continue; + + console.log(`\n${suite}:`); + + for (const [test, currentHz] of Object.entries(tests)) { + const baselineHz = baseline.results[suite][test]; + if (!baselineHz) continue; + + const change = (currentHz - baselineHz) / baselineHz; + const changePercent = (change * 100).toFixed(2); + + let status = 'โœ“'; + if (change < -REGRESSION_THRESHOLD) { + status = 'โŒ REGRESSION'; + hasRegression = true; + } else if (change < -0.05) { + status = 'โš ๏ธ SLOWER'; + } else if (change > 0.05) { + status = '๐Ÿš€ FASTER'; + } + + console.log(` ${status} ${test}`); + console.log(` ${baselineHz.toLocaleString()} โ†’ ${currentHz.toLocaleString()} ops/sec (${changePercent}%)`); + } + } + + if (hasRegression) { + console.log(`\nโŒ Performance regressions detected (>${REGRESSION_THRESHOLD * 100}% slower)`); + process.exit(1); + } else { + console.log('\nโœ… No performance regressions detected'); + } +} + +const command = process.argv[2]; + +if (command === 'save') { + await saveBaseline(); +} else if (command === 'compare' || command === 'check') { + await compareWithBaseline(); +} else { + console.log(` +Usage: + node test/benchmark/track-performance.mjs save # Save baseline + node test/benchmark/track-performance.mjs compare # Compare with baseline + node test/benchmark/track-performance.mjs check # Check for regressions (CI) +`); + process.exit(1); +} From 468d04ff96418729f651de722e42206d4678a860 Mon Sep 17 00:00:00 2001 From: Claude Code Date: Mon, 6 Oct 2025 16:44:33 +0000 Subject: [PATCH 3/3] Add GitHub Actions workflow for performance benchmarks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Runs benchmarks on PRs that modify lib/ or benchmarks - Manual workflow_dispatch trigger available - Regression check is informational (warning only) - Uses Node.js 20.x for consistent results ๐Ÿค– Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- .github/workflows/performance.yml | 33 +++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 .github/workflows/performance.yml diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml new file mode 100644 index 00000000..77a81069 --- /dev/null +++ b/.github/workflows/performance.yml @@ -0,0 +1,33 @@ +name: performance +on: + pull_request: + paths: + - 'lib/**' + - 'test/benchmark/**' + workflow_dispatch: + +jobs: + benchmark: + runs-on: ubuntu-latest + name: Performance Benchmarks + steps: + - uses: actions/checkout@v2 + + - uses: pnpm/action-setup@v2 + with: + version: 8 + + - uses: actions/setup-node@v3 + with: + node-version: 20.x + cache: 'pnpm' + + - name: Install dependencies + run: pnpm install + + - name: Run benchmarks + run: pnpm run bench + + - name: Check for regressions (informational) + run: pnpm run bench:check || echo "::warning::Performance regressions detected. Review benchmark results." + continue-on-error: true