feat(bench): add npm run bench + bench:cold-start runners

package.json

@@ -13,6 +13,8 @@
     "sync-templates": "node scripts/sync-templates.mjs",
     "test": "vitest run",
     "test:watch": "vitest",
+    "bench": "node scripts/run-benchmarks.mjs",
+    "bench:cold-start": "node scripts/measure-cold-start.mjs",
     "lint": "tsc --noEmit -p packages/squad-sdk/tsconfig.json && tsc --noEmit -p packages/squad-cli/tsconfig.json",
     "lint:eslint": "eslint packages/ test/",
     "lint:docs": "markdownlint-cli2 && cspell --no-progress --dot \"docs/src/content/**/*.md\" \"README.md\"",

scripts/measure-cold-start.mjs

@@ -0,0 +1,210 @@
+#!/usr/bin/env node
+/**
+ * scripts/measure-cold-start.mjs
+ *
+ * Measure CLI cold-start latency for `squad --version` and `squad help`
+ * over N runs.
+ *
+ * Why this script exists
+ * ----------------------
+ * The squad-cli entry point (packages/squad-cli/src/cli-entry.ts, ~28 KB)
+ * has many top-level imports plus postinstall ESM patches that affect
+ * cold-start latency. v0.8.23 added lazy imports for some commands; this
+ * script lets us see whether further deferral helps.
+ *
+ * Usage
+ * -----
+ *   npm run bench:cold-start                       # 5 runs of each command
+ *   npm run bench:cold-start -- --runs=10
+ *   npm run bench:cold-start -- --json
+ *
+ * Exit codes
+ * ----------
+ *   0  success
+ *   1  CLI not built or runtime error
+ */
+
+import { spawn } from 'node:child_process';
+import { argv, exit, stdout, stderr, execPath } from 'node:process';
+import { resolve as resolvePath, dirname } from 'node:path';
+import { existsSync } from 'node:fs';
+import { fileURLToPath } from 'node:url';
+import { performance } from 'node:perf_hooks';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+const REPO_ROOT = resolvePath(__dirname, '..');
+const CLI_ENTRY = resolvePath(REPO_ROOT, 'packages/squad-cli/dist/cli-entry.js');
+
+const DEFAULT_COMMANDS = [
+  ['--version'],
+  ['help'],
+];
+
+function parseArgs(args) {
+  const opts = { runs: 5, json: false };
+  for (const arg of args) {
+    if (arg === '--json') opts.json = true;
+    else if (arg === '--help' || arg === '-h') opts.help = true;
+    else if (arg.startsWith('--runs=')) {
+      const n = Number.parseInt(arg.slice('--runs='.length), 10);
+      if (!Number.isFinite(n) || n < 1) {
+        throw new Error(`--runs must be a positive integer, got: ${arg}`);
+      }
+      opts.runs = n;
+    } else if (arg.startsWith('-')) {
+      throw new Error(`Unknown argument: ${arg}`);
+    }
+  }
+  return opts;
+}
+
+function printHelp() {
+  stdout.write(
+    [
+      'Usage: npm run bench:cold-start -- [options]',
+      '',
+      'Options:',
+      '  --runs=N          Number of runs per command (default: 5)',
+      '  --json            Emit JSON instead of a formatted table',
+      '  -h, --help        Show this help',
+      '',
+      'Note: requires `npm run build` to have produced packages/squad-cli/dist/.',
+      '',
+    ].join('\n'),
+  );
+}
+
+/**
+ * Run the CLI once with the given args; resolve with elapsed milliseconds.
+ *
+ * We measure wall-clock from spawn() until the child fully exits. This
+ * captures everything the user perceives as cold start: process spawn,
+ * Node startup, ESM patching, top-level imports, and the actual command.
+ */
+function timeSingleRun(args) {
+  return new Promise((resolveP, rejectP) => {
+    const start = performance.now();
+    const child = spawn(execPath, [CLI_ENTRY, ...args], {
+      stdio: ['ignore', 'ignore', 'pipe'],
+      env: { ...process.env, NO_COLOR: '1' },
+    });
+    let stderrBuf = '';
+    child.stderr.on('data', (chunk) => {
+      stderrBuf += chunk.toString();
+    });
+    child.on('error', rejectP);
+    child.on('exit', (code) => {
+      const elapsed = performance.now() - start;
+      if (code !== 0) {
+        // Some help/version paths may exit non-zero on errors; surface stderr
+        // but still record the timing so users see what happened.
+        stderr.write(`(exit ${code} from squad ${args.join(' ')}): ${stderrBuf.trim()}\n`);
+      }
+      resolveP({ elapsed, code });
+    });
+  });
+}
+
+function summarise(timings) {
+  const sorted = [...timings].sort((a, b) => a - b);
+  const sum = sorted.reduce((a, b) => a + b, 0);
+  const avg = sum / sorted.length;
+  const idx95 = Math.max(0, Math.ceil(0.95 * sorted.length) - 1);
+  return {
+    runs: sorted.length,
+    avg,
+    min: sorted[0],
+    max: sorted[sorted.length - 1],
+    p95: sorted[idx95],
+  };
+}
+
+async function main() {
+  let opts;
+  try {
+    opts = parseArgs(argv.slice(2));
+  } catch (err) {
+    stderr.write(`error: ${err.message}\n`);
+    printHelp();
+    exit(1);
+  }
+
+  if (opts.help) {
+    printHelp();
+    exit(0);
+  }
+
+  if (!existsSync(CLI_ENTRY)) {
+    stderr.write(
+      `error: CLI build output not found at ${CLI_ENTRY}\n` +
+        'Run `npm run build` first.\n',
+    );
+    exit(1);
+  }
+
+  const results = [];
+  let anyFailures = false;
+  for (const args of DEFAULT_COMMANDS) {
+    const label = `squad ${args.join(' ')}`;
+    const timings = [];
+    let failures = 0;
+    for (let i = 0; i < opts.runs; i++) {
+      const { elapsed, code } = await timeSingleRun(args);
+      timings.push(elapsed);
+      if (code !== 0) {
+        failures++;
+        anyFailures = true;
+      }
+    }
+    results.push({ command: label, timings, failures, ...summarise(timings) });
+  }
+
+  if (opts.json) {
+    stdout.write(JSON.stringify({ results, timestamp: new Date().toISOString() }, null, 2) + '\n');
+    exit(anyFailures ? 1 : 0);
+  }
+
+  // Formatted table
+  const COL_CMD = 24;
+  const COL_NUM = 12;
+  const header = [
+    'Command'.padEnd(COL_CMD),
+    'Runs'.padStart(COL_NUM),
+    'Fails'.padStart(COL_NUM),
+    'Avg (ms)'.padStart(COL_NUM),
+    'Min (ms)'.padStart(COL_NUM),
+    'Max (ms)'.padStart(COL_NUM),
+    'P95 (ms)'.padStart(COL_NUM),
+  ].join('  ');
+  const sep = '─'.repeat(header.length);
+  stdout.write(sep + '\n');
+  stdout.write(header + '\n');
+  stdout.write(sep + '\n');
+  for (const r of results) {
+    stdout.write(
+      [
+        r.command.padEnd(COL_CMD),
+        String(r.runs).padStart(COL_NUM),
+        String(r.failures).padStart(COL_NUM),
+        r.avg.toFixed(2).padStart(COL_NUM),
+        r.min.toFixed(2).padStart(COL_NUM),
+        r.max.toFixed(2).padStart(COL_NUM),
+        r.p95.toFixed(2).padStart(COL_NUM),
+      ].join('  ') + '\n',
+    );
+  }
+  stdout.write(sep + '\n');
+  if (anyFailures) {
+    stderr.write(
+      'warning: one or more runs exited with a non-zero status — timings are still reported, but the CLI is broken.\n',
+    );
+  }
+
+  exit(anyFailures ? 1 : 0);
+}
+
+main().catch((err) => {
+  stderr.write(`error: ${err.stack ?? err.message ?? String(err)}\n`);
+  exit(1);
+});

scripts/run-benchmarks.mjs

@@ -0,0 +1,149 @@
+#!/usr/bin/env node
+/**
+ * scripts/run-benchmarks.mjs
+ *
+ * Foundation runner for the existing @bradygaster/squad-sdk BenchmarkSuite.
+ *
+ * Why this script exists
+ * ----------------------
+ * The SDK already ships `BenchmarkSuite` (packages/squad-sdk/src/runtime/benchmarks.ts)
+ * with five built-in micro-benchmarks (configLoad, charterCompile, routing,
+ * modelSelection, exportImport). Until now there was no `npm run bench` entry
+ * point, so the suite was only used programmatically and from its own tests.
+ *
+ * This script gives perf-improvement PRs a single command to capture
+ * before/after numbers consistently. It is intentionally tiny — all logic
+ * lives in `BenchmarkSuite`; this is a CLI shim.
+ *
+ * Usage
+ * -----
+ *   npm run bench                      # default 100 iterations, all benches
+ *   npm run bench -- --iterations=50   # custom iteration count
+ *   npm run bench -- --json            # emit JSON (for CI diffing)
+ *   npm run bench -- --filter routing  # only run benches whose name matches
+ *
+ * Exit codes
+ * ----------
+ *   0  success
+ *   1  argument error or runtime failure
+ *
+ * Note: this script imports from the BUILT SDK (`packages/squad-sdk/dist/...`).
+ * Run `npm run build` first if you have not built the workspace.
+ */
+
+import { argv, exit, stdout, stderr } from 'node:process';
+import { pathToFileURL } from 'node:url';
+import { resolve as resolvePath, dirname } from 'node:path';
+import { existsSync } from 'node:fs';
+import { fileURLToPath } from 'node:url';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+const REPO_ROOT = resolvePath(__dirname, '..');
+const SDK_DIST = resolvePath(REPO_ROOT, 'packages/squad-sdk/dist/runtime/benchmarks.js');
+
+function parseArgs(args) {
+  const opts = { iterations: 100, json: false, filter: null };
+  for (let i = 0; i < args.length; i++) {
+    const arg = args[i];
+    if (arg === '--json') {
+      opts.json = true;
+    } else if (arg === '--help' || arg === '-h') {
+      opts.help = true;
+    } else if (arg.startsWith('--iterations=')) {
+      const n = Number.parseInt(arg.slice('--iterations='.length), 10);
+      if (!Number.isFinite(n) || n < 1) {
+        throw new Error(`--iterations must be a positive integer, got: ${arg}`);
+      }
+      opts.iterations = n;
+    } else if (arg.startsWith('--filter=')) {
+      opts.filter = arg.slice('--filter='.length);
+    } else if (arg === '--filter') {
+      const next = args[i + 1];
+      if (next === undefined || next.startsWith('-')) {
+        throw new Error('--filter requires a value');
+      }
+      opts.filter = next;
+      i++;
+    } else {
+      throw new Error(`Unknown argument: ${arg}`);
+    }
+  }
+  return opts;
+}
+
+function printHelp() {
+  stdout.write(
+    [
+      'Usage: npm run bench -- [options]',
+      '',
+      'Options:',
+      '  --iterations=N    Number of iterations per benchmark (default: 100)',
+      '  --filter=NAME     Only run benchmarks whose name contains NAME',
+      '  --json            Emit JSON instead of a formatted table',
+      '  -h, --help        Show this help',
+      '',
+      'Note: requires `npm run build` to have produced packages/squad-sdk/dist/.',
+      '',
+    ].join('\n'),
+  );
+}
+
+async function main() {
+  let opts;
+  try {
+    opts = parseArgs(argv.slice(2));
+  } catch (err) {
+    stderr.write(`error: ${err.message}\n`);
+    printHelp();
+    exit(1);
+  }
+
+  if (opts.help) {
+    printHelp();
+    exit(0);
+  }
+
+  if (!existsSync(SDK_DIST)) {
+    stderr.write(
+      `error: SDK build output not found at ${SDK_DIST}\n` +
+        'Run `npm run build` first.\n',
+    );
+    exit(1);
+  }
+
+  const { BenchmarkSuite, formatBenchmarkReport } = await import(
+    pathToFileURL(SDK_DIST).href
+  );
+
+  const suite = new BenchmarkSuite();
+
+  if (opts.filter) {
+    const keep = new Set(suite.list().filter((n) => n.includes(opts.filter)));
+    if (keep.size === 0) {
+      stderr.write(`error: no benchmarks match filter "${opts.filter}"\n`);
+      stderr.write(`available: ${suite.list().join(', ')}\n`);
+      exit(1);
+    }
+    for (const name of suite.list()) {
+      if (!keep.has(name)) suite.unregister(name);
+    }
+  }
+
+  const report = await suite.runAll(opts.iterations);
+
+  if (opts.json) {
+    stdout.write(JSON.stringify(report, null, 2) + '\n');
+  } else {
+    stdout.write(formatBenchmarkReport(report.results) + '\n');
+    stdout.write(`\nTotal time: ${report.totalTime.toFixed(2)} ms\n`);
+    stdout.write(`Timestamp:  ${report.timestamp}\n`);
+  }
+
+  exit(0);
+}
+
+main().catch((err) => {
+  stderr.write(`error: ${err.stack ?? err.message ?? String(err)}\n`);
+  exit(1);
+});