From eee9d1602e3a3cefce874a9ee254891d12a6c376 Mon Sep 17 00:00:00 2001 From: Titouan Mathis Date: Sun, 16 Aug 2026 13:45:20 +0000 Subject: [PATCH 1/6] test(v4): measure and guard mounting at page scale MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `mount.bench.ts` measures one instance. Nothing measured a page of them, and v4 had no performance coverage in CI at all, so the demo page that stalled while mounting had no number to point at. Two layers. `mount-at-scale.bench.ts` reports throughput in a real Chromium for flat, four-deep, realistic, batched, destroyed, `in-view` and responsive-option pages at 100, 1 000 and 5 000 components, against a control of declared-but-unregistered elements. `mount-at-scale.spec.ts` turns the findings into blocking guards. The measurement corrects the premise. Eager mounting already chunks: `applyMountStrategy()` posts one background task per element, drained inside the scheduler's 5 ms budget. What does not chunk is the observer batch — `processMutations()` scans an inserted subtree and tears down a removed one in one synchronous pass — so blocking cost tracks DOM nodes, not component weight. The long-task cliff sits near 12 000 inserted nodes and 13 000 removed ones, and 4 000 realistic components settle in 363 ms while blocking for only 59 ms of it. Guards therefore stay far under that cliff wherever the threshold would depend on machine speed, and are tight only where it does not: five times the components must stay within 2.5x per component, and a four-deep tree within 1.8x of a flat one. Both sides of each ratio are measured in the tens of milliseconds, because taking the best of several repeats favours the shorter side — a 3 ms run dodges the interruption a 70 ms run absorbs — and a ratio inherits that bias. All four guards survive four repeats quiet and three with every core saturated. `V4_BENCH_SIZES` narrows the matrix so CI can compare a subset, and the config's reference to a `packages/tests` CodSpeed suite — which does not exist in this repository — is replaced by what actually guards these. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_011nNdFD3aQhzfdm3EsCSbS9 --- packages/v4/scripts/build.js | 6 +- packages/v4/scripts/check-package.js | 5 +- packages/v4/src/mount-at-scale.bench.ts | 159 +++++++++++++++++++ packages/v4/src/mount-at-scale.fixtures.ts | 145 ++++++++++++++++++ packages/v4/src/mount-at-scale.spec.ts | 169 +++++++++++++++++++++ packages/v4/vitest.bench.config.js | 21 ++- 6 files changed, 498 insertions(+), 7 deletions(-) create mode 100644 packages/v4/src/mount-at-scale.bench.ts create mode 100644 packages/v4/src/mount-at-scale.fixtures.ts create mode 100644 packages/v4/src/mount-at-scale.spec.ts diff --git a/packages/v4/scripts/build.js b/packages/v4/scripts/build.js index 1ce81f74b..9dc7fdf06 100644 --- a/packages/v4/scripts/build.js +++ b/packages/v4/scripts/build.js @@ -6,14 +6,16 @@ const pkgRoot = resolve(dirname(new URL(import.meta.url).pathname), '..'); const srcRoot = resolve(pkgRoot, 'src'); const outDir = resolve(pkgRoot, 'dist'); -// Every consumer module under `src/`. Specs, benchmarks and test utilities stay -// source-only. `unbundle` keeps the emitted `dist/` tree one-to-one with the entries. +// Every consumer module under `src/`. Specs, benchmarks, fixtures and test +// utilities stay source-only. `unbundle` keeps the emitted `dist/` tree +// one-to-one with the entries. const entryPoints = glob.sync( [ '**/*.ts', '!**/*.d.ts', '!**/*.spec.ts', '!**/*.bench.ts', + '!**/*.fixtures.ts', '!test-utils.ts', '!**/node_modules/**', ], diff --git a/packages/v4/scripts/check-package.js b/packages/v4/scripts/check-package.js index 193337c21..9d05ee29e 100644 --- a/packages/v4/scripts/check-package.js +++ b/packages/v4/scripts/check-package.js @@ -55,12 +55,13 @@ function assertPackageContent(metadata) { const forbiddenTests = files.filter( (path) => - path.startsWith('dist/test-utils.') || /\.(?:spec|bench)\.(?:js|js\.map|d\.ts)$/.test(path), + path.startsWith('dist/test-utils.') || + /\.(?:spec|bench|fixtures)\.(?:js|js\.map|d\.ts)$/.test(path), ); assert.deepEqual( forbiddenTests, [], - `Test utilities, specs and benchmarks must not be packed:\n${forbiddenTests.join('\n')}`, + `Test utilities, specs, benchmarks and fixtures must not be packed:\n${forbiddenTests.join('\n')}`, ); assert(fileSet.has('dist/index.js'), 'dist/index.js is missing from the package.'); diff --git a/packages/v4/src/mount-at-scale.bench.ts b/packages/v4/src/mount-at-scale.bench.ts new file mode 100644 index 000000000..cc78c09b3 --- /dev/null +++ b/packages/v4/src/mount-at-scale.bench.ts @@ -0,0 +1,159 @@ +/** + * Mounting throughput at page scale. + * + * `mount.bench.ts` measures one instance; this measures a page of them. The + * quantity that matters is not the total but the cost per component, and + * whether it stays flat as the count grows — `.github/actions/bench-diff` + * divides each result by the count in its group title to report it. + * + * Every number here is a settle time, from the DOM write to + * `whenDOMSettled()`. It is not a blocking time: eager mounts are posted one + * background task per element and drained inside the scheduler's 5 ms budget. + * `mount-at-scale.spec.ts` guards the part that does block. + */ +import { bench, describe } from 'vitest'; +import { whenDOMSettled } from './dom-mutations.js'; +import { + buildPool, + clearDocument, + registerScaleComponents, + scenarios, + type ScenarioName, +} from './mount-at-scale.fixtures.js'; + +registerScaleComponents(); + +/** Injected by `vitest.bench.config.js` from `V4_BENCH_SIZES`. */ +declare const __BENCH_SIZES__: number[]; + +/** The page weights the design has to hold up under. */ +const SIZES = __BENCH_SIZES__; + +/** Arrivals for the batching comparison — one morph per chunk. */ +const BATCHES = 10; + +/** + * Sample counts, not sample time: 5 000 realistic components take ~500 ms to + * settle, so tinybench's 500 ms default would stop at its ten-iteration floor + * for the small sizes and at one sample for the large ones. Fixing the count + * keeps every size comparably sampled and the suite bounded. + */ +function samplesFor(size: number) { + return { + time: 0, + iterations: size >= 5000 ? 9 : size >= 1000 ? 13 : 17, + warmupTime: 0, + warmupIterations: 1, + }; +} + +/** + * Insert a prebuilt tree and wait for the registry to finish with it. + * + * Trees are cloned in the per-cycle `setup` hook, never in the timed body: + * vitest exposes no per-iteration hook, so anything the body touches is + * measured. Hosts accumulate across a cycle's iterations, which is harmless — + * the registry scans the inserted subtree only, never the document. + */ +function mountBench(label: string, scenario: ScenarioName, size: number): void { + const options = samplesFor(size); + let pool: HTMLElement[] = []; + let index = 0; + + bench( + label, + async () => { + document.body.append(pool[index]); + index += 1; + await whenDOMSettled(); + }, + { + ...options, + async setup() { + await clearDocument(); + pool = buildPool(scenarios[scenario](size), options.iterations + 1); + index = 0; + }, + // Drop the cycle's trees so the next benchmark does not start under the + // heap pressure this one left behind. + teardown() { + document.body.replaceChildren(); + pool = []; + }, + }, + ); +} + +describe.each(SIZES)('mount %i components, one insertion', (size) => { + mountBench('control — declared but unregistered', 'control', size); + mountBench('flat', 'flat', size); + mountBench('nested 4 deep', 'nested', size); + mountBench('realistic — 5 refs, 3 options, 4 handlers', 'realistic', size); + mountBench('in-view — one controller per element', 'inView', size); + mountBench('responsive option — breakpoint cascade per mount', 'responsive', size); +}); + +describe.each(SIZES)(`mount %i flat components, 1 vs ${BATCHES} insertions`, (size) => { + const options = samplesFor(size); + let pool: HTMLElement[][] = []; + let index = 0; + + mountBench('1 insertion', 'flat', size); + + bench( + `${BATCHES} insertions`, + async () => { + for (const chunk of pool[index]) { + document.body.append(chunk); + // Settling per chunk is what makes them separate observer deliveries. + await whenDOMSettled(); + } + index += 1; + }, + { + ...options, + async setup() { + await clearDocument(); + pool = Array.from({ length: options.iterations + 1 }, () => + buildPool(scenarios.flat(Math.ceil(size / BATCHES)), BATCHES), + ); + index = 0; + }, + teardown() { + document.body.replaceChildren(); + pool = []; + }, + }, + ); +}); + +describe.each(SIZES)('destroy %i flat components, one removal', (size) => { + const options = samplesFor(size); + let pool: HTMLElement[] = []; + let index = 0; + + bench( + 'flat', + async () => { + pool[index].remove(); + index += 1; + await whenDOMSettled(); + }, + { + ...options, + async setup() { + await clearDocument(); + pool = buildPool(scenarios.flat(size), options.iterations + 1); + for (const host of pool) { + document.body.append(host); + } + await whenDOMSettled(); + index = 0; + }, + teardown() { + document.body.replaceChildren(); + pool = []; + }, + }, + ); +}); diff --git a/packages/v4/src/mount-at-scale.fixtures.ts b/packages/v4/src/mount-at-scale.fixtures.ts new file mode 100644 index 000000000..a7f0a413a --- /dev/null +++ b/packages/v4/src/mount-at-scale.fixtures.ts @@ -0,0 +1,145 @@ +/** + * Component shapes and markup for the at-scale mounting measurements. + * + * The throughput benchmarks and the long-task guard specs must describe the + * same page, or a guard would defend a shape nobody measured. They therefore + * share these fixtures instead of restating them. + */ +import { Base, type BaseConfig, type RefEvent } from './Base.js'; +import { whenDOMSettled } from './dom-mutations.js'; +import { registerComponents } from './registry.js'; + +/** Nothing declared: the floor of what mounting one component costs. */ +class ScaleFlat extends Base { + static config: BaseConfig = { name: 'ScaleFlat' }; +} + +/** + * A four-deep chain. Each level declares the next so the whole family + * registers, and no level declares a handler: the comparison against + * `ScaleFlat` is about tree shape alone. + */ +class ScaleDepth4 extends Base { + static config: BaseConfig = { name: 'ScaleDepth4' }; +} + +class ScaleDepth3 extends Base { + static config: BaseConfig = { name: 'ScaleDepth3', components: { ScaleDepth4 } }; +} + +class ScaleDepth2 extends Base { + static config: BaseConfig = { name: 'ScaleDepth2', components: { ScaleDepth3 } }; +} + +class ScaleDepth1 extends Base { + static config: BaseConfig = { name: 'ScaleDepth1', components: { ScaleDepth2 } }; +} + +/** Five refs, three declared options and four handlers — an ordinary component. */ +class ScaleRealistic extends Base<{ + $refs: { trigger: HTMLElement; panel: HTMLElement; items: HTMLElement[]; label: HTMLElement }; + $options: { columns: number; label: string; open: boolean }; +}> { + static config: BaseConfig = { + name: 'ScaleRealistic', + refs: ['trigger', 'panel', 'items[]', 'label', 'close'], + options: { columns: Number, label: String, open: Boolean }, + }; + + count = 0; + + mounted(): void { + // Real components read their options and touch their refs on mount. + this.count = this.$options.columns + this.$refs.items.length; + this.$refs.panel.hidden = !this.$options.open; + } + + onTriggerClick(): void { + this.count += 1; + } + + onItemsPointerdown({ index }: RefEvent): void { + this.count += index; + } + + onLabelMouseover(): void { + this.count += 1; + } + + onCloseClick(): void { + this.count -= 1; + } +} + +/** One responsive option, so every mount walks the breakpoint cascade. */ +class ScaleResponsive extends Base<{ $options: { columns: number } }> { + static config: BaseConfig = { + name: 'ScaleResponsive', + options: { columns: Number }, + }; + + count = 0; + + mounted(): void { + this.count = this.$options.columns; + } +} + +/** Register every at-scale fixture. Idempotent, like `registerComponent()`. */ +export function registerScaleComponents(): void { + registerComponents(ScaleFlat, ScaleDepth1, ScaleRealistic, ScaleResponsive); +} + +/** How many `data-component` elements each scenario's markup holds. */ +export const NESTING_DEPTH = 4; + +const REALISTIC_MARKUP = `
+ +
+ + +
`; + +/** + * Markup builders, keyed by scenario. Each returns exactly `size` + * `data-component` elements, so results divide into a per-component cost that + * is comparable across scenarios. + */ +export const scenarios = { + /** + * Nothing registered under this name: the mutation observer still delivers + * the records and the registry still walks the subtree, so this is the + * discovery floor every other scenario is built on. + */ + control: (size: number) => `
`.repeat(size), + flat: (size: number) => `
`.repeat(size), + nested: (size: number) => + `
`.repeat( + Math.ceil(size / NESTING_DEPTH), + ), + realistic: (size: number) => REALISTIC_MARKUP.repeat(size), + inView: (size: number) => + `
`.repeat(size), + responsive: (size: number) => + `
`.repeat( + size, + ), +} satisfies Record string>; + +export type ScenarioName = keyof typeof scenarios; + +/** + * Clone-ready trees, built before the timer starts. Parsing markup costs more + * than mounting it, so no measurement may include `innerHTML`. + */ +export function buildPool(html: string, count: number): HTMLElement[] { + const template = document.createElement('div'); + template.innerHTML = html; + return Array.from({ length: count }, () => template.cloneNode(true) as HTMLElement); +} + +/** Empty the document and let the registry finish tearing everything down. */ +export async function clearDocument(): Promise { + document.body.replaceChildren(); + await whenDOMSettled(); +} diff --git a/packages/v4/src/mount-at-scale.spec.ts b/packages/v4/src/mount-at-scale.spec.ts new file mode 100644 index 000000000..3ebd88ae4 --- /dev/null +++ b/packages/v4/src/mount-at-scale.spec.ts @@ -0,0 +1,169 @@ +/** + * Blocking guards for mounting at page scale. + * + * `mount-at-scale.bench.ts` reports a number; this file fails the build. + * + * What is guarded, and why these shapes. Eager mounting is already + * time-sliced: `applyMountStrategy()` posts one background task per element + * and the scheduler's 5 ms budget bounds each turn. What is *not* sliced is + * the observer batch itself — `processMutations()` scans an inserted subtree + * and tears down a removed one in one synchronous pass, so the blocking cost + * of a page grows with its DOM node count, not with how heavy its components + * are. That pass is what these guards defend. + * + * Measured in this Chromium, best of three, on a developer machine: + * + * | insertion | settle | longest task | + * | ------------------ | ------ | ------------ | + * | 10 000 flat | 178 ms | none | + * | 12 000 flat | 199 ms | 51 ms | + * | 2 000 realistic | 181 ms | none | + * | 4 000 realistic | 363 ms | 59 ms | + * | remove 10 000 flat | 40 ms | none | + * | remove 15 000 flat | 61 ms | 53 ms | + * + * So the long-task cliff sits near 12 000 inserted DOM nodes and near 13 000 + * removed ones. The guards below stay well under it rather than asserting the + * cliff itself: a faster machine would never reach it, a slower one would + * reach it early, and an assertion that depends on the runner's speed is a + * flake. The one guard that does not depend on machine speed — a ratio — is + * where the tight threshold goes. + * + * Every threshold is a ceiling on the *best* of several repeats. A CI runner + * is slower and noisier than this machine, and a guard that flakes gets + * deleted by the next person, which is worse than no guard. + */ +import { describe, expect, it } from 'vitest'; +import { whenDOMSettled } from './dom-mutations.js'; +import { + buildPool, + clearDocument, + registerScaleComponents, + scenarios, +} from './mount-at-scale.fixtures.js'; + +registerScaleComponents(); + +/** Repeats per measurement, so one scheduling hiccup cannot fail the build. */ +const REPEATS = 4; + +/** + * A catastrophe ceiling on wall time, not a regression detector: a shared + * runner's wall clock is too soft for that, and the bench-diff job owns it. + * Roughly 8× the local best, past any plausible runner handicap and still + * short of the order-of-magnitude blow-up this is here to catch. + */ +const SETTLE_CEILING = 400; + +interface MountCost { + /** Wall time from the DOM write to `whenDOMSettled()`. */ + duration: number; + /** The longest `longtask` entry the write produced, or 0 for none. */ + longestTask: number; +} + +/** Watch for long tasks while `work` runs, and report the longest one. */ +async function watchLongTasks(work: () => Promise): Promise { + const durations: number[] = []; + const observer = new PerformanceObserver((list) => { + for (const entry of list.getEntries()) { + durations.push(entry.duration); + } + }); + observer.observe({ entryTypes: ['longtask'] }); + + const start = performance.now(); + await work(); + const duration = performance.now() - start; + + // Long-task entries are queued and delivered on a later task. + await new Promise((resolve) => { + setTimeout(resolve, 0); + }); + for (const entry of observer.takeRecords()) { + durations.push(entry.duration); + } + observer.disconnect(); + + return { duration, longestTask: Math.max(0, ...durations) }; +} + +/** + * Insert one tree and wait for the registry to finish with it. The tree is + * built before the clock starts, so parsing markup is never counted as + * framework work. + */ +async function mountOnce(html: string): Promise { + const [host] = buildPool(html, 1); + const cost = await watchLongTasks(async () => { + document.body.append(host); + await whenDOMSettled(); + }); + await clearDocument(); + return cost; +} + +/** The cheapest of `REPEATS` runs, after one unmeasured run warms the paths. */ +async function bestMountCost(html: string): Promise { + await mountOnce(html); + const runs: MountCost[] = []; + for (let index = 0; index < REPEATS; index += 1) { + runs.push(await mountOnce(html)); + } + return { + duration: Math.min(...runs.map((run) => run.duration)), + longestTask: Math.min(...runs.map((run) => run.longestTask)), + }; +} + +describe('mounting at page scale', () => { + // 500 realistic components is a heavy real page. Its un-chunked pass costs + // ~7 ms here, so reaching 50 ms needs a machine seven times slower. + it('mounts 500 realistic components without blocking', async () => { + const cost = await bestMountCost(scenarios.realistic(500)); + expect(cost.longestTask).toBe(0); + expect(cost.duration).toBeLessThan(SETTLE_CEILING); + }, 60000); + + // 2 000 flat components walk about as many nodes, with the same headroom. + it('mounts 2 000 flat components without blocking', async () => { + const cost = await bestMountCost(scenarios.flat(2000)); + expect(cost.longestTask).toBe(0); + expect(cost.duration).toBeLessThan(SETTLE_CEILING); + }, 60000); + + /** + * The failure this really guards against: a change that makes the scan + * re-query the document per element turns mounting quadratic, and a + * quadratic page only reveals itself at scale. A ratio of per-component + * costs does not care how fast the runner is, so it can be tighter than + * any wall-clock ceiling — five times the components must not cost more + * than 2.5× per component, where quadratic would cost five. + * + * Both sides are deliberately in the tens of milliseconds. An earlier + * version compared 500 against 4 000 and read 2.22 on a CI runner while + * reading 1.0 here, because taking the best of several repeats favours the + * shorter side: a 3 ms run dodges an interruption that a 70 ms run + * absorbs. Comparable magnitudes remove that bias. Measured at 1.07 here + * and 0.78 on `ubuntu-latest`, where per-component cost falls with size. + */ + it('costs the same per component at 5 000 as at 1 000', async () => { + const small = await bestMountCost(scenarios.flat(1000)); + const large = await bestMountCost(scenarios.flat(5000)); + expect(large.duration / 5000 / (small.duration / 1000)).toBeLessThan(2.5); + }, 60000); + + /** + * v4 claims nesting costs nothing, because no parent orchestrates its + * children's mounting. Reintroducing that orchestration means a second + * pass that finds and mounts children from each parent, so it would at + * least double the nested side. Measured between 0.82 and 1.15 across + * quiet and fully contended runs here, and below 1.0 on `ubuntu-latest`, + * where a four-deep tree mounts slightly faster than a flat one. + */ + it('mounts a four-deep tree for what a flat one costs', async () => { + const flat = await bestMountCost(scenarios.flat(2000)); + const nested = await bestMountCost(scenarios.nested(2000)); + expect(nested.duration / flat.duration).toBeLessThan(1.8); + }, 60000); +}); diff --git a/packages/v4/vitest.bench.config.js b/packages/v4/vitest.bench.config.js index a1615c080..f2e311977 100644 --- a/packages/v4/vitest.bench.config.js +++ b/packages/v4/vitest.bench.config.js @@ -4,12 +4,27 @@ import { decorators } from './vite-plugin-decorators.js'; /** * Benchmarks answering design questions, run in a real browser because the - * APIs they measure — `matchMedia`, `ResizeObserver`, `AbortSignal` — have - * no meaningful cost in an emulated DOM. Deliberately separate from the - * CodSpeed suite in `packages/tests`, which guards against regressions. + * APIs they measure — `matchMedia`, `ResizeObserver`, `AbortSignal`, the + * mutation observer driving the registry — have no meaningful cost in an + * emulated DOM. + * + * CodSpeed's simulation mode cannot instrument these: its plugin forces + * `pool: "forks"` and measures the Node process, while a browser-mode + * benchmark body runs in Chromium over CDP. Regressions are caught instead by + * `.github/actions/bench-diff`, which runs this suite for the base and the + * head commit on one runner and comments the difference. + * + * `V4_BENCH_SIZES` narrows the at-scale suite — CI compares a subset so the + * job stays inside a few minutes. */ +const sizes = (process.env.V4_BENCH_SIZES ?? '100,1000,5000') + .split(',') + .map((size) => Number(size.trim())) + .filter((size) => Number.isFinite(size) && size > 0); + export default defineConfig({ plugins: [decorators()], + define: { __BENCH_SIZES__: JSON.stringify(sizes) }, test: { include: ['src/**/*.bench.ts'], benchmark: { include: ['src/**/*.bench.ts'] }, From e144747c7a1a8b4dae459b0cc299f29c60a785e5 Mon Sep 17 00:00:00 2001 From: Titouan Mathis Date: Sun, 16 Aug 2026 13:55:05 +0000 Subject: [PATCH 2/6] ci(v4): compare browser benchmarks against the base commit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit v4 had no performance coverage in CI. It cannot join the v3 CodSpeed job: simulation mode forces `pool: "forks"` and instruments the Node process, while a browser-mode benchmark body runs in Chromium over CDP, so it would measure the driver. So this borrows the shape `weareikko/export-size` already proved in this repository — measure head, check the base sha into a subdirectory of the same runner, measure it with the same action-provided script, diff, upsert a sticky comment — and adds what a stopwatch needs that a byte counter does not. Sides alternate within a round and the order flips between rounds, so drift lands on both. Every value is a median of round medians. A benchmark under 5 ms is reported but never flagged, because a sub-millisecond measurement cannot resolve a 25 % move. The threshold is measured, not chosen. Running one commit against itself with three interleaved rounds per side moves benchmarks by 4.5 % median and 12.5 % p90; every case above 13 % is a benchmark under 2 ms, which the floor already excludes. 25 % leaves room for a shared runner. No cached baseline, deliberately. Timed here, `npm ci` is 13.7 s and no build is needed at all — vitest runs the browser suite from TypeScript — against ~30 s per benchmark run, six of them. Caching would save under a tenth of the job and would reintroduce the cross-machine noise the whole design exists to remove, since a stored number cannot be interleaved. The action knows nothing about environments: it takes a directory, a command and an output path, and reads the JSON vitest emits, which is the same schema in Node, in happy-dom and in a browser. Whose benchmarks these are is an input — `id`, `title`, `unit` — so a second suite is a step with its own sticky comment rather than a branch inside the action. It comments and never blocks: a wall-clock gate on a shared runner is a gate that gets deleted. `npm run bench:v4` prints the same table locally. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_011nNdFD3aQhzfdm3EsCSbS9 --- .github/actions/bench-diff/README.md | 83 +++++++++ .github/actions/bench-diff/action.yml | 180 +++++++++++++++++++ .github/actions/bench-diff/bench-comment.mjs | 175 ++++++++++++++++++ .github/actions/bench-diff/bench-report.mjs | Bin 0 -> 3815 bytes .github/workflows/benchmarks-v4.yml | 66 +++++++ .gitignore | 3 + package.json | 1 + packages/v4/package.json | 3 +- 8 files changed, 510 insertions(+), 1 deletion(-) create mode 100644 .github/actions/bench-diff/README.md create mode 100644 .github/actions/bench-diff/action.yml create mode 100644 .github/actions/bench-diff/bench-comment.mjs create mode 100644 .github/actions/bench-diff/bench-report.mjs create mode 100644 .github/workflows/benchmarks-v4.yml diff --git a/.github/actions/bench-diff/README.md b/.github/actions/bench-diff/README.md new file mode 100644 index 000000000..ecbbf353a --- /dev/null +++ b/.github/actions/bench-diff/README.md @@ -0,0 +1,83 @@ +# bench-diff + +Run a [vitest](https://vitest.dev) benchmark suite for the base and the head commit **on one runner**, alternating between them, and upsert a sticky pull-request comment with the difference. + +It is the timing counterpart of [`weareikko/export-size`](https://github.com/weareikko/export-size): same shape — measure head, check the base sha out into a subdirectory of the same runner, measure it with the same action-provided script, diff, comment — with the changes a stopwatch needs that a byte counter does not. + +## Why it works this way + +**Both sides on one runner.** Cross-machine timing noise is the reason services like CodSpeed exist. Measuring the base on the runner that measures the head removes it, with no account, no token and nothing to store. A cached baseline from an earlier `main` run would put that noise straight back, and would be un-interleavable by construction. + +**Alternating, not sequential.** Export-size measures head then base. For byte counts that is fine. For timings it puts every bit of thermal drift and every noisy neighbour on one side of the comparison. This action alternates the sides within a round and flips the order between rounds, so drift lands on both. + +**Median, never mean.** A benchmark's value for a round is tinybench's median; its value for the report is the median across rounds. One GC pause should not become the headline. + +**A measured threshold, and a resolution floor.** Run one commit against itself, read the spread, and set `threshold` from it. Benchmarks whose median falls under `floor` milliseconds are reported but never flagged: Chromium clamps `performance.now()` to 100 µs, so a 1 ms benchmark cannot resolve a percentage. + +**It comments; it does not block.** A timing threshold that fails a build on a shared runner is a threshold that gets deleted. Hard gates belong in the test suite, as assertions that do not depend on how fast the runner is. + +**It knows nothing about environments.** It takes a directory, a command and an output path, and reads the JSON vitest emits — the same schema whether the benchmark bodies run in Node, in happy-dom or in a real browser over CDP. A second suite is another step with another `id`, not a branch inside the action. Keep it that way. + +## Usage + +A browser suite, which needs a browser downloaded first: + +```yaml +- uses: actions/checkout@v4 +- uses: ./.github/actions/bench-diff + with: + id: v4-mount + title: v4 mount benchmarks + unit: component + working-directory: packages/v4 + prepare: npx playwright install --with-deps chromium + bench: npm exec vitest bench -- --config vitest.bench.config.js --run --outputJson "$BENCH_JSON" + rounds: '3' + threshold: '25' + floor: '5' +``` + +A Node suite, which needs nothing extra — the only difference is the command: + +```yaml +- uses: ./.github/actions/bench-diff + with: + id: v3 + title: v3 benchmarks + working-directory: packages/js-toolkit + bench: npm exec vitest bench -- --config vitest.bench.config.ts --run --outputJson "$BENCH_JSON" +``` + +`bench` must write a `vitest bench --outputJson` file to `$BENCH_JSON`. Anything the suite needs from the environment can be set as `env:` on the step — composite `run` steps inherit it — or written as a prefix assignment on the command itself, which is unambiguous. + +`id` namespaces the sticky comment and the temporary files, so several suites can each keep their own comment on one pull request. Two suites in the _same job_ would share the base checkout; prefer a job or a workflow each, which also lets each one carry its own `paths:` filter. + +The job needs `pull-requests: write` to comment. + +### Inputs + +| input | default | meaning | +| ------------------- | ----------------------------- | ---------------------------------------------------------- | +| `id` | `bench-diff` | Suite identifier; namespaces the comment and temp files. | +| `title` | `Benchmarks` | Heading of the sticky comment. | +| `unit` | `unit` | What a group's count counts, for the per-unit column. | +| `bench` | — | Command writing a bench JSON to `$BENCH_JSON`. | +| `working-directory` | `.` | Where to run it, inside each checkout. | +| `install` | `npm ci --no-audit --no-fund` | Dependency install, at each checkout root. | +| `prepare` | — | Anything else each checkout needs before benchmarking. | +| `rounds` | `3` | Sampling rounds per side. | +| `threshold` | `25` | Percent change reported as a change. | +| `floor` | `5` | Benchmarks under this many ms are reported, never flagged. | +| `comment` | `true` | Upsert the sticky comment. | + +## Reading a group as a per-unit cost + +`bench-report.mjs` divides a benchmark's median by the first integer in its group title. A group named `mount 5000 components, one insertion` therefore reports microseconds per component alongside milliseconds per operation, which is what makes a non-linear curve legible as a number rather than as a shape. Name the unit with `unit:`. A group whose title carries no number simply has no per-unit figure, and the column reads `-`. + +## Locally + +```sh +node .github/actions/bench-diff/bench-report.mjs # one run, as a table +node .github/actions/bench-diff/bench-report.mjs a.json b.json --json out.json # several rounds, aggregated +node .github/actions/bench-diff/bench-comment.mjs base.json head.json # the comment body +``` diff --git a/.github/actions/bench-diff/action.yml b/.github/actions/bench-diff/action.yml new file mode 100644 index 000000000..e714b0ac7 --- /dev/null +++ b/.github/actions/bench-diff/action.yml @@ -0,0 +1,180 @@ +name: 'Benchmark diff' +description: 'Run a vitest benchmark suite for the base and the head commit on one runner, alternating, and comment the difference.' +author: 'Studio Meta' +branding: + icon: 'activity' + color: 'blue' + +# The action knows nothing about how a suite runs. It takes a directory, a +# command and an output path, and reads the JSON vitest emits — the same +# schema whether the benchmark bodies execute in Node or in a browser. Keep it +# that way: a second suite is added by adding a step, not by adding a branch. + +inputs: + id: + description: 'Identifier for this suite. Namespaces the sticky comment and the temporary files, so several suites can each keep their own comment on one pull request.' + required: false + default: 'bench-diff' + title: + description: 'Heading of the sticky comment.' + required: false + default: 'Benchmarks' + unit: + description: "What a group's count counts, for the per-unit column — `component`, `element`, `node`. Groups whose title carries no number have no per-unit figure." + required: false + default: 'unit' + bench: + description: 'Command producing a `vitest bench --outputJson` file at `$BENCH_JSON`, run in `working-directory` inside each checkout.' + required: true + working-directory: + description: 'Directory to run `bench` in, relative to a checkout root.' + required: false + default: '.' + install: + description: 'Command that installs dependencies, run at the root of each checkout.' + required: false + default: 'npm ci --no-audit --no-fund' + prepare: + description: 'Optional command run after `install` in each checkout, for anything the benchmarks need on disk.' + required: false + default: '' + rounds: + description: 'Sampling rounds per side. Sides alternate within a round, and the order flips between rounds, so thermal drift and runner contention land on both sides equally.' + required: false + default: '3' + threshold: + description: 'Percent change reported as a change. Set it from a measured same-commit noise floor, never from taste.' + required: false + default: '25' + floor: + description: 'Benchmarks with a median under this many milliseconds are reported but never flagged: the timer cannot resolve them.' + required: false + default: '5' + node-version: + description: 'Node.js version.' + required: false + default: '24' + comment: + description: 'Upsert a sticky pull-request comment with the diff.' + required: false + default: 'true' + github-token: + description: 'Token used to upsert the comment (needs `pull-requests: write`).' + required: false + default: ${{ github.token }} + +runs: + using: composite + steps: + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: ${{ inputs.node-version }} + cache: npm + + - name: Install the pull request + shell: bash + run: | + ${{ inputs.install }} + ${{ inputs.prepare }} + + # The base is measured on this runner, from the same action-provided + # scripts, rather than read from a cache. A cached baseline comes from + # another machine under other contention, which re-imports exactly the + # cross-machine noise this design exists to remove — and a cached number + # cannot be interleaved with anything. + - name: Check out the base branch + if: github.event_name == 'pull_request' + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.base.sha }} + path: __bench-diff-base + + - name: Install the base branch + if: github.event_name == 'pull_request' + shell: bash + working-directory: __bench-diff-base + run: | + ${{ inputs.install }} || true + ${{ inputs.prepare }} || true + + - name: Run both sides, alternating + shell: bash + env: + BENCH_ID: ${{ inputs.id }} + BENCH_ROUNDS: ${{ inputs.rounds }} + BENCH_WORKDIR: ${{ inputs.working-directory }} + BENCH_HAS_BASE: ${{ github.event_name == 'pull_request' }} + run: | + set -u + run_side() { + local side="$1" round="$2" root="$3" + local out="$RUNNER_TEMP/$BENCH_ID-$side-$round.json" + # `export` on its own line, not a prefix assignment: `$BENCH_JSON` + # inside the command is expanded by this shell before the command + # runs, so a prefix assignment would come too late for it. + ( cd "$root/$BENCH_WORKDIR" && export BENCH_JSON="$out" && ${{ inputs.bench }} ) \ + || echo "::warning::the $side benchmark run failed in round $round" + } + for round in $(seq 1 "$BENCH_ROUNDS"); do + if [ "$BENCH_HAS_BASE" != 'true' ]; then + run_side head "$round" "$GITHUB_WORKSPACE" + elif [ $((round % 2)) -eq 1 ]; then + run_side head "$round" "$GITHUB_WORKSPACE" + run_side base "$round" "$GITHUB_WORKSPACE/__bench-diff-base" + else + run_side base "$round" "$GITHUB_WORKSPACE/__bench-diff-base" + run_side head "$round" "$GITHUB_WORKSPACE" + fi + done + + - name: Aggregate the rounds + shell: bash + env: + BENCH_ID: ${{ inputs.id }} + run: | + set -u + shopt -s nullglob + head_rounds=("$RUNNER_TEMP/$BENCH_ID"-head-*.json) + base_rounds=("$RUNNER_TEMP/$BENCH_ID"-base-*.json) + # A missing base degrades to "everything is new"; a missing head means + # the action itself is broken, and must not pass quietly. + if [ ${#head_rounds[@]} -eq 0 ]; then + echo '::error::the benchmark command produced no output for the head commit' >&2 + exit 1 + fi + node "$GITHUB_ACTION_PATH/bench-report.mjs" "${head_rounds[@]}" \ + --json "$RUNNER_TEMP/$BENCH_ID-head.json" --markdown + if [ ${#base_rounds[@]} -eq 0 ]; then + echo '[]' > "$RUNNER_TEMP/$BENCH_ID-base.json" + else + node "$GITHUB_ACTION_PATH/bench-report.mjs" "${base_rounds[@]}" \ + --json "$RUNNER_TEMP/$BENCH_ID-base.json" + fi + + - name: Comment on the pull request + if: inputs.comment == 'true' && github.event_name == 'pull_request' + shell: bash + env: + GH_TOKEN: ${{ inputs.github-token }} + BENCH_ID: ${{ inputs.id }} + run: | + node "$GITHUB_ACTION_PATH/bench-comment.mjs" \ + "$RUNNER_TEMP/$BENCH_ID-base.json" "$RUNNER_TEMP/$BENCH_ID-head.json" \ + --threshold '${{ inputs.threshold }}' \ + --floor '${{ inputs.floor }}' \ + --rounds '${{ inputs.rounds }}' \ + --id '${{ inputs.id }}' \ + --title '${{ inputs.title }}' \ + --unit '${{ inputs.unit }}' > "$RUNNER_TEMP/$BENCH_ID-comment.md" + cat "$RUNNER_TEMP/$BENCH_ID-comment.md" >> "$GITHUB_STEP_SUMMARY" + marker="" + repo='${{ github.repository }}' + pr='${{ github.event.pull_request.number }}' + comment_id=$(gh api "repos/$repo/issues/$pr/comments" --paginate \ + --jq ".[] | select(.body | contains(\"$marker\")) | .id" | head -n1) + if [ -n "$comment_id" ]; then + gh api --method PATCH "repos/$repo/issues/comments/$comment_id" -F body=@"$RUNNER_TEMP/$BENCH_ID-comment.md" >/dev/null + else + gh api --method POST "repos/$repo/issues/$pr/comments" -F body=@"$RUNNER_TEMP/$BENCH_ID-comment.md" >/dev/null + fi diff --git a/.github/actions/bench-diff/bench-comment.mjs b/.github/actions/bench-diff/bench-comment.mjs new file mode 100644 index 000000000..9a4a02b57 --- /dev/null +++ b/.github/actions/bench-diff/bench-comment.mjs @@ -0,0 +1,175 @@ +// Render a pull-request comment diffing two `bench-report.mjs --json` reports +// (base vs head) and print it to stdout. The workflow upserts the output as a +// sticky comment, matched by the marker below. +// +// Timings are not byte counts. Two things follow, and both are visible in the +// comment itself so nobody reads a number as more precise than it is: +// +// * A change under `--threshold` percent is not reported as a change. The +// default is set from a measured same-commit noise floor, not from taste. +// * A benchmark whose median is under `--floor` milliseconds is never +// flagged at all. A sub-millisecond benchmark cannot resolve a 25 % move: +// Chromium clamps `performance.now()` to 100 us, and a Node suite hits +// its own resolution and scheduling limits at a similar scale. +import { readFileSync } from 'node:fs'; + +const args = process.argv.slice(2); +const files = []; +let threshold = 25; +let floor = 5; +let rounds = 0; +// The suite's identity, so several suites can each keep their own sticky +// comment in one repository. Nothing here knows whether a suite runs in Node +// or in a browser: it reads the JSON vitest emits either way. +let id = 'bench-diff'; +let title = 'Benchmarks'; +let unit = 'unit'; + +const flags = { + '--threshold': (value) => { + threshold = Number(value); + }, + '--floor': (value) => { + floor = Number(value); + }, + '--rounds': (value) => { + rounds = Number(value); + }, + '--id': (value) => { + id = value; + }, + '--title': (value) => { + title = value; + }, + '--unit': (value) => { + unit = value; + }, +}; + +for (let index = 0; index < args.length; index += 1) { + const flag = flags[args[index]]; + if (flag) { + index += 1; + flag(args[index]); + } else { + files.push(args[index]); + } +} + +const MARKER = ''; + +const [basePath, headPath] = files; +if (!basePath || !headPath) { + console.error( + 'Usage: node bench-comment.mjs [--threshold 25] [--floor 5] [--rounds 3] [--id name] [--title text] [--unit component]', + ); + process.exit(1); +} + +function read(path) { + try { + return JSON.parse(readFileSync(path, 'utf8')); + } catch { + return []; + } +} + +function index(entries) { + const map = new Map(); + for (const entry of entries) { + map.set(entry.group + ' ' + entry.name, entry); + } + return map; +} + +const base = index(read(basePath)); +const head = index(read(headPath)); + +function ms(value) { + return value === undefined ? '-' : value.toFixed(value < 10 ? 2 : 1) + ' ms'; +} + +function per(entry) { + return entry?.perUnit === undefined ? '-' : entry.perUnit.toFixed(2); +} + +const keys = [...new Set([...base.keys(), ...head.keys()])].sort(); +const changed = []; +const steady = []; + +for (const key of keys) { + const b = base.get(key); + const h = head.get(key); + const entry = h ?? b; + const delta = b && h ? ((h.median - b.median) / b.median) * 100 : undefined; + // Below the resolution floor, or too small a move to distinguish from the + // runner: report the value, but never as a change. + const resolvable = b && h && Math.max(b.median, h.median) >= floor; + const significant = resolvable && Math.abs(delta) >= threshold; + const row = + '| ' + + entry.group + + ' | ' + + entry.name + + ' | ' + + ms(b?.median) + + ' | ' + + ms(h?.median) + + ' | ' + + per(h ?? b) + + ' | ' + + (delta === undefined ? 'new' : (delta > 0 ? '+' : '') + delta.toFixed(1) + '%') + + ' |'; + if (significant) { + changed.push({ delta, row }); + } else { + steady.push(row); + } +} + +changed.sort((a, z) => z.delta - a.delta); + +const HEADER = [ + '| Group | Benchmark | Base | Head | us / ' + unit + ' | Change |', + '| :-- | :-- | --: | --: | --: | --: |', +]; + +const body = [MARKER, '## ' + title, '']; +body.push( + 'Base and head measured on this runner, alternating' + + (rounds ? ' over ' + rounds + ' rounds each' : '') + + '; every value is the median of the round medians. Running both sides on one machine is what removes cross-machine noise — a cached baseline from another runner would put it back.', + '', + 'A move under **' + + threshold + + '%**, or on a benchmark under **' + + floor + + ' ms**, is not reported as a change: it is inside the measured noise of a shared runner.', + '', +); + +if (base.size === 0) { + // The base could not be installed or benchmarked — or, on the pull request + // that adds them, has no such benchmarks yet. Degrade to "everything is + // new" rather than failing the job, as `export-size` does, and show every + // row: none of them is hidden noise, they are all the whole story. + body.push('No base measurement: every benchmark below is new.', '', ...HEADER, ...steady, ''); +} else { + if (changed.length === 0) { + body.push('No benchmark moved beyond the noise floor.', ''); + } else { + body.push(...HEADER, ...changed.map((entry) => entry.row), ''); + } + if (steady.length > 0) { + body.push( + '
Within noise (' + steady.length + ')', + '', + ...HEADER, + ...steady, + '', + '
', + ); + } +} + +console.log(body.join('\n')); diff --git a/.github/actions/bench-diff/bench-report.mjs b/.github/actions/bench-diff/bench-report.mjs new file mode 100644 index 0000000000000000000000000000000000000000..8dadf96fec414ca12450b5fa033b2692199825f6 GIT binary patch literal 3815 zcmai1+iu%N5bd+RVhXe%6-$)uqClZUPTQa#fK zl<62j4Ok*~=YBaen$75}3AL%*7fX%Df1tVvmM(5wYhz27)|Xc_oi<&&?b^R%<1c9K zDjUi9wjo2&`AubMU72a&j%1NVN&qtLD}hpl6mp z3^X_o@#LZy5t`1g3i?g8Gkq56##9~jz(7Mnz|^ht_hMoeSyXLT&Un@C+96MSWs_~f zhBu`pQ-&r+l^Ev`Si>O^W)OyWZEx+0F7Js2G-YX`xI}-xqpj(p1T1vk+dx;YEW7Q5 zB8R26#?x)nRV%iu4W_&n5w}f1RHLc42wE_Z8gLF;mfzlQU1=(gK_8rdquJOw%OLX2 z%4$fa8s?(@oGhOn6`zqCI+w^ykM*r6iY1*-r*ef1%Bh_6IarQd%?YCiM8~Y&BeNgx zeL11qfc*Ebw4-%sYRcGx`8sB=N29XwaKwa7qy^I3ls3i!uQx@kTxs*;@#{n*sb#A3 zp9wYgxSqdwKl<1hk=pM}zkN>&}u%G&#QrY{dlITpd6ldm)L!()Zs8Vzyst6t_^RnQ>JU0`%9Mj?iZ8yc! z6Ho0{E40}HEy>KivXc|dbkg?x@<%!rYmVq`VmntcOPW)zwCM!ej+S)r71@|(bga>v z(Ldj`XWIbFxD}wS%hvCX$y?m^RTc>8y%pec2(tuL=7kn3qt<`&49`g z52giVUjy+Cs?AR;=dFRjF5FlaR@sfaM3jN-AkfV1?Z3T(73vg~CIkmDzDd!Dz#hzo zD{w+EAa7qY5##8?I+qwcRX^L&!m0($K3(RYRtMuxm$Sm&*)mab=2KFY@@*=mTwc$Q zmSZFlv&F9A#Hvry$2PdF05DHBfEljuC=EbvRdg(82nlS~WRbP|L3##ei@j$^S-$S7 ziXVo?N6T&00^9&~>At8;Z2w~Xfx8)&I#{{@f;t%Mjef!e8 zrmAcS%Q4*3$m4p@3Cc*T{>Ttz;KK0KiKyo~mpLv4ceVDSFf*cidrPNgn?s#|wq7d_ zLZw(XjKM=%9(Wi46c_aO#}8)(;u38>96$<#6Uw@F{Yy5Mo*rnBJV60}Crlw(4RWRz+v7JMmpS`3_)-I(*d zUkJYFLZH20R1MZ@`wnvxOMU)DPsjB5NJiT21p()XQAbv?w67|3JX?(ObTv^Nix?z+ z^m?@{#7#DUS7XI-TQLJ~P>Z2bd>a#dxQt_PD6pQ+`n0+Ozw2jG_Jb zMBcJ#hC%d` .measure-*.json + +# Benchmark output written by `npm run bench:v4` +.bench-report.json /packages/docs/.vitepress/cache /packages/docs/.vitepress/dist coverage/ diff --git a/package.json b/package.json index f43063a19..27e60de19 100644 --- a/package.json +++ b/package.json @@ -25,6 +25,7 @@ "test:eslint-plugin": "npm run test -w @studiometa/eslint-plugin-js-toolkit", "bench": "npm run bench -w @studiometa/js-toolkit", "bench:run": "npm run bench:run -w @studiometa/js-toolkit", + "bench:v4": "npm run bench:report -w @studiometa/js-toolkit-v4", "lint": "npm run subpaths:check && npm run lint:static && npm run lint:fmt && npm run lint:types", "lint:static": "oxlint .", "lint:fmt": "oxfmt --check .", diff --git a/packages/v4/package.json b/packages/v4/package.json index ba3c7973f..977781667 100644 --- a/packages/v4/package.json +++ b/packages/v4/package.json @@ -6,7 +6,7 @@ "sideEffects": [ "./dist/responsive-options.js" ], - "description": "v4 prototype — tested in a real browser through Vitest browser mode", + "description": "v4 prototype \u2014 tested in a real browser through Vitest browser mode", "files": [ "dist" ], @@ -767,6 +767,7 @@ "test:watch": "vitest", "test:debug": "vitest --browser.headless=false", "bench": "vitest bench --config vitest.bench.config.js", + "bench:report": "npm run bench -- --run --outputJson .bench-report.json && node ../../.github/actions/bench-diff/bench-report.mjs .bench-report.json", "lint:types": "tsc -p tsconfig.json" }, "dependencies": { From 4d99be0b1600979aa923fbe19d84d0aba62eed29 Mon Sep 17 00:00:00 2001 From: Titouan Mathis Date: Sun, 16 Aug 2026 14:16:59 +0000 Subject: [PATCH 3/6] fix(v4): reduce the long-task repeats conservatively MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `mountCost()` took the minimum of every signal, which is right for a cost and backwards for a defect: one lucky repeat out of four made `expect(longestTask).toBe(0)` pass on a page that blocked the main thread in the other three. The maximum is wrong the other way. A `longtask` entry attributes any 50 ms task in the frame, mounting's or not, so a single GC pause would fail the build — and a guard that flakes gets deleted. Measured across the cliff, four repeats each, the two failure modes are visible side by side: | flat components | repeats (ms) | min | second worst | max | | --------------- | ------------- | --- | ------------ | --- | | 2 000 (guarded) | 0, 0, 0, 0 | 0 | 0 | 0 | | 9 000 | 0, 0, 0, 67 | 0 | 0 | 67 | | 11 000 | 0, 0, 51, 58 | 0 | 51 | 58 | | 12 000 | 50, 53, 60, 61| 50 | 60 | 61 | At 11 000 the minimum passes a page that blocks half the time. At 9 000 the maximum fails on one sample out of four. The second worst is the only reduction that separates them, so that is what a long task uses; wall time keeps the minimum, where the least interrupted run really is the best estimate of the work. Noise is absorbed in how many repeats blocked, never by allowing some blocking, so the assertion still reads "no long task". At the guarded sizes it costs nothing: 240 repeats across ten runs, quiet and with all eight cores saturated, every one of them clean. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_011nNdFD3aQhzfdm3EsCSbS9 --- packages/v4/src/mount-at-scale.spec.ts | 56 ++++++++++++++++++++------ 1 file changed, 43 insertions(+), 13 deletions(-) diff --git a/packages/v4/src/mount-at-scale.spec.ts b/packages/v4/src/mount-at-scale.spec.ts index 3ebd88ae4..77abfa780 100644 --- a/packages/v4/src/mount-at-scale.spec.ts +++ b/packages/v4/src/mount-at-scale.spec.ts @@ -29,9 +29,11 @@ * flake. The one guard that does not depend on machine speed — a ratio — is * where the tight threshold goes. * - * Every threshold is a ceiling on the *best* of several repeats. A CI runner - * is slower and noisier than this machine, and a guard that flakes gets - * deleted by the next person, which is worse than no guard. + * Every threshold tolerates noise, because a CI runner is slower and noisier + * than this machine and a guard that flakes gets deleted by the next person, + * which is worse than no guard. Each signal tolerates it in the way that + * suits it: wall time takes the best of several repeats, a long task takes + * the second worst. `mountCost()` explains why they differ. */ import { describe, expect, it } from 'vitest'; import { whenDOMSettled } from './dom-mutations.js'; @@ -44,7 +46,11 @@ import { registerScaleComponents(); -/** Repeats per measurement, so one scheduling hiccup cannot fail the build. */ +/** + * Repeats per measurement, so one scheduling hiccup cannot fail the build. + * Must stay at two or more: the long-task reduction below discards the worst + * repeat and reads the one behind it. + */ const REPEATS = 4; /** @@ -103,16 +109,40 @@ async function mountOnce(html: string): Promise { return cost; } -/** The cheapest of `REPEATS` runs, after one unmeasured run warms the paths. */ -async function bestMountCost(html: string): Promise { +/** + * Run the measurement `REPEATS` times, after one unmeasured run warms the + * paths, and reduce the repeats — differently for each signal, because they + * are different kinds of number. + * + * **Wall time takes the minimum.** It is a cost, and the least interrupted + * run is the closest any of them gets to the cost of the work itself. Every + * other repeat measured that same work plus something else. + * + * **A long task takes the second worst.** It is a defect, not a cost: any + * occurrence is the finding, so the minimum is exactly wrong — one lucky + * repeat out of four would hide a page that blocks the main thread three + * times out of four. The maximum is wrong in the other direction, because a + * `longtask` entry attributes *any* 50 ms task in the frame, including a GC + * pause or a runner hiccup that has nothing to do with mounting, and one of + * those would fail the build. Discarding only the single worst repeat keeps + * the signal — a page that really blocks does so on every repeat — while + * forgiving one bad sample. One blocked run is noise; two is the finding. + * + * Tolerating noise in *how many repeats blocked*, rather than by allowing + * some blocking, is what keeps the assertion honest: it still reads "no long + * task", never "no long task longer than N". + */ +async function mountCost(html: string): Promise { await mountOnce(html); const runs: MountCost[] = []; for (let index = 0; index < REPEATS; index += 1) { runs.push(await mountOnce(html)); } + const blocked = runs.map((run) => run.longestTask).sort((a, b) => a - b); return { duration: Math.min(...runs.map((run) => run.duration)), - longestTask: Math.min(...runs.map((run) => run.longestTask)), + // The second worst, so the single worst repeat is discarded. + longestTask: blocked[blocked.length - 2], }; } @@ -120,14 +150,14 @@ describe('mounting at page scale', () => { // 500 realistic components is a heavy real page. Its un-chunked pass costs // ~7 ms here, so reaching 50 ms needs a machine seven times slower. it('mounts 500 realistic components without blocking', async () => { - const cost = await bestMountCost(scenarios.realistic(500)); + const cost = await mountCost(scenarios.realistic(500)); expect(cost.longestTask).toBe(0); expect(cost.duration).toBeLessThan(SETTLE_CEILING); }, 60000); // 2 000 flat components walk about as many nodes, with the same headroom. it('mounts 2 000 flat components without blocking', async () => { - const cost = await bestMountCost(scenarios.flat(2000)); + const cost = await mountCost(scenarios.flat(2000)); expect(cost.longestTask).toBe(0); expect(cost.duration).toBeLessThan(SETTLE_CEILING); }, 60000); @@ -148,8 +178,8 @@ describe('mounting at page scale', () => { * and 0.78 on `ubuntu-latest`, where per-component cost falls with size. */ it('costs the same per component at 5 000 as at 1 000', async () => { - const small = await bestMountCost(scenarios.flat(1000)); - const large = await bestMountCost(scenarios.flat(5000)); + const small = await mountCost(scenarios.flat(1000)); + const large = await mountCost(scenarios.flat(5000)); expect(large.duration / 5000 / (small.duration / 1000)).toBeLessThan(2.5); }, 60000); @@ -162,8 +192,8 @@ describe('mounting at page scale', () => { * where a four-deep tree mounts slightly faster than a flat one. */ it('mounts a four-deep tree for what a flat one costs', async () => { - const flat = await bestMountCost(scenarios.flat(2000)); - const nested = await bestMountCost(scenarios.nested(2000)); + const flat = await mountCost(scenarios.flat(2000)); + const nested = await mountCost(scenarios.nested(2000)); expect(nested.duration / flat.duration).toBeLessThan(1.8); }, 60000); }); From 2ee80d6cc65fe078499499f169420ecfaaf0a8b2 Mon Sep 17 00:00:00 2001 From: Titouan Mathis Date: Sun, 16 Aug 2026 14:16:59 +0000 Subject: [PATCH 4/6] fix(ci): fail a broken baseline instead of inventing an empty one MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `${{ inputs.install }} || true` and the same on `prepare` came from `weareikko/export-size`, where degrading is harmless: a missing byte count is obviously missing. A missing *baseline* is not obviously missing. It becomes "everything is new", which reads as a successful comparison that never happened. Absent and broken are now different states, decided by counting output rather than by the action knowing anything about the command it was given: | side | every round measured | none measured | some measured | | ---- | -------------------- | ---------------------------- | ------------- | | head | compare | fail | fail | | base | compare | empty base, warn, comment says so | fail | A base with no benchmark suite yet is real — it is the state of every pull request that adds one, including this one — so it still degrades, and the comment now says that a base which failed to build or measure would have failed the job rather than appearing there. A base that measured some rounds and not others is a broken suite, not an absent one, and fails. Base `install` and `prepare` are no longer suppressed at all. All six combinations exercised against the step's own shell. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_011nNdFD3aQhzfdm3EsCSbS9 --- .github/actions/bench-diff/README.md | 9 ++++- .github/actions/bench-diff/action.yml | 40 ++++++++++++++++---- .github/actions/bench-diff/bench-comment.mjs | 17 ++++++--- 3 files changed, 53 insertions(+), 13 deletions(-) diff --git a/.github/actions/bench-diff/README.md b/.github/actions/bench-diff/README.md index ecbbf353a..67aca6f65 100644 --- a/.github/actions/bench-diff/README.md +++ b/.github/actions/bench-diff/README.md @@ -14,7 +14,14 @@ It is the timing counterpart of [`weareikko/export-size`](https://github.com/wea **A measured threshold, and a resolution floor.** Run one commit against itself, read the spread, and set `threshold` from it. Benchmarks whose median falls under `floor` milliseconds are reported but never flagged: Chromium clamps `performance.now()` to 100 µs, so a 1 ms benchmark cannot resolve a percentage. -**It comments; it does not block.** A timing threshold that fails a build on a shared runner is a threshold that gets deleted. Hard gates belong in the test suite, as assertions that do not depend on how fast the runner is. +**It comments; it does not block — but it fails when it could not measure.** A timing _threshold_ that fails a build on a shared runner is a threshold that gets deleted, so regressions are reported rather than enforced. A broken _measurement_ is the opposite: the job fails, because a comparison that silently did not happen is worse than no comparison. The one state that degrades is a base with no benchmark suite yet, which is a real thing on the pull request that adds one. Absent is not broken: + +| side | every round measured | none measured | some measured | +| ---- | -------------------- | ----------------------------------------- | ------------- | +| head | compare | fail | fail | +| base | compare | empty base, warn, and the comment says so | fail | + +Base `install` and `prepare` are never suppressed either. That distinction is the whole reason counting output is worth doing. **It knows nothing about environments.** It takes a directory, a command and an output path, and reads the JSON vitest emits — the same schema whether the benchmark bodies run in Node, in happy-dom or in a real browser over CDP. A second suite is another step with another `id`, not a branch inside the action. Keep it that way. diff --git a/.github/actions/bench-diff/action.yml b/.github/actions/bench-diff/action.yml index e714b0ac7..de7170e74 100644 --- a/.github/actions/bench-diff/action.yml +++ b/.github/actions/bench-diff/action.yml @@ -90,13 +90,18 @@ runs: ref: ${{ github.event.pull_request.base.sha }} path: __bench-diff-base + # No `|| true` here. A base that cannot install or prepare is a broken + # job, and suppressing it would turn a broken baseline into a confident + # "everything is new" comment — a green report of a comparison that never + # happened. Only a base with no benchmark suite may degrade, and that is + # decided further down, from whether it produced any output at all. - name: Install the base branch if: github.event_name == 'pull_request' shell: bash working-directory: __bench-diff-base run: | - ${{ inputs.install }} || true - ${{ inputs.prepare }} || true + ${{ inputs.install }} + ${{ inputs.prepare }} - name: Run both sides, alternating shell: bash @@ -128,25 +133,46 @@ runs: fi done + # Three outcomes per side, and they are not the same thing. + # + # every round measured -> compare + # no round measured -> only the base may do this, and only because + # "the suite does not exist on this commit yet" + # is a real, expected state on the pull request + # that adds it. Degrade to an empty base, and + # say so in the comment. + # some rounds measured -> a suite that ran and then did not is broken, + # not absent. Never degrade; fail. + # + # Counting output is what separates absent from broken without the action + # having to know anything about the command it was given. - name: Aggregate the rounds shell: bash env: BENCH_ID: ${{ inputs.id }} + BENCH_ROUNDS: ${{ inputs.rounds }} + BENCH_HAS_BASE: ${{ github.event_name == 'pull_request' }} run: | set -u shopt -s nullglob head_rounds=("$RUNNER_TEMP/$BENCH_ID"-head-*.json) base_rounds=("$RUNNER_TEMP/$BENCH_ID"-base-*.json) - # A missing base degrades to "everything is new"; a missing head means - # the action itself is broken, and must not pass quietly. - if [ ${#head_rounds[@]} -eq 0 ]; then - echo '::error::the benchmark command produced no output for the head commit' >&2 + + if [ ${#head_rounds[@]} -ne "$BENCH_ROUNDS" ]; then + echo "::error::the head commit measured ${#head_rounds[@]} of $BENCH_ROUNDS rounds" >&2 exit 1 fi node "$GITHUB_ACTION_PATH/bench-report.mjs" "${head_rounds[@]}" \ --json "$RUNNER_TEMP/$BENCH_ID-head.json" --markdown - if [ ${#base_rounds[@]} -eq 0 ]; then + + if [ "$BENCH_HAS_BASE" != 'true' ] || [ ${#base_rounds[@]} -eq 0 ]; then + if [ "$BENCH_HAS_BASE" = 'true' ]; then + echo '::warning::the base commit produced no benchmark output; reporting every benchmark as new' >&2 + fi echo '[]' > "$RUNNER_TEMP/$BENCH_ID-base.json" + elif [ ${#base_rounds[@]} -ne "$BENCH_ROUNDS" ]; then + echo "::error::the base commit measured ${#base_rounds[@]} of $BENCH_ROUNDS rounds, so its suite is broken rather than absent" >&2 + exit 1 else node "$GITHUB_ACTION_PATH/bench-report.mjs" "${base_rounds[@]}" \ --json "$RUNNER_TEMP/$BENCH_ID-base.json" diff --git a/.github/actions/bench-diff/bench-comment.mjs b/.github/actions/bench-diff/bench-comment.mjs index 9a4a02b57..f58f148ab 100644 --- a/.github/actions/bench-diff/bench-comment.mjs +++ b/.github/actions/bench-diff/bench-comment.mjs @@ -149,11 +149,18 @@ body.push( ); if (base.size === 0) { - // The base could not be installed or benchmarked — or, on the pull request - // that adds them, has no such benchmarks yet. Degrade to "everything is - // new" rather than failing the job, as `export-size` does, and show every - // row: none of them is hidden noise, they are all the whole story. - body.push('No base measurement: every benchmark below is new.', '', ...HEADER, ...steady, ''); + // The base has no benchmark suite — the expected state on the pull request + // that adds one. A base that failed to install, prepare, or measure only + // some of its rounds never reaches here: the action fails the job instead, + // so an empty base always means absent and never means broken. Show every + // row, since none of them is hidden noise; they are the whole story. + body.push( + 'The base commit has no benchmark suite, so every benchmark below is new. A base that failed to build or measure would have failed this job rather than appearing here.', + '', + ...HEADER, + ...steady, + '', + ); } else { if (changed.length === 0) { body.push('No benchmark moved beyond the noise floor.', ''); From ebb3d4845088b9d75fc0eba2899c9c8929ea0d06 Mon Sep 17 00:00:00 2001 From: Titouan Mathis Date: Sun, 16 Aug 2026 14:20:48 +0000 Subject: [PATCH 5/6] fix(ci): fail on a report that cannot be read MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `bench-comment.mjs` caught every read and parse error and returned `[]`. That fallback existed for the absent base, but an absent base is already expressed as a report which *is* `[]`, written deliberately by the aggregation step. So the catch only ever hid the other case: a report that could not be read or parsed became an empty base, and the comment announced a comparison that never happened — the same silent degradation just removed from the base install. Let it throw. A malformed or missing report now fails the step, while a base that legitimately measured nothing still comes through as `[]` and still comments. The shell flags are spelled out too. `shell: bash` already runs with `-e -o pipefail` — confirmed in the runner log, not just the docs — so a failing `bench-report.mjs` did abort the step, but a script that says only `set -u` invites the reader to conclude otherwise. It now says `set -euo pipefail`. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_011nNdFD3aQhzfdm3EsCSbS9 --- .github/actions/bench-diff/action.yml | 6 ++++-- .github/actions/bench-diff/bench-comment.mjs | 11 ++++++----- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/.github/actions/bench-diff/action.yml b/.github/actions/bench-diff/action.yml index de7170e74..fac35a787 100644 --- a/.github/actions/bench-diff/action.yml +++ b/.github/actions/bench-diff/action.yml @@ -111,7 +111,9 @@ runs: BENCH_WORKDIR: ${{ inputs.working-directory }} BENCH_HAS_BASE: ${{ github.event_name == 'pull_request' }} run: | - set -u + # `shell: bash` already runs with `-e -o pipefail`; saying so here + # keeps the guarantee visible to whoever edits this next. + set -euo pipefail run_side() { local side="$1" round="$2" root="$3" local out="$RUNNER_TEMP/$BENCH_ID-$side-$round.json" @@ -153,7 +155,7 @@ runs: BENCH_ROUNDS: ${{ inputs.rounds }} BENCH_HAS_BASE: ${{ github.event_name == 'pull_request' }} run: | - set -u + set -euo pipefail shopt -s nullglob head_rounds=("$RUNNER_TEMP/$BENCH_ID"-head-*.json) base_rounds=("$RUNNER_TEMP/$BENCH_ID"-base-*.json) diff --git a/.github/actions/bench-diff/bench-comment.mjs b/.github/actions/bench-diff/bench-comment.mjs index f58f148ab..cce5e39c8 100644 --- a/.github/actions/bench-diff/bench-comment.mjs +++ b/.github/actions/bench-diff/bench-comment.mjs @@ -66,12 +66,13 @@ if (!basePath || !headPath) { process.exit(1); } +// No fallback to `[]` on a read or parse error. An absent base is expressed +// by a report that *is* `[]`, written deliberately by the action, so a file +// that cannot be read or parsed means the measurement broke — and turning +// that into an empty base would report a comparison that never happened. Let +// it throw; a non-zero exit fails the step. function read(path) { - try { - return JSON.parse(readFileSync(path, 'utf8')); - } catch { - return []; - } + return JSON.parse(readFileSync(path, 'utf8')); } function index(entries) { From 4f2857e699f578045b71300db262a2e30ae68852 Mon Sep 17 00:00:00 2001 From: Titouan Mathis Date: Sun, 16 Aug 2026 14:31:04 +0000 Subject: [PATCH 6/6] fix(v4): interleave the two sides of each ratio guard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The nesting guard read 1.91 on a CI runner against a 1.8 ceiling, while the benchmark suite put the same pair at 1.01 on the same runner class. The difference is the harness, not the code: the spec measured one side to completion and then the other, so any drift between the two blocks landed entirely in the ratio. Measure them alternately, exactly as `bench-diff` alternates base and head, and compare 5 000 against 5 000 rather than 2 000 so one scheduling gap is a small fraction of each side. The nesting ratio is now 0.898 to 1.024 across ten runs, quiet and with all eight cores saturated, against 0.82 to 1.15 before. The linearity guard keeps a bias interleaving cannot remove, because its two sides must differ in size: the best of several repeats favours the shorter one, since a 14 ms run slips between interruptions an 80 ms run absorbs. Measured 1.11 to 1.20 per component quiet, up to 1.57 contended. Its threshold therefore moves from 2.5 to 3, placed between the worst noise and the signal it exists to catch — five times the components read about 1.6 per component when linear and would read 5 if quadratic — rather than just above the noise. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_011nNdFD3aQhzfdm3EsCSbS9 --- packages/v4/src/mount-at-scale.spec.ts | 69 ++++++++++++++++++-------- 1 file changed, 49 insertions(+), 20 deletions(-) diff --git a/packages/v4/src/mount-at-scale.spec.ts b/packages/v4/src/mount-at-scale.spec.ts index 77abfa780..f366f0683 100644 --- a/packages/v4/src/mount-at-scale.spec.ts +++ b/packages/v4/src/mount-at-scale.spec.ts @@ -146,6 +146,30 @@ async function mountCost(html: string): Promise { }; } +/** + * Measure two pages against each other and return `b / a`, the ratio of their + * best wall times. + * + * The two sides are measured **alternately**, not one after the other. A + * ratio of two separately-taken measurements absorbs every bit of drift + * between the two blocks: a CI runner that slows down between them reports it + * as a difference between the pages. Nesting read 1.91 that way on + * `ubuntu-latest` while the benchmark suite, which interleaves, put the same + * pair at 1.01 on the same runner class. Alternating puts any drift on both + * sides, which is the same reason `bench-diff` alternates base and head. + */ +async function costRatio(a: string, b: string): Promise { + await mountOnce(a); + await mountOnce(b); + const first: number[] = []; + const second: number[] = []; + for (let index = 0; index < REPEATS; index += 1) { + first.push((await mountOnce(a)).duration); + second.push((await mountOnce(b)).duration); + } + return Math.min(...second) / Math.min(...first); +} + describe('mounting at page scale', () => { // 500 realistic components is a heavy real page. Its un-chunked pass costs // ~7 ms here, so reaching 50 ms needs a machine seven times slower. @@ -165,35 +189,40 @@ describe('mounting at page scale', () => { /** * The failure this really guards against: a change that makes the scan * re-query the document per element turns mounting quadratic, and a - * quadratic page only reveals itself at scale. A ratio of per-component - * costs does not care how fast the runner is, so it can be tighter than - * any wall-clock ceiling — five times the components must not cost more - * than 2.5× per component, where quadratic would cost five. + * quadratic page only reveals itself at scale. * - * Both sides are deliberately in the tens of milliseconds. An earlier - * version compared 500 against 4 000 and read 2.22 on a CI runner while - * reading 1.0 here, because taking the best of several repeats favours the - * shorter side: a 3 ms run dodges an interruption that a 70 ms run - * absorbs. Comparable magnitudes remove that bias. Measured at 1.07 here - * and 0.78 on `ubuntu-latest`, where per-component cost falls with size. + * This guard cannot use two equal sizes — differing sizes are the whole + * question — so it keeps a bias the nesting guard below does not have. + * Taking the best of several repeats favours the shorter side, because a + * 14 ms run slips between interruptions that a 80 ms run absorbs, and + * contention widens the gap: measured at 1.11 to 1.20 per component when + * quiet, and up to 1.57 with all eight cores saturated. An earlier version + * compared 500 against 4 000 and read 2.22 on a CI runner. + * + * So the threshold is placed between the noise and the signal rather than + * just above the noise: five times the components read about 1.6 per + * component at worst when linear, and would read 5 if quadratic. Three + * sits between the two, with room on both sides. */ it('costs the same per component at 5 000 as at 1 000', async () => { - const small = await mountCost(scenarios.flat(1000)); - const large = await mountCost(scenarios.flat(5000)); - expect(large.duration / 5000 / (small.duration / 1000)).toBeLessThan(2.5); + const ratio = await costRatio(scenarios.flat(1000), scenarios.flat(5000)); + expect(ratio / 5).toBeLessThan(3); }, 60000); /** * v4 claims nesting costs nothing, because no parent orchestrates its * children's mounting. Reintroducing that orchestration means a second - * pass that finds and mounts children from each parent, so it would at - * least double the nested side. Measured between 0.82 and 1.15 across - * quiet and fully contended runs here, and below 1.0 on `ubuntu-latest`, - * where a four-deep tree mounts slightly faster than a flat one. + * pass that finds and mounts children from each parent. + * + * Equal sizes on both sides — same component count, same node count, only + * the shape differs — so this one has no short-side bias and can be tight. + * Measured between 0.898 and 1.024 over ten runs, quiet and with every + * core saturated. 5 000 rather than 2 000 so each side is long enough that + * one scheduling gap is a small fraction of it; at 2 000, sequentially, + * this read 1.91 on a CI runner. */ it('mounts a four-deep tree for what a flat one costs', async () => { - const flat = await mountCost(scenarios.flat(2000)); - const nested = await mountCost(scenarios.nested(2000)); - expect(nested.duration / flat.duration).toBeLessThan(1.8); + const ratio = await costRatio(scenarios.flat(5000), scenarios.nested(5000)); + expect(ratio).toBeLessThan(1.8); }, 60000); });