From e64a719bb223276e261f0ec733ecc6892df8a95d Mon Sep 17 00:00:00 2001 From: Bitcoin Universe Date: Sun, 30 Aug 2026 02:41:49 +0000 Subject: [PATCH 1/3] The performance gate was measuring the runner in two places It failed on develop and passed on the pull request, on identical trees. Both readings were honest; the check was not. Largest paint. The gate did not threshold it, on the stated grounds that the number swings with the load, and then failed a route for reporting no paint at all on the stated grounds that "that answer does not move with the load". It does. On the busy runner the Dogecoin transaction route fired its load event at ten and a half seconds, and the six seconds the probe waited after that were not enough for a contentful paint to land, so it read zero on a build that measured twelve seconds on a quiet machine. Two changes. The paint is now waited for, up to twenty seconds, instead of read once and taken as final. And when it still has not arrived the page is asked what is on it: a screenful of text with no paint entry is a measurement problem and is reported as one, while a page with nothing on it is the fault the check exists for. That distinction genuinely does not move with the load, which is the property the old comment claimed and did not have. Layout shift. The header called it "nearly deterministic, a property of the stylesheet rather than of the clock". It is less deterministic than that: the same tree measured 0.042, 0.046 and 0.069 on the blocks route across three machines, because a busier machine delivers content later and a shift that lands after the first paint counts while the same shift before it does not. Load can only add shifts, never remove one, so the smallest of several readings is the closest available estimate of the shift the stylesheet is actually responsible for. A route over its ceiling is now measured again, up to twice, and judged on the minimum. The table says when a route was measured more than once, because a gate that quietly retries until it passes is worse than one that fails. Neither change lowers a threshold. The eager payload budget, the 0.1 layout budget and the recorded exception on the transaction route are all unchanged. --- scripts/universe/visual-qa/mobile-perf.mjs | 172 +++++++++++++++------ 1 file changed, 128 insertions(+), 44 deletions(-) diff --git a/scripts/universe/visual-qa/mobile-perf.mjs b/scripts/universe/visual-qa/mobile-perf.mjs index ac508ffa1f4..b7d0e28e8ae 100644 --- a/scripts/universe/visual-qa/mobile-perf.mjs +++ b/scripts/universe/visual-qa/mobile-perf.mjs @@ -15,9 +15,13 @@ * machine, so this is the one worth gating on. * * layout cumulative layout shift, and every shift large enough to be - * worth naming, with the element that moved. Nearly deterministic - * once the page has settled, because it is a property of the - * stylesheet rather than of the clock. + * worth naming, with the element that moved. Less deterministic + * than it first appears: a busier machine delivers content later, + * and a shift that lands after the first paint counts while the + * same shift before it does not. The same tree measured 0.042, + * 0.046 and 0.069 on one route across three machines. Load can + * only add shifts, never remove one, so a route over budget is + * measured again and judged on the smallest reading. * * paint largest contentful paint under a throttled profile. Reported, * and only failed on a wide margin, because it is the one that @@ -201,6 +205,22 @@ function observeVitals() { } catch { /* as above */ } } +/** + * Whether the page has rendered anything a visitor would call content. + * + * This is the second half of the largest-paint check. On its own, an absent + * paint entry says one of two very different things, and the useful one is + * rare: either the shell did not render, or it had not rendered yet when the + * measurement was taken. Asking the page what is on it separates them, and it + * is a question whose answer does not depend on how busy the machine is. + */ +function renderedSomething() { + const main = document.querySelector('main') || document.body; + const text = (main.innerText || '').trim(); + const paintable = main.querySelectorAll('img, svg, canvas, table, h1, h2, h3, p, li').length; + return { textLength: text.length, paintable, sample: text.slice(0, 120) }; +} + async function run() { mkdirSync(OUT, { recursive: true }); @@ -228,7 +248,20 @@ async function run() { const known = []; const rows = []; - for (const route of routes) { + /** + * One route, measured once. + * + * Extracted so a route can be measured again. Layout shift is not as + * deterministic as it looks: the same tree measured 0.042, 0.046 and 0.069 + * on the blocks route on three machines, because a busier machine delivers + * content later and a shift that lands after the first paint counts while + * the same shift before it does not. Load can only add shifts, never remove + * one, so the smallest of several measurements is the closest thing to the + * figure the stylesheet is actually responsible for. That is why a route + * over budget is measured again rather than failed on one reading, and why + * the statistic is the minimum rather than an average. + */ + const measure = async (route) => { const context = await browser.newContext({ viewport: { width: 390, height: 844 }, deviceScaleFactor: 1, @@ -258,64 +291,114 @@ async function run() { // that matter, which are the ones a visitor sees rather than the ones // that happen before the first paint. await page.waitForTimeout(6_000); - const vitals = await page.evaluate(() => window.__vitals); + + // A paint that has not been reported yet is not a paint that will never + // happen. On a loaded runner this route's `load` event fired at ten and + // a half seconds, and six more were not enough for a contentful paint to + // land, so it read zero on a build that measured twelve seconds on a + // quiet machine. Wait for it properly instead of taking the first read + // as final. + let vitals = await page.evaluate(() => window.__vitals); + const paintDeadline = Date.now() + 20_000; + while (vitals.lcp === 0 && Date.now() < paintDeadline) { + await page.waitForTimeout(1_000); + vitals = await page.evaluate(() => window.__vitals); + } + const nav = await page.evaluate(() => { const [entry] = performance.getEntriesByType('navigation'); return entry ? { domContentLoaded: Math.round(entry.domContentLoadedEventEnd), load: Math.round(entry.loadEventEnd) } : { domContentLoaded: 0, load: 0 }; }); + const rendered = await page.evaluate(renderedSomething); - const row = { + return { route: route.id, lcpMs: Math.round(vitals.lcp), cls: Math.round(vitals.cls * 1000) / 1000, shifts: vitals.shifts, longTasks: vitals.longTasks, longTaskMs: Math.round(vitals.longTaskMs), + rendered, ...nav, }; - rows.push(row); - - if (row.cls > BUDGETS.cls) { - const worst = row.shifts - .sort((a, b) => b.value - a.value) - .slice(0, 3) - .map((s) => `${s.value} from ${s.sources.join(', ')}`) - .join('; '); - const debt = KNOWN_LAYOUT_DEBT[route.id]; - if (debt && row.cls <= debt.cls + DEBT_TOLERANCE) { - known.push(`${route.id}: layout shifted ${row.cls} against a recorded ${debt.cls}. ${debt.note}`); - } else if (debt) { - failures.push( - `${route.id}: layout shifted ${row.cls}, worse than the ${debt.cls} recorded for it` - + ` by more than ${DEBT_TOLERANCE}${worst ? ` (${worst})` : ''}`, - ); - } else { - failures.push(`${route.id}: layout shifted ${row.cls}, over the ${BUDGETS.cls} budget${worst ? ` (${worst})` : ''}`); - } - } - // Largest paint is reported and not gated on a threshold. - // - // It was, at four times the field target, and that was still wrong. This - // runner builds, serves and drives several browsers at once, and a - // measurement taken while it does reflects the queue rather than the - // build: the same commit measured twelve seconds and twenty on two - // routes whose shells are identical. A number that swings by eight - // seconds between runs of the same code cannot be a gate; used as one it - // would fail honest changes and pass slow ones depending on what else - // the machine was doing. - // - // What is still worth failing on is a paint that never happens at all, - // which is a shell that did not render rather than one that rendered - // slowly, and that answer does not move with the load. - if (row.lcpMs === 0) { - failures.push(`${route.id}: no largest contentful paint was reported at all, so either nothing painted or the observer never ran`); - } + } finally { await page.close().catch(() => undefined); await context.close().catch(() => undefined); } + }; + + for (const route of routes) { + let row = await measure(route); + + // A route over the layout budget is measured again, up to twice, and + // judged on the smallest reading. See the note on `measure`: a busy + // machine can only add shifts, so the minimum is the closest available + // estimate of the shift the stylesheet is responsible for, and a single + // reading taken while the runner was building something else is not + // evidence about this commit. + const debt = KNOWN_LAYOUT_DEBT[route.id]; + const ceiling = debt ? debt.cls + DEBT_TOLERANCE : BUDGETS.cls; + for (let attempt = 0; attempt < 2 && row.cls > ceiling; attempt++) { + const again = await measure(route); + if (again.cls < row.cls) row = { ...again, remeasured: attempt + 1 }; + else row = { ...row, remeasured: attempt + 1 }; + } + rows.push(row); + + if (row.cls > BUDGETS.cls) { + const worst = row.shifts + .slice() + .sort((a, b) => b.value - a.value) + .slice(0, 3) + .map((s) => `${s.value} from ${s.sources.join(', ')}`) + .join('; '); + if (debt && row.cls <= ceiling) { + known.push(`${route.id}: layout shifted ${row.cls} against a recorded ${debt.cls}. ${debt.note}`); + } else if (debt) { + failures.push( + `${route.id}: layout shifted ${row.cls}, worse than the ${debt.cls} recorded for it` + + ` by more than ${DEBT_TOLERANCE}${worst ? ` (${worst})` : ''}`, + ); + } else { + failures.push(`${route.id}: layout shifted ${row.cls}, over the ${BUDGETS.cls} budget${worst ? ` (${worst})` : ''}`); + } + } + + // Largest paint is reported and not gated on a threshold. + // + // It was, at four times the field target, and that was wrong: this runner + // builds, serves and drives several browsers at once, and the same commit + // measured twelve seconds and twenty on two routes whose shells are + // identical. A number that swings by eight seconds between runs of the + // same code cannot gate anything. + // + // The absence of a paint is not load-independent either, which is what + // this check assumed until a run proved otherwise: on a busy runner the + // Dogecoin transaction route fired `load` at ten and a half seconds and + // still had no contentful paint six seconds later, on a build that + // measured twelve seconds on a quiet machine. So the paint is waited for + // properly, and when it still has not arrived the page is asked what is on + // it. A page with a screenful of text that has reported no paint entry is + // a measurement problem. A page with nothing on it is the fault this check + // is for, and that distinction does not move with the load. + if (row.lcpMs === 0) { + const { textLength, paintable } = row.rendered || { textLength: 0, paintable: 0 }; + if (textLength < 40 && paintable < 3) { + failures.push( + `${route.id}: nothing painted. No largest contentful paint after waiting, and the page holds` + + ` ${textLength} characters of text in ${paintable} paintable elements`, + ); + } else { + known.push( + `${route.id}: no largest contentful paint was reported, but the page has rendered` + + ` (${textLength} characters, ${paintable} paintable elements). Treated as a measurement` + + ` artefact of a loaded runner rather than a shell that did not render`, + ); + } + } } await browser.close(); @@ -343,7 +426,8 @@ async function run() { console.log('route LCP CLS long tasks'); for (const r of rows) { console.log( - `${r.route.padEnd(16)} ${String(r.lcpMs + 'ms').padEnd(8)} ${String(r.cls).padEnd(6)} ${r.longTasks} (${r.longTaskMs}ms)`, + `${r.route.padEnd(16)} ${String(r.lcpMs + 'ms').padEnd(8)} ${String(r.cls).padEnd(6)} ${r.longTasks} (${r.longTaskMs}ms)` + + (r.remeasured ? ` measured ${r.remeasured + 1} times, smallest kept` : ''), ); for (const s of r.shifts.sort((a, b) => b.value - a.value).slice(0, 3)) { console.log(` shift ${s.value} from ${s.sources.join(', ')}`); From 8581f4794331420f8a208a2a610a344565d54f2c Mon Sep 17 00:00:00 2001 From: Bitcoin Universe Date: Sun, 30 Aug 2026 03:45:54 +0000 Subject: [PATCH 2/3] The gateway refused the script it was serving A release changed the document's inline theme bootstrap and left gateway.mjs byte identical. The cutover therefore did the right thing and left the gateway running, and the running gateway went on allowing the previous build's script hash while refusing the one it was itself serving: Executing inline script violates the following Content Security Policy directive 'script-src 'self' 'sha256-TLx68/+2SeR3+dZreFBX1fiMo91olqe4aVTfKFd48Ik='' Every page loaded with the theme bootstrap blocked. The only sign was a console error, which is why this is the second defect this week whose whole visible symptom was something nobody was looking at. The cause is a contradiction inside this file. The static root is a fixed path whose contents a release swaps underneath it, and that is deliberate: it is exactly why a frontend change needs no gateway restart, and the release script says so in as many words. The policy was computed once at start-up and did not follow that swap. One half of the file tracked the build behind the path and the other half remembered the build that was there when the process began. It follows the file now, keyed on size and modification time, so a document costs one stat and a hash is computed only when the file actually changes. The test writes two builds behind the same path and asserts the policy names the second and not the first, including the case where both land in the same second, which is a window a release fits inside. Against the previous start-up-pinned version it fails. --- scripts/universe/gateway.mjs | 48 +++++++++++++++++++++++++++---- scripts/universe/gateway.test.mjs | 48 ++++++++++++++++++++++++++++++- 2 files changed, 89 insertions(+), 7 deletions(-) diff --git a/scripts/universe/gateway.mjs b/scripts/universe/gateway.mjs index bfcca6a5f3a..292735df32c 100644 --- a/scripts/universe/gateway.mjs +++ b/scripts/universe/gateway.mjs @@ -111,8 +111,8 @@ const CONTENT_SECURITY_POLICY_PARTS = [ /** * The build injects one inline script into the document to name the theme * files, and its content changes with every build. Rather than weaken the - * policy with 'unsafe-inline', its hash is computed once at start-up and - * allowed by name. Anything else inline stays blocked. + * policy with 'unsafe-inline', its hash is allowed by name. Anything else + * inline stays blocked. */ function inlineScriptHashes() { const index = join(ROOT, 'index.html'); @@ -133,18 +133,54 @@ function inlineScriptHashes() { return hashes; } -const CONTENT_SECURITY_POLICY = (() => { +/** + * The policy for the document being served now, not for the one that was there + * at start-up. + * + * This used to be computed once when the process began. That is wrong here for + * the same reason the static root is resolved per request: `ROOT` is a fixed + * path whose contents a release swaps underneath it, which is exactly why a + * frontend change needs no gateway restart. A policy pinned at start-up does + * not follow that swap. + * + * It reached production. A release changed the document's inline script and + * left `gateway.mjs` byte identical, so the cutover correctly left the gateway + * running, and the running gateway went on allowing the previous build's hash + * while refusing the script it was itself serving. Every page loaded with the + * theme bootstrap blocked, and the only sign was a console error. + * + * Keyed on the file's size and modification time, so a document costs one stat + * and a hash is computed only when the file behind the path actually changes. + */ +let policyCache = null; + +function indexIdentity() { + try { + const stats = statSync(join(ROOT, 'index.html')); + return `${stats.size}:${stats.mtimeMs}`; + } catch { + return 'absent'; + } +} + +export function contentSecurityPolicy() { + const key = indexIdentity(); + if (policyCache?.key === key) { + return policyCache.value; + } const hashes = inlineScriptHashes(); - return CONTENT_SECURITY_POLICY_PARTS.map((part) => + const value = CONTENT_SECURITY_POLICY_PARTS.map((part) => part.startsWith('script-src') && hashes.length ? `${part} ${hashes.join(' ')}` : part, ).join('; '); -})(); + policyCache = { key, value }; + return value; +} function withSecurityHeaders(headers, isDocument) { const merged = { ...headers, ...SECURITY_HEADERS }; - if (isDocument) merged['content-security-policy'] = CONTENT_SECURITY_POLICY; + if (isDocument) merged['content-security-policy'] = contentSecurityPolicy(); return merged; } diff --git a/scripts/universe/gateway.test.mjs b/scripts/universe/gateway.test.mjs index d093c972005..c90312b3cc7 100644 --- a/scripts/universe/gateway.test.mjs +++ b/scripts/universe/gateway.test.mjs @@ -3,7 +3,8 @@ import assert from 'node:assert/strict'; // Importing the gateway must not open a socket. process.env.UNIVERSE_GATEWAY_NO_LISTEN = '1'; -const { routeFor, websocketUpstreamFor, inheritedListenerFd } = await import('./gateway.mjs'); +const { routeFor, websocketUpstreamFor, inheritedListenerFd, contentSecurityPolicy } = + await import('./gateway.mjs'); /** * The path rewrite is load bearing. The explorer backend registers every route @@ -140,3 +141,48 @@ test('a handover of no sockets is not a handover', () => { assert.equal(inheritedListenerFd({ LISTEN_PID: '42' }, 42), null); assert.equal(inheritedListenerFd({ LISTEN_PID: '42', LISTEN_FDS: 'two' }, 42), null); }); + +/** + * The document policy has to describe the document being served. + * + * `UNIVERSE_GATEWAY_ROOT` is a fixed path whose contents a release swaps + * underneath it, which is why a frontend change needs no gateway restart. The + * policy was computed once at start-up and did not follow that swap, so a + * release that changed the document's inline script and left `gateway.mjs` + * byte identical produced a running gateway allowing the previous build's hash + * and refusing the script it was itself serving. It reached production, and + * the only sign was a console error on every page. + */ +test('the content policy follows the build behind the static root', async () => { + const { mkdtempSync, writeFileSync, utimesSync } = await import('node:fs'); + const { tmpdir } = await import('node:os'); + const { join } = await import('node:path'); + const { createHash } = await import('node:crypto'); + + const root = mkdtempSync(join(tmpdir(), 'gateway-csp-')); + process.env.UNIVERSE_GATEWAY_ROOT = root; + process.env.UNIVERSE_GATEWAY_NO_LISTEN = '1'; + // A second copy of the module, bound to a root this test controls. The one + // imported at the top of this file is bound to the default root. + const gateway = await import(`./gateway.mjs?csp=${Date.now()}`); + + const hashOf = (body) => + `'sha256-${createHash('sha256').update(body, 'utf8').digest('base64')}'`; + + const first = 'window.__a=1;'; + writeFileSync(join(root, 'index.html'), ``); + const before = gateway.contentSecurityPolicy(); + assert.ok(before.includes(hashOf(first)), 'the first build is allowed by name'); + + const second = 'window.__b=2;window.__c=3;'; + writeFileSync(join(root, 'index.html'), ``); + // Same second, different content: the identity has to be more than a + // timestamp at one second resolution, which a release can land inside. + const when = new Date(1780000000000); + utimesSync(join(root, 'index.html'), when, when); + writeFileSync(join(root, 'index.html'), ``); + + const after = gateway.contentSecurityPolicy(); + assert.ok(after.includes(hashOf(second)), 'the build now behind the path is allowed'); + assert.ok(!after.includes(hashOf(first)), 'the build that is gone is no longer allowed'); +}); From ce96d28f2558d5e5d9080ad6a7ab1edacfaadaf6 Mon Sep 17 00:00:00 2001 From: Bitcoin Universe Date: Sun, 30 Aug 2026 03:49:30 +0000 Subject: [PATCH 3/3] Read the policy from the file, not from a key that can collide The first version of this fix cached the policy against the document's size and modification time. That key is wrong in a way that would have been very hard to find afterwards: two builds of the same length restored to the same instant share it, which is exactly what a tar extraction preserving mtimes can produce, and the resulting failure is a blocked script and nothing else. The test asserted that case and passed for the wrong reason. `utimesSync` takes a Date, so it wrote a millisecond-truncated time while `mtimeMs` carries finer precision, and the keys differed by an accident of rounding rather than by anything the code intended. The document is 3.4 kilobytes and is the same file the response is about to serve. The saving was never worth the class of bug it kept open, so it is read each time. The test now covers both dimensions on purpose: a build that changes the length and the time, and a build of exactly the same length with the modification time put back. Neither may be load bearing on its own. Against a start-up-pinned policy it still fails. --- scripts/universe/gateway.mjs | 28 ++++++++-------------------- scripts/universe/gateway.test.mjs | 25 ++++++++++++++++++------- 2 files changed, 26 insertions(+), 27 deletions(-) diff --git a/scripts/universe/gateway.mjs b/scripts/universe/gateway.mjs index 292735df32c..06a29c14f68 100644 --- a/scripts/universe/gateway.mjs +++ b/scripts/universe/gateway.mjs @@ -149,33 +149,21 @@ function inlineScriptHashes() { * while refusing the script it was itself serving. Every page loaded with the * theme bootstrap blocked, and the only sign was a console error. * - * Keyed on the file's size and modification time, so a document costs one stat - * and a hash is computed only when the file behind the path actually changes. + * Read from the file on each document rather than cached against its size and + * modification time. That cheaper key is wrong in a way that is hard to see + * afterwards: two builds of the same length restored to the same instant, which + * is what a tar extraction preserving mtimes can produce, share it, and the + * failure it causes is a blocked script and nothing else. The document is 3.4 + * kilobytes and this is the same file the response is about to serve, so the + * saving was never worth the class of bug it kept open. */ -let policyCache = null; - -function indexIdentity() { - try { - const stats = statSync(join(ROOT, 'index.html')); - return `${stats.size}:${stats.mtimeMs}`; - } catch { - return 'absent'; - } -} - export function contentSecurityPolicy() { - const key = indexIdentity(); - if (policyCache?.key === key) { - return policyCache.value; - } const hashes = inlineScriptHashes(); - const value = CONTENT_SECURITY_POLICY_PARTS.map((part) => + return CONTENT_SECURITY_POLICY_PARTS.map((part) => part.startsWith('script-src') && hashes.length ? `${part} ${hashes.join(' ')}` : part, ).join('; '); - policyCache = { key, value }; - return value; } function withSecurityHeaders(headers, isDocument) { diff --git a/scripts/universe/gateway.test.mjs b/scripts/universe/gateway.test.mjs index c90312b3cc7..b25023e119c 100644 --- a/scripts/universe/gateway.test.mjs +++ b/scripts/universe/gateway.test.mjs @@ -154,7 +154,7 @@ test('a handover of no sockets is not a handover', () => { * the only sign was a console error on every page. */ test('the content policy follows the build behind the static root', async () => { - const { mkdtempSync, writeFileSync, utimesSync } = await import('node:fs'); + const { mkdtempSync, writeFileSync, utimesSync, statSync } = await import('node:fs'); const { tmpdir } = await import('node:os'); const { join } = await import('node:path'); const { createHash } = await import('node:crypto'); @@ -174,15 +174,26 @@ test('the content policy follows the build behind the static root', async () => const before = gateway.contentSecurityPolicy(); assert.ok(before.includes(hashOf(first)), 'the first build is allowed by name'); + // A longer script: the file changes size as well as time. const second = 'window.__b=2;window.__c=3;'; writeFileSync(join(root, 'index.html'), ``); - // Same second, different content: the identity has to be more than a - // timestamp at one second resolution, which a release can land inside. - const when = new Date(1780000000000); - utimesSync(join(root, 'index.html'), when, when); - writeFileSync(join(root, 'index.html'), ``); - const after = gateway.contentSecurityPolicy(); assert.ok(after.includes(hashOf(second)), 'the build now behind the path is allowed'); assert.ok(!after.includes(hashOf(first)), 'the build that is gone is no longer allowed'); + + // And a script of exactly the same length, with the modification time forced + // back to what it was. Neither dimension of the cache key may be load + // bearing on its own: a build that changes the file without changing its + // size has to be noticed, and so has one that lands at the same instant. + const stamped = statSync(join(root, 'index.html')); + const third = 'window.__b=9;window.__c=8;'; + assert.equal(third.length, second.length, 'the two scripts are the same length'); + writeFileSync(join(root, 'index.html'), ``); + utimesSync(join(root, 'index.html'), stamped.atime, stamped.mtime); + + const sameSizeSameTime = gateway.contentSecurityPolicy(); + assert.ok( + sameSizeSameTime.includes(hashOf(third)), + 'a build of the same size at the same instant is still the build being served', + ); });