|
| 1 | +#!/usr/bin/env node |
| 2 | +// check-kpi.js — the search KPI as a BUILD GATE, not a report somebody remembers to read. |
| 3 | +// |
| 4 | +// node scripts/check-kpi.js everything (needs _site-org) |
| 5 | +// node scripts/check-kpi.js --labels-only just the label set and the judge; no build needed |
| 6 | +// |
| 7 | +// WHY THIS EXISTS |
| 8 | +// |
| 9 | +// `npm test` ran fourteen checks and not one of them touched the gold set, while the only KPI with |
| 10 | +// teeth anywhere in the repo was a recall@6 floor inside a script `npm test` does not run, over |
| 11 | +// labels that had just been shown to call a verified improvement a significant regression. The |
| 12 | +// build gated on the weakest available number and the primary KPI gated on nothing at all. |
| 13 | +// |
| 14 | +// WHAT IT ASSERTS, and why each one is a thing that has actually gone wrong: |
| 15 | +// |
| 16 | +// 1. EVERY HARVESTED QUERY IS DECIDED EXACTLY ONCE, AND NOTHING IS INVENTED. The labels are |
| 17 | +// hand-written per-query lists in judged/*.js, so the failure mode is not a bad rule — it is |
| 18 | +// a query silently leaving the measurement, or a query that was never harvested being typed |
| 19 | +// into a list from memory. On its first run this caught 18 queries that do not exist, 27 that |
| 20 | +// had been missed and five judged in two places at once. |
| 21 | +// 2. THE LABEL SET IS SELF-CONSISTENT. No query in both gold and quarantine, no target repeated |
| 22 | +// in its own `also`, no duplicates. A KPI that averages over a contradiction is unactionable. |
| 23 | +// 3. THE FILE ON DISK IS WHAT judged/ ASSEMBLES TO. gold.json is generated; if it has drifted |
| 24 | +// from the decisions, the measured labels are not the reviewed labels. |
| 25 | +// 4. EVERY LABELLED PAGE EXISTS. A page that moves would otherwise score 0 for ever and read as |
| 26 | +// a ranking collapse. Section targets carry a #fragment, so both forms count as present. |
| 27 | +// 5. P@1 MACRO CLEARS A COMMITTED FLOOR, and the agent metric clears its own. Not a target — a |
| 28 | +// tripwire, deliberately set below the current value, so an accidental regression fails the |
| 29 | +// build and a deliberate one has to move the floor and say why. |
| 30 | +// |
| 31 | +// Exits non-zero on any failure. |
| 32 | + |
| 33 | +'use strict'; |
| 34 | + |
| 35 | +const fs = require('node:fs'); |
| 36 | +const { execFileSync } = require('node:child_process'); |
| 37 | +const path = require('node:path'); |
| 38 | + |
| 39 | +const ROOT = path.join(__dirname, '..'); |
| 40 | +const KPI = path.join(ROOT, 'scripts', 'search-kpi'); |
| 41 | +const DATA = path.join(KPI, 'data'); |
| 42 | + |
| 43 | +const { fingerprint, integrity } = require(path.join(KPI, 'lib', 'labels.js')); |
| 44 | + |
| 45 | +const labelsOnly = process.argv.includes('--labels-only'); |
| 46 | + |
| 47 | +let failures = 0; |
| 48 | +const fail = (msg) => { failures++; console.error(` FAIL ${msg}`); }; |
| 49 | +const pass = (msg) => console.log(` ok ${msg}`); |
| 50 | + |
| 51 | +// THE FLOORS. Set below the measured value on purpose — a floor at the current number turns every |
| 52 | +// run into a coin toss on rounding, and a floor far below it catches nothing. Roughly two points of |
| 53 | +// slack, which is one standard error of P@1 at this sample size. |
| 54 | +const FLOOR = { |
| 55 | + // Measured on labels 12336a824b18 — 985 headline queries (+246 reported as `seo`) judged from page content, 64 of them targeting an |
| 56 | + // anchored section. Each floor sits roughly one standard error below the measured value: a floor at |
| 57 | + // the current number makes every run a coin toss on rounding, and one far below it catches nothing. |
| 58 | + // |
| 59 | + // NOT comparable with the pre-rebuild numbers. The label set more than doubled (517 natural -> 1083), |
| 60 | + // 18 of the 19 agent queries were retargeted from a reference page to the FAQ section that actually |
| 61 | + // answers them, and the aggregation now pools 86 topics into 43 groups. |
| 62 | + p1Macro: 50.5, // measured 52.6 |
| 63 | + p1Micro: 59.5, // measured 61.6 |
| 64 | + recall6Micro: 88.5, // measured 90.4 |
| 65 | + intentRecall6: 100.0, // measured 100.0 — the agent set has no slack to give |
| 66 | + targetUnreachable: 4.5, // measured 3.8, and this one is a CEILING |
| 67 | + // The reference pages that have to stay FINDABLE even where something answers better. 66.7% in the |
| 68 | + // top six (12 of 18), up from 42.3% before — not because ranking changed, but because the labels are |
| 69 | + // now true, so `mustReach` is measuring the reference page against the query that actually needed it. |
| 70 | + reachTop6: 61.0, |
| 71 | + seoP1: 79.0, // measured 81.3 — the 246 MCP-setup keywords, reported apart from the headline |
| 72 | +}; |
| 73 | + |
| 74 | + |
| 75 | +const goldFile = path.join(DATA, 'gold.json'); |
| 76 | +const quarantineFile = path.join(DATA, 'quarantine.json'); |
| 77 | + |
| 78 | +if (!fs.existsSync(goldFile)) { |
| 79 | + console.error('scripts/search-kpi/data/gold.json is missing — run `node scripts/search-kpi/build-gold.js`'); |
| 80 | + process.exit(1); |
| 81 | +} |
| 82 | + |
| 83 | +const gold = JSON.parse(fs.readFileSync(goldFile, 'utf8')); |
| 84 | +const quarantine = fs.existsSync(quarantineFile) |
| 85 | + ? JSON.parse(fs.readFileSync(quarantineFile, 'utf8')) |
| 86 | + : null; |
| 87 | + |
| 88 | +console.log('\nevery harvested query is judged exactly once, and nothing is invented'); |
| 89 | + |
| 90 | +// The natural labels no longer come from regex rules — they come from scripts/search-kpi/judged/*.js, |
| 91 | +// where each decision is an explicit per-query membership list with the content reason it was made. |
| 92 | +// So the thing worth asserting changed: not "do the rules still fire the same way" but "is every |
| 93 | +// harvested query accounted for, exactly once, with a target that really exists". |
| 94 | +const harvested = JSON.parse(fs.readFileSync(path.join(DATA, 'natural-queries.json'), 'utf8')); |
| 95 | +const judgedDir = path.join(KPI, 'judged'); |
| 96 | +const decided = new Map(); |
| 97 | +const invented = []; |
| 98 | + |
| 99 | +for (const file of fs.readdirSync(judgedDir).filter((f) => f.endsWith('.js')).sort()) { |
| 100 | + const mod = require(path.join(judgedDir, file)); |
| 101 | + |
| 102 | + for (const [target, topic, list] of mod.positive || []) { |
| 103 | + for (const q of list) { |
| 104 | + if (!decided.has(q)) decided.set(q, []); |
| 105 | + decided.get(q).push(`${file} + ${topic} -> ${target}`); |
| 106 | + } |
| 107 | + } |
| 108 | + for (const [why, list] of mod.negative || []) { |
| 109 | + for (const q of list) { |
| 110 | + if (!decided.has(q)) decided.set(q, []); |
| 111 | + decided.get(q).push(`${file} - ${why}`); |
| 112 | + } |
| 113 | + } |
| 114 | +} |
| 115 | + |
| 116 | +const harvestedSet = new Set(harvested); |
| 117 | + |
| 118 | +for (const q of decided.keys()) if (!harvestedSet.has(q)) invented.push(q); |
| 119 | + |
| 120 | +const twice = [...decided.entries()].filter(([, w]) => w.length > 1); |
| 121 | +const unjudged = harvested.filter((q) => !decided.has(q)); |
| 122 | + |
| 123 | +if (invented.length) { |
| 124 | + fail(`${invented.length} judged quer(ies) are not in the harvest — mistyped: ${invented.slice(0, 5).join(', ')}`); |
| 125 | +} else if (twice.length) { |
| 126 | + fail(`${twice.length} quer(ies) judged twice, e.g. "${twice[0][0]}"\n ${twice[0][1].join('\n ')}`); |
| 127 | +} else if (unjudged.length) { |
| 128 | + fail(`${unjudged.length} harvested quer(ies) have no decision: ${unjudged.slice(0, 5).join(', ')}`); |
| 129 | +} else { |
| 130 | + pass(`all ${harvested.length} harvested queries decided exactly once, none invented`); |
| 131 | +} |
| 132 | + |
| 133 | +// The other two query populations are covered the same way, and for the same reason. `judged/ |
| 134 | +// question.js` and `judged/intent.js` are hand-written lists of one case per line, so the failure |
| 135 | +// mode is a query going missing from the set — which does not look like a failure, it just makes the |
| 136 | +// measurement smaller. These two JSON files are the population of record: question-queries.json is |
| 137 | +// the assistant's 115 phrasings, intent-queries.json the 19 calls a real agent made while building |
| 138 | +// an app, along with what two rankers returned for each at the time. Asserting coverage is what |
| 139 | +// keeps them inputs rather than souvenirs. |
| 140 | +const covers = (label, file, judgedFile, exported) => { |
| 141 | + const raw = JSON.parse(fs.readFileSync(path.join(DATA, file), 'utf8')); |
| 142 | + const want = new Set(raw.queries.map((c) => c.query)); |
| 143 | + const have = new Set(require(path.join(judgedDir, judgedFile))[exported].map((c) => c[0])); |
| 144 | + const gone = [...want].filter((q) => !have.has(q)); |
| 145 | + const extra = [...have].filter((q) => !want.has(q)); |
| 146 | + |
| 147 | + if (gone.length || extra.length) { |
| 148 | + fail(`judged/${judgedFile} does not cover ${file} — ${gone.length} missing, ${extra.length} not in it` |
| 149 | + + (gone.length ? `\n missing: ${gone.slice(0, 3).join(' | ')}` : '') |
| 150 | + + (extra.length ? `\n invented: ${extra.slice(0, 3).join(' | ')}` : '')); |
| 151 | + } else { |
| 152 | + pass(`all ${want.size} ${label} queries are judged, none invented`); |
| 153 | + } |
| 154 | +}; |
| 155 | + |
| 156 | +covers('chat-shaped', 'question-queries.json', 'question.js', 'QUESTION'); |
| 157 | +covers('agent search_docs', 'intent-queries.json', 'intent.js', 'INTENT'); |
| 158 | + |
| 159 | +console.log('\nthe label set is self-consistent'); |
| 160 | + |
| 161 | +const problems = integrity(gold, quarantine); |
| 162 | + |
| 163 | +if (problems.length) { |
| 164 | + fail(`${problems.length} integrity problem(s) in the label set:`); |
| 165 | + for (const line of problems.slice(0, 12)) console.error(` ${line}`); |
| 166 | +} else { |
| 167 | + pass(`${gold.queries.length} labels: no duplicates, no gold/quarantine overlap, no self-referencing also`); |
| 168 | +} |
| 169 | + |
| 170 | +const stamp = fingerprint(gold.queries); |
| 171 | + |
| 172 | +if (gold.fingerprint !== stamp) { |
| 173 | + fail(`gold.json says its fingerprint is ${gold.fingerprint}, but its own labels hash to ${stamp}` |
| 174 | + + ' — the file was hand-edited without re-stamping'); |
| 175 | +} else { |
| 176 | + pass(`fingerprint ${stamp} matches the labels in the file`); |
| 177 | +} |
| 178 | + |
| 179 | +// The committed file has to be what the rules produce. Re-judging in-process rather than shelling |
| 180 | +// out, so this cannot be defeated by a stale build. |
| 181 | +console.log('\nthe committed labels are what build-gold.js assembles'); |
| 182 | + |
| 183 | +// gold.json is generated. If it has drifted from the decisions in judged/, the measured labels are |
| 184 | +// not the reviewed labels. |
| 185 | +const rebuilt = execFileSync(process.execPath, [path.join(KPI, 'build-gold.js')], { encoding: 'utf8' }); |
| 186 | +const rebuiltStamp = (rebuilt.match(/fingerprint ([0-9a-f]{12})/) || [])[1]; |
| 187 | + |
| 188 | +if (rebuiltStamp !== stamp) { |
| 189 | + fail(`re-assembling produces ${rebuiltStamp}, but gold.json holds ${stamp}` |
| 190 | + + ' — run `node scripts/search-kpi/build-gold.js` and review the diff'); |
| 191 | +} else { |
| 192 | + pass('re-assembling from judged/ reproduces the committed label set exactly'); |
| 193 | +} |
| 194 | + |
| 195 | +if (labelsOnly) { |
| 196 | + console.log(`\n${failures ? `${failures} failure(s)` : 'all label checks passed'}\n`); |
| 197 | + process.exit(failures ? 1 : 0); |
| 198 | +} |
| 199 | + |
| 200 | +// ---- everything below needs a built index --------------------------------- |
| 201 | +const { load, evaluate, summarise } = require(path.join(KPI, 'lib', 'harness.js')); |
| 202 | + |
| 203 | +console.log('\nevery labelled page is in the built index'); |
| 204 | + |
| 205 | +const ranker = load(null); |
| 206 | +const known = new Set(); |
| 207 | + |
| 208 | +// A record is addressable both by its own url and by the page it sits on, because a section record |
| 209 | +// carries a #fragment and a label may legitimately name either. Stripping the fragment here was |
| 210 | +// harmless while every target was a whole page; it silently rejects every anchored target. |
| 211 | +const remember = (url) => { known.add(String(url)); known.add(String(url).split('#')[0]); }; |
| 212 | +for (const record of ranker.state.t1.records) remember(record.u); |
| 213 | +for (const record of (ranker.state.x1 ? ranker.state.x1.records : [])) { |
| 214 | + remember(record.u); |
| 215 | +} |
| 216 | + |
| 217 | +const missing = new Set(); |
| 218 | + |
| 219 | +for (const testCase of gold.queries) { |
| 220 | + for (const url of [testCase.target, ...(testCase.also || [])]) { |
| 221 | + if (!known.has(url)) missing.add(url); |
| 222 | + } |
| 223 | +} |
| 224 | + |
| 225 | +if (missing.size) { |
| 226 | + fail(`${missing.size} labelled page(s) are not in the index: ${[...missing].slice(0, 8).join(', ')}`); |
| 227 | +} else { |
| 228 | + pass(`${known.size} indexed pages cover every target and every also`); |
| 229 | +} |
| 230 | + |
| 231 | +console.log('\nthe KPI clears its floors'); |
| 232 | + |
| 233 | +// Score the headline set. `seo` is measured by gold.js and reported there; it reads ~19 points |
| 234 | +// above everything else, so folding 246 of it into the floors would let the site's real weak spots |
| 235 | +// regress behind one page that always wins. |
| 236 | +const scored = gold.queries.filter((c) => c.src !== 'seo'); |
| 237 | +const results = evaluate(ranker, scored); |
| 238 | +const micro = summarise(results); |
| 239 | + |
| 240 | +// Macro over pooled groups, the same way gold.js reports it — a floor on a different aggregation |
| 241 | +// than the headline would be a floor on a number nobody reads. |
| 242 | +const MIN_TOPIC = 5; |
| 243 | +const byTopic = new Map(); |
| 244 | + |
| 245 | +for (const result of results) { |
| 246 | + if (!byTopic.has(result.topic)) byTopic.set(result.topic, []); |
| 247 | + byTopic.get(result.topic).push(result); |
| 248 | +} |
| 249 | + |
| 250 | +const groups = new Map(); |
| 251 | + |
| 252 | +for (const [topic, list] of byTopic) { |
| 253 | + const key = list.length >= MIN_TOPIC ? topic : 'misc'; |
| 254 | + |
| 255 | + if (!groups.has(key)) groups.set(key, []); |
| 256 | + groups.get(key).push(...list); |
| 257 | +} |
| 258 | + |
| 259 | +const groupP1 = [...groups.values()].map((list) => summarise(list).p1); |
| 260 | +const p1Macro = groupP1.reduce((a, b) => a + b, 0) / groupP1.length; |
| 261 | + |
| 262 | +const floor = (label, value, min) => { |
| 263 | + if (value + 1e-9 < min) { |
| 264 | + fail(`${label} ${value.toFixed(1)}% is below the committed floor ${min.toFixed(1)}%`); |
| 265 | + } else { |
| 266 | + pass(`${label} ${value.toFixed(1)}% (floor ${min.toFixed(1)}%)`); |
| 267 | + } |
| 268 | +}; |
| 269 | + |
| 270 | +floor('P@1 macro', p1Macro, FLOOR.p1Macro); |
| 271 | +floor('P@1 micro', micro.p1, FLOOR.p1Micro); |
| 272 | +floor('recall@6 micro', micro.top6, FLOOR.recall6Micro); |
| 273 | + |
| 274 | +const intent = summarise(results.filter((r) => r.src === 'intent')); |
| 275 | + |
| 276 | +floor('intent recall@6', intent.top6, FLOOR.intentRecall6); |
| 277 | + |
| 278 | +const seo = summarise(evaluate(ranker, gold.queries.filter((c) => c.src === 'seo'))); |
| 279 | + |
| 280 | +floor('seo P@1 (held out of the headline, still gated)', seo.p1, FLOOR.seoP1); |
| 281 | + |
| 282 | +const reach = results.filter((r) => r.mustReach); |
| 283 | + |
| 284 | +if (reach.length) { |
| 285 | + const inSix = reach.filter((r) => r.mustReachRank >= 1 && r.mustReachRank <= 6).length; |
| 286 | + |
| 287 | + floor('reference page in top 6', (inSix / reach.length) * 100, FLOOR.reachTop6); |
| 288 | +} |
| 289 | + |
| 290 | +if (micro.targetUnreachable > FLOOR.targetUnreachable + 1e-9) { |
| 291 | + fail(`${micro.targetUnreachable.toFixed(1)}% of targets are never returned at all, above the ` |
| 292 | + + `${FLOOR.targetUnreachable.toFixed(1)}% ceiling — that is an indexing regression, not a ranking one`); |
| 293 | +} else { |
| 294 | + pass(`targets never returned ${micro.targetUnreachable.toFixed(1)}% (ceiling ${FLOOR.targetUnreachable.toFixed(1)}%)`); |
| 295 | +} |
| 296 | + |
| 297 | +if (failures) { |
| 298 | + console.error(`\n${failures} KPI check(s) failed.`); |
| 299 | + console.error('A floor is a tripwire, not a target: if the change is deliberate, move the floor in'); |
| 300 | + console.error('scripts/check-kpi.js and say why in the commit message.\n'); |
| 301 | + process.exit(1); |
| 302 | +} |
| 303 | + |
| 304 | +console.log(`\nall KPI checks passed — ${gold.queries.length} labels, ${groups.size} macro groups\n`); |
0 commit comments