feat(kpi): commit the two measurements the ranker work was judged by

Both existed only as throwaway scripts while five ranker fixes were decided on
their output, which meant the evidence for every one of those commits was one
`rm -rf /tmp` from being unreproducible.

kpi:recall — recall@6 for BOTH rankers on one corpus: this site's, and the
@imqueue MCP server's rankEntries, whose corpus it rebuilds to match loadIndex()
+ loadApiIndex() exactly. An agent reads all six results search_docs returns, so
membership is the whole question and rank inside the set is noise; measure.js's
position decay answers the human question instead. --list enumerates the queries
the MCP ranker answers and this one does not, which is the number that actually
gates the switch. It degrades to the site half alone when the sibling checkout is
absent rather than failing.

kpi:compare — the working tree's ranker against another commit's, query by query,
both loaded in one process over one corpus so the only variable is the code, and
the baseline read from git so it cannot drift. This exists because an aggregate
hid a real regression twice: a -0.2 macro that was three named queries, one
leaving the top ten; and a 0.0 natural that was simultaneously dropping 13
artificial queries, two of them 1 -> 11 and 10 -> 24. That change was reverted on
the strength of this list, and nothing in measure.js would have objected.

harness.js gains recall@6, and finally loads the PEER tiers — x1/x2 — which it
ignored from the start. That omission is the plan's biggest risk, not a detail:
imqueue.com reaches the ranker only through those feeds, so `pricing commercial
license` was scored against imqueue.org's own /license/ alone. The one question
with revenue attached was the half measured least. Optional, since a local
build:org produces no peer feed and evaluate() already scores an external hit as
a miss.

Reproduces the numbers the ranker commits claimed: site recall@6 99.5% against
the MCP ranker's 83.9%, and zero queries where the MCP ranker still wins. The
site KPI is unchanged at 94.0/88.9 natural and 91.2/95.4 artificial, so loading
the peers moves no aggregate.

One correction those commits deserve: compare.js reports 392 artificial queries
better against a38250a, not the 376 recorded, because the scratchpad script did
not load the peer feeds. The committed figures were conservative.

Author: Mikhus

package.json

@@ -33,18 +33,18 @@
     "probe:agent-analytics": "node scripts/probe-agent-analytics.js",
     "check:dates": "node scripts/gen-page-dates.js --check",
     "check:sitemap": "node scripts/check-sitemap.js",
-  "check:llms": "node scripts/check-llms.js",
-  "check:search-index": "node scripts/check-search-index.js _site-org && node scripts/check-search-index.js _site-com",
-  "check:search-ranking": "node scripts/check-search-ranking.js",
-  "check:search-ui": "node scripts/check-search-ui.js",
-  "kpi:search": "node scripts/search-kpi/measure.js",
-  "kpi:search:worst": "node scripts/search-kpi/measure.js --worst 40",
-  "kpi:search:harvest": "node scripts/search-kpi/harvest-natural.js && node scripts/search-kpi/judge-natural.js",
-  "kpi:search:gen": "node scripts/search-kpi/gen-artificial.js --count 10000",
-  "gen-search-index": "node scripts/gen-search-index.js",
-  "check:jsonld": "node scripts/check-jsonld.js",
+    "check:llms": "node scripts/check-llms.js",
+    "check:search-index": "node scripts/check-search-index.js _site-org && node scripts/check-search-index.js _site-com",
+    "check:search-ranking": "node scripts/check-search-ranking.js",
+    "check:search-ui": "node scripts/check-search-ui.js",
+    "kpi:search": "node scripts/search-kpi/measure.js",
+    "kpi:search:worst": "node scripts/search-kpi/measure.js --worst 40",
+    "kpi:search:harvest": "node scripts/search-kpi/harvest-natural.js && node scripts/search-kpi/judge-natural.js",
+    "kpi:search:gen": "node scripts/search-kpi/gen-artificial.js --count 10000",
+    "gen-search-index": "node scripts/gen-search-index.js",
+    "check:jsonld": "node scripts/check-jsonld.js",
     "check:mermaid": "node scripts/check-mermaid.js",
-  "check:geo-live": "node scripts/check-geo-live.js",
+    "check:geo-live": "node scripts/check-geo-live.js",
     "check:api-versions": "node scripts/check-api-versions.js",
     "check:links": "npm run build:all && node scripts/check-links.js",
     "check:links:external": "npm run build:all && node scripts/check-links.js --external --allow-external=scripts/external-allowlist.txt",
@@ -70,7 +70,9 @@
     "indexnow:org": "node scripts/indexnow-ping.js org --exclude=/api/",
     "indexnow:com": "node scripts/indexnow-ping.js com",
     "gen-syndication": "node scripts/gen-syndication.js",
-    "gen-social": "node scripts/gen-social-preview.js"
+    "gen-social": "node scripts/gen-social-preview.js",
+    "kpi:recall": "node scripts/search-kpi/recall.js",
+    "kpi:compare": "node scripts/search-kpi/compare.js"
   },
   "repository": {
     "type": "git",

scripts/search-kpi/compare.js

@@ -0,0 +1,137 @@
+// compare.js — the working tree's ranker against another commit's, query by query.
+//
+//   node scripts/search-kpi/compare.js [--ref HEAD] [--dir _site-org] [--worst 40]
+//
+// WHY, and it is the whole reason this file exists: an aggregate can hold still while the results
+// churn underneath it, and it has hidden a real regression twice.
+//
+//   * A change that read -0.2 on natural macro turned out to be three named queries moving, one of
+//     them out of the top ten. The macro alone looked like rounding.
+//   * A change that read 0.0 on natural — genuinely unmoved — was simultaneously dropping 13
+//     artificial queries, two of them from #1 to #11 and #10 to #24. It was reverted on the
+//     strength of this list, and nothing in measure.js's output would have objected.
+//
+// So: never judge a ranker change by the summary. Read who won and who lost.
+//
+// Both rankers are loaded in ONE process over ONE prepared corpus, so the only variable is the
+// code. The baseline comes out of git rather than a hand-kept copy, which means it cannot drift
+// from what it claims to be.
+'use strict';
+
+const fs = require('node:fs');
+const os = require('node:os');
+const path = require('node:path');
+const { execFileSync } = require('node:child_process');
+
+const { load, page, accuracyFor } = require('./lib/harness');
+
+const ROOT = path.join(__dirname, '..', '..');
+const RANKER = path.join('src', '_shared', 'js', 'search.js');
+
+const arg = (name, fallback) => {
+  const i = process.argv.indexOf(name);
+
+  return i === -1 ? fallback : process.argv[i + 1];
+};
+
+const REF = arg('--ref', 'HEAD');
+const DIR = arg('--dir', path.join(ROOT, '_site-org'));
+const WORST = Number(arg('--worst', 40));
+
+function baselineRanker() {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'kpi-baseline-'));
+  const file = path.join(dir, 'search.js');
+
+  // maxBuffer: the ranker is ~100 kB and execFileSync's default is 1 MB, so this is comfortable —
+  // but it is the kind of limit that fails only after the file has grown, so it is stated.
+  const source = execFileSync('git', ['show', `${REF}:${RANKER}`], {
+    cwd: ROOT,
+    encoding: 'utf8',
+    maxBuffer: 8 * 1024 * 1024,
+  });
+
+  fs.writeFileSync(file, source);
+
+  return file;
+}
+
+function rankOf(ranker, query, expect) {
+  let hits;
+
+  try {
+    hits = ranker.search(ranker.parseQuery(query));
+  } catch {
+    return 0;
+  }
+
+  for (let i = 0; i < hits.length; i++) {
+    if (!hits[i].external && expect.includes(page(hits[i].record.u))) {
+      return i + 1;
+    }
+  }
+
+  return 0; // absent
+}
+
+function main() {
+  const baselineFile = baselineRanker();
+  // Order matters only in that both must see the same corpus; load() prepares it per ranker, so
+  // neither can be measured against a corpus the other did not have.
+  const before = load(DIR, baselineFile);
+  const after = load(DIR);
+
+  const readJson = (file) => JSON.parse(fs.readFileSync(path.join(__dirname, 'data', file), 'utf8'));
+  const sets = {
+    natural: readJson('natural-judged.json').judged,
+    artificial: readJson('artificial-queries.json').main,
+  };
+
+  let dirty = false;
+
+  for (const [name, cases] of Object.entries(sets)) {
+    const better = [];
+    const worse = [];
+
+    for (const testCase of cases) {
+      const expect = (Array.isArray(testCase.expect) ? testCase.expect : [testCase.expect])
+        .map(page);
+      const b = rankOf(before, testCase.query, expect);
+      const a = rankOf(after, testCase.query, expect);
+
+      if (accuracyFor(a) > accuracyFor(b)) better.push({ ...testCase, b, a });
+      if (accuracyFor(a) < accuracyFor(b)) worse.push({ ...testCase, b, a });
+    }
+
+    console.log(`\n=== ${name} (n = ${cases.length}) vs ${REF} ===`);
+    console.log(`better: ${better.length}   worse: ${worse.length}   `
+      + `unchanged: ${cases.length - better.length - worse.length}`);
+
+    if (!worse.length) {
+      continue;
+    }
+
+    dirty = true;
+
+    console.log(`\n  WORSE (before -> after, 0 = absent from the result set):`);
+
+    worse
+      .sort((x, y) => (accuracyFor(y.b) - accuracyFor(y.a)) - (accuracyFor(x.b) - accuracyFor(x.a)))
+      .slice(0, WORST)
+      .forEach((w) => console.log(`    ${String(w.b).padStart(3)} -> ${String(w.a).padStart(3)}  `
+        + `${w.query.slice(0, 52).padEnd(52)} [${w.label || w.bucket}]`));
+
+    if (worse.length > WORST) {
+      console.log(`    … and ${worse.length - WORST} more (pass --worst N)`);
+    }
+  }
+
+  // Not an exit failure: a change that trades some queries for more of others can still be right,
+  // and only a person can decide that. This is a flag, not a verdict.
+  if (dirty) {
+    console.log('\nSome queries got worse. Read them before keeping the change.');
+  } else {
+    console.log('\nNo query got worse.');
+  }
+}
+
+main();

scripts/search-kpi/lib/harness.js

@@ -23,14 +23,21 @@ const path = require('node:path');
 
 const ROOT = path.join(__dirname, '..', '..', '..');
 
-function load(dir) {
+// `rankerFile` measures a DIFFERENT copy of the ranker — a snapshot from another commit — which
+// is what compare.js uses to diff two versions in one process. Left undefined it measures the
+// working tree.
+function load(dir, rankerFile) {
   const indexDir = dir || path.join(ROOT, '_site-org');
-  const ranker = require(path.join(ROOT, 'src', '_shared', 'js', 'search.js'));
+  const ranker = require(rankerFile || path.join(ROOT, 'src', '_shared', 'js', 'search.js'));
 
-  const read = (name) => {
+  const read = (name, optional) => {
     const file = path.join(indexDir, name);
 
     if (!fs.existsSync(file)) {
+      if (optional) {
+        return null;
+      }
+
       throw new Error(`${file} is missing — run \`npm run build:org\` first`);
     }
 
@@ -40,6 +47,21 @@ function load(dir) {
   ranker.state.t1 = ranker.prepare(read('search-index.json'));
   ranker.state.t2 = ranker.prepareSections(read('search-text.json'));
 
+  // The PEER tiers, which this harness ignored for its first three months — and that omission is
+  // the plan's own biggest risk, not a detail. imqueue.com reaches the ranker only through these,
+  // so without them `pricing commercial license` scores against imqueue.org's `/license/` alone:
+  // a plausible answer, from the wrong edition, with nothing anywhere reporting a problem. The
+  // one question with revenue attached was the half being measured least.
+  //
+  // Optional, because a local `npm run build:org` produces no peer feed and that is a supported
+  // state — evaluate() already scores an `external` hit as a miss, so a measurement without them
+  // is pessimistic rather than wrong.
+  const peerIndex = read('search-peer-index.json', true);
+  const peerText = read('search-peer-text.json', true);
+
+  ranker.state.x1 = peerIndex ? ranker.prepare(peerIndex) : null;
+  ranker.state.x2 = peerText ? ranker.prepareSections(peerText) : null;
+
   return ranker;
 }
 
@@ -122,6 +144,11 @@ function summarise(results) {
     top1: (count((r) => r.position === 1) / total) * 100,
     top3: (count((r) => r.position >= 1 && r.position <= 3) / total) * 100,
     top5: (count((r) => r.position >= 1 && r.position <= 5) / total) * 100,
+    // recall@6 — the AGENT metric, and deliberately not position-decayed. search_docs returns six
+    // results and an agent reads all six, so whether the page is in the set is the whole question
+    // and its rank inside the set is noise. For a human the opposite holds, which is why accuracy
+    // above exists too. Same runs, two readers.
+    top6: (count((r) => r.position >= 1 && r.position <= 6) / total) * 100,
     top10: (count((r) => r.position >= 1 && r.position <= 10) / total) * 100,
     absent: (count((r) => r.position === 0) / total) * 100,
     mrr: mrr * 100,
@@ -148,6 +175,7 @@ function table(label, summary) {
     `  #1 exactly       ${pct(summary.top1)}`,
     `  in top 3         ${pct(summary.top3)}`,
     `  in top 5         ${pct(summary.top5)}`,
+    `  recall@6         ${pct(summary.top6)}`,
     `  in top 10        ${pct(summary.top10)}`,
     `  never found      ${pct(summary.absent)}`,
     `  MRR              ${pct(summary.mrr)}`,