From d4b95ce79b15b8fce53f41e452fab61036ce65f2 Mon Sep 17 00:00:00 2001
From: TheCryptoDonkey <TheCryptoDonkey@users.noreply.github.com>
Date: Wed, 23 Sep 2026 00:09:47 +0100
Subject: [PATCH 1/3] docs: design reviewer agreement probe with cost estimate

---
 .../reviewer-agreement-20260923/README.md     |  42 ++++
 .../reviewer-agreement-20260923/agree.mjs     | 238 ++++++++++++++++++
 .../reviewer-agreement-20260923/protocol.json |  42 ++++
 3 files changed, 322 insertions(+)
 create mode 100644 docs/experiments/reviewer-agreement-20260923/README.md
 create mode 100644 docs/experiments/reviewer-agreement-20260923/agree.mjs
 create mode 100644 docs/experiments/reviewer-agreement-20260923/protocol.json

diff --git a/docs/experiments/reviewer-agreement-20260923/README.md b/docs/experiments/reviewer-agreement-20260923/README.md
new file mode 100644
index 0000000..b0bedff
--- /dev/null
+++ b/docs/experiments/reviewer-agreement-20260923/README.md
@@ -0,0 +1,42 @@
+# Reviewer agreement probe (design, not yet run)
+
+The dimension rescore found that no arm separates and that one task's verdicts
+swing between runs, without being able to say whether the executor or the
+reviewer moved. This probe holds the answers fixed and re-reviews each of the
+80 recorded answers (v1, v2, S5, v3) three times under two conditions:
+
+- **replay**: the recorded reviewer prompt, byte for byte, with the recorded
+  schema;
+- **fixed**: the same prompt plus an explicit dimension list, enforced by the
+  schema, so every review reports the same dimension ids.
+
+No executor runs. Design, measures, decision rule and cost basis are in
+`protocol.json`, which is locked (`lockedAt`) before the first model call.
+
+```sh
+node docs/experiments/reviewer-agreement-20260923/agree.mjs \
+  --evidence v1=/private/v1 --evidence v2=/private/v2 \
+  --evidence s5=/private/s5 --evidence v3=/private/v3 \
+  --out /private/reviewer-agreement --estimate
+```
+
+Drop `--estimate` to run, add `--conditions replay` for the cheaper half, and
+use `--summarise` to rebuild the report from finished reviews. Finished
+reviews are never repeated, so an interrupted run resumes where it stopped. A
+provider or quota failure stops the probe without retry.
+
+## Cost estimate
+
+From the 80 recorded reviews (client-reported, Sonnet 5, high effort), with
+`--estimate` reproducing these figures:
+
+| Scope | Reviews | Client-reported cost | Input (uncached) | Output | Review time |
+|-------|--------:|---------------------:|-----------------:|-------:|------------:|
+| replay and fixed, 3 repeats | 480 | about $65 | 13.8M (7.6M) | 3.3M | 8.9 h serial, about 3 h at concurrency 3 |
+| replay only, 3 repeats | 240 | about $32 | 6.9M (3.8M) | 1.7M | 4.5 h serial, about 1.5 h |
+
+The client's cost is an API-price estimate; on a subscription the reviews draw
+on the Sonnet weekly quota instead. The pipeline was exercised end to end
+against a stub reviewer (no model calls) before this was committed.
+
+Results, when present, are in `RESULTS.md`.
diff --git a/docs/experiments/reviewer-agreement-20260923/agree.mjs b/docs/experiments/reviewer-agreement-20260923/agree.mjs
new file mode 100644
index 0000000..6d92eed
--- /dev/null
+++ b/docs/experiments/reviewer-agreement-20260923/agree.mjs
@@ -0,0 +1,238 @@
+#!/usr/bin/env node
+// Reviewer agreement probe: re-review recorded answers with the recorded
+// reviewer prompt (replay) and with a fixed dimension list (fixed), several
+// times each, and measure how often the verdicts agree. No executor runs.
+// Usage: node agree.mjs --evidence NAME=DIR [...] --out PRIVATE_DIR
+//          [--conditions replay,fixed] [--repeats 3] [--concurrency 3] [--estimate | --summarise]
+// --estimate prints the job count and a cost estimate from the recorded reviews without any model call.
+// --summarise recomputes the report from results already in --out. Finished reviews are never repeated.
+import { spawn, spawnSync } from 'node:child_process'
+import { createHash } from 'node:crypto'
+import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from 'node:fs'
+import { dirname, join, resolve } from 'node:path'
+import { fileURLToPath } from 'node:url'
+
+const here = dirname(fileURLToPath(import.meta.url))
+const packDir = resolve(here, '../d5-20260921')
+const harness = JSON.parse(readFileSync(resolve(here, '../graphify-20260922/protocol.json'), 'utf8'))
+const protocol = JSON.parse(readFileSync(join(here, 'protocol.json'), 'utf8'))
+const argv = process.argv.slice(2)
+const all = (key) => argv.flatMap((value, i) => (argv[i - 1] === `--${key}` ? [value] : []))
+const one = (key) => all(key)[0]
+const flag = (key) => argv.includes(`--${key}`)
+const runs = all('evidence').map((arg) => { const [name, dir] = arg.split('='); return { name, dir: resolve(dir) } })
+const out = one('out') && resolve(one('out'))
+const conditions = (one('conditions') ?? 'replay,fixed').split(',')
+const repeats = Number(one('repeats') ?? protocol.repeats)
+const concurrency = Number(one('concurrency') ?? 3)
+if (!runs.length || !out) throw new Error('usage: agree.mjs --evidence NAME=DIR [...] --out DIR')
+const sha = (text) => createHash('sha256').update(text).digest('hex')
+const arms = ['plain', 'graphify', 'context']
+
+// The recorded review schema from graphify-20260922/run.mjs.
+const reviewSchema = {
+  type: 'object',
+  properties: {
+    accepted: { type: 'boolean' },
+    dimensions: { type: 'array', items: { type: 'object', properties: { id: { type: 'string' }, pass: { type: 'boolean' }, note: { type: 'string' } }, required: ['id', 'pass', 'note'] } },
+    materialIssues: { type: 'array', items: { type: 'string' } },
+    summary: { type: 'string' },
+  },
+  required: ['accepted', 'dimensions', 'materialIssues', 'summary'],
+}
+
+function fixedIds(acceptance) {
+  if (acceptance.kind === 'structured') return Object.keys(acceptance.reviewerRubric)
+  return [...(acceptance.behaviour ?? []).map((_, i) => `b${i + 1}`), 'scope']
+}
+
+function fixedPrompt(recorded, ids) {
+  const lines = recorded.split('\n')
+  const last = lines.lastIndexOf('Return JSON only, matching the provided schema.')
+  if (last < 0) throw new Error('recorded prompt has no final schema line')
+  const block = [`Report exactly these dimensions, once each, in this order: ${ids.join(', ')}. Do not add, split, merge or rename dimensions; fold any other concern into the nearest listed dimension or into materialIssues.`, '']
+  return [...lines.slice(0, last), ...block, ...lines.slice(last)].join('\n')
+}
+
+function fixedSchema(ids) {
+  const schema = structuredClone(reviewSchema)
+  schema.properties.dimensions.items.properties.id = { type: 'string', enum: ids }
+  Object.assign(schema.properties.dimensions, { minItems: ids.length, maxItems: ids.length })
+  return schema
+}
+
+const cells = []
+for (const { name, dir } of runs) {
+  for (const task of readdirSync(dir).sort()) {
+    const acceptancePath = join(packDir, 'acceptance', `${task}.json`)
+    if (!existsSync(acceptancePath)) continue
+    const acceptance = JSON.parse(readFileSync(acceptancePath, 'utf8'))
+    for (const arm of arms) {
+      const armDir = join(dir, task, arm)
+      if (!existsSync(join(armDir, 'receipt.json')) || !existsSync(join(armDir, 'reviewer.prompt.txt'))) continue
+      const receipt = JSON.parse(readFileSync(join(armDir, 'receipt.json'), 'utf8'))
+      if (!receipt.reviewerRun?.verdict) continue
+      const prompt = readFileSync(join(armDir, 'reviewer.prompt.txt'), 'utf8')
+      const recordedSha = JSON.parse(readFileSync(join(armDir, 'reviewer.argv.json'), 'utf8')).promptSha256
+      if (sha(prompt) !== recordedSha) throw new Error(`${name}/${task}/${arm}: reviewer prompt does not match its recorded hash`)
+      cells.push({ run: name, task, arm, category: receipt.category, checkerPassed: Boolean(receipt.checker?.passed), acceptance, prompt,
+        recorded: { verdict: receipt.reviewerRun.verdict, accepted: Boolean(receipt.accepted), cost: receipt.reviewerRun.totalCostUsd, usage: receipt.reviewerRun.usage, seconds: receipt.reviewerRun.seconds } })
+    }
+  }
+}
+
+const jobs = cells.flatMap((cell) => conditions.flatMap((condition) => Array.from({ length: repeats }, (_, i) => ({ cell, condition, rep: i + 1 }))))
+const resultPath = (job) => join(out, job.condition, job.cell.run, job.cell.task, job.cell.arm, `r${job.rep}.json`)
+
+if (flag('estimate')) {
+  const recorded = (k) => cells.reduce((s, c) => s + (k(c) ?? 0), 0)
+  const perReview = { cost: recorded((c) => c.recorded.cost) / cells.length, seconds: recorded((c) => c.recorded.seconds) / cells.length,
+    input: recorded((c) => (c.recorded.usage?.input_tokens ?? 0) + (c.recorded.usage?.cache_creation_input_tokens ?? 0) + (c.recorded.usage?.cache_read_input_tokens ?? 0)) / cells.length,
+    uncached: recorded((c) => (c.recorded.usage?.input_tokens ?? 0) + (c.recorded.usage?.cache_creation_input_tokens ?? 0)) / cells.length,
+    output: recorded((c) => c.recorded.usage?.output_tokens) / cells.length }
+  const pending = jobs.filter((job) => !existsSync(resultPath(job)))
+  const byCost = pending.reduce((s, job) => s + (job.cell.recorded.cost ?? perReview.cost), 0)
+  process.stdout.write(`${JSON.stringify({ cells: cells.length, jobs: jobs.length, pending: pending.length, conditions, repeats, recordedPerReview: perReview,
+    estimate: { costUsd: Number(byCost.toFixed(2)), input: Math.round(perReview.input * pending.length), uncached: Math.round(perReview.uncached * pending.length),
+      output: Math.round(perReview.output * pending.length), serialHours: Number((perReview.seconds * pending.length / 3600).toFixed(1)),
+      hoursAtConcurrency: Number((perReview.seconds * pending.length / 3600 / concurrency).toFixed(1)), concurrency } }, null, 2)}\n`)
+  process.exit(0)
+}
+
+function cleanEnv() {
+  const env = { ...process.env }
+  for (const key of ['GEMINI_API_KEY', 'GOOGLE_API_KEY', 'OPENAI_API_KEY', 'ANTHROPIC_API_KEY', 'CLAUDE_CODE_SUBAGENT_MODEL']) delete env[key]
+  return env
+}
+
+const blocked = /rate.?limit|429|out_of_credits|usage limit|overloaded|authentication|401/i
+function review(dir, name, prompt, schema) {
+  const args = ['-p', '--model', protocol.reviewer.model, '--effort', protocol.reviewer.effort, '--output-format', 'json',
+    '--dangerously-skip-permissions', '--strict-mcp-config', '--mcp-config', '{"mcpServers":{}}', '--disable-slash-commands',
+    '--no-session-persistence', '--setting-sources', harness.executor.settingSources, '--settings', JSON.stringify(harness.executor.settingsOverride),
+    '--max-turns', String(protocol.reviewer.maxTurns), '--disallowedTools', [...harness.executor.disallowedTools, ...harness.reviewer.disallowedTools].join(','),
+    '--json-schema', JSON.stringify(schema)]
+  const started = Date.now()
+  return new Promise((resolveRun) => {
+    const child = spawn('claude', args, { cwd: dir, env: cleanEnv(), stdio: ['pipe', 'pipe', 'pipe'] })
+    const chunks = [], errors = []
+    child.stdout.on('data', (d) => chunks.push(d))
+    child.stderr.on('data', (d) => errors.push(d))
+    child.on('close', (code) => {
+      const raw = Buffer.concat(chunks).toString('utf8')
+      const stderr = Buffer.concat(errors).toString('utf8')
+      writeFileSync(join(dir, `${name}.result.json`), raw)
+      let parsed = null
+      try { parsed = JSON.parse(raw) } catch {}
+      let verdict = parsed?.structured_output ?? null
+      if (!verdict && typeof parsed?.result === 'string') { try { verdict = JSON.parse(parsed.result.replace(/^```json\s*|```\s*$/g, '')) } catch {} }
+      const failed = !parsed || (parsed.is_error && parsed.subtype !== 'error_max_turns') || blocked.test(stderr)
+      resolveRun({ code, failed, seconds: (Date.now() - started) / 1000, verdict, usage: parsed?.usage ?? null, cost: parsed?.total_cost_usd ?? null, subtype: parsed?.subtype ?? null, stderr: stderr.slice(0, 500) })
+    })
+    child.stdin.end(prompt)
+  })
+}
+
+if (!flag('summarise')) {
+  const clientVersion = spawnSync('claude', ['--version'], { encoding: 'utf8' }).stdout.trim()
+  mkdirSync(out, { recursive: true })
+  writeFileSync(join(out, 'run-meta.json'), JSON.stringify({ probeId: protocol.probeId, protocolSha256: sha(readFileSync(join(here, 'protocol.json'), 'utf8')), clientVersion, conditions, repeats, startedAt: new Date().toISOString() }, null, 2))
+  const pending = jobs.filter((job) => !existsSync(resultPath(job)))
+  let next = 0, done = 0, stop = null
+  async function worker() {
+    while (!stop && next < pending.length) {
+      const job = pending[next++]
+      const { cell, condition, rep } = job
+      const dir = dirname(resultPath(job))
+      mkdirSync(dir, { recursive: true })
+      const ids = fixedIds(cell.acceptance)
+      const prompt = condition === 'fixed' ? fixedPrompt(cell.prompt, ids) : cell.prompt
+      const schema = condition === 'fixed' ? fixedSchema(ids) : reviewSchema
+      const attempts = []
+      for (let attempt = 1; attempt <= 2 && !stop; attempt += 1) {
+        const run = await review(dir, `r${rep}-a${attempt}`, prompt, schema)
+        attempts.push(run)
+        if (run.failed) { stop = { job: `${condition}/${cell.run}/${cell.task}/${cell.arm}/r${rep}`, subtype: run.subtype, stderr: run.stderr }; break }
+        if (run.verdict) break
+      }
+      if (stop) break
+      const verdict = attempts.at(-1).verdict
+      writeFileSync(resultPath(job), JSON.stringify({ condition, run: cell.run, task: cell.task, arm: cell.arm, rep, promptSha256: sha(prompt), verdict,
+        accepted: cell.checkerPassed && verdict?.accepted === true && (verdict?.materialIssues?.length ?? 1) === 0,
+        attempts: attempts.map(({ subtype, seconds, usage, cost, verdict: v }) => ({ subtype, seconds, usage, cost, parsed: Boolean(v) })) }, null, 2))
+      done += 1
+      process.stderr.write(`${done}/${pending.length} ${condition} ${cell.run} ${cell.task} ${cell.arm} r${rep} accepted=${verdict?.accepted ?? 'unparsed'}\n`)
+    }
+  }
+  await Promise.all(Array.from({ length: concurrency }, worker))
+  if (stop) {
+    writeFileSync(join(out, 'stopped.json'), JSON.stringify({ at: new Date().toISOString(), ...stop }, null, 2))
+    process.stderr.write(`stopped without retry: ${JSON.stringify(stop)}\n`)
+    process.exit(2)
+  }
+}
+
+// Fleiss' kappa for binary ratings; rows are arrays of booleans of equal length.
+function fleiss(rows) {
+  const usable = rows.filter((r) => r.length >= 2 && r.every((v) => typeof v === 'boolean'))
+  if (!usable.length) return null
+  const n = usable[0].length
+  const same = usable.filter((r) => r.length === n)
+  const p = same.flat().filter(Boolean).length / (same.length * n)
+  const pe = p * p + (1 - p) * (1 - p)
+  const pbar = same.reduce((s, r) => { const yes = r.filter(Boolean).length; return s + (yes * (yes - 1) + (n - yes) * (n - yes - 1)) / (n * (n - 1)) }, 0) / same.length
+  return pe === 1 ? null : Number(((pbar - pe) / (1 - pe)).toFixed(3))
+}
+const pairwiseDisagreement = (rows) => {
+  const usable = rows.filter((r) => r.length >= 2)
+  if (!usable.length) return null
+  const d = usable.reduce((s, r) => { const yes = r.filter(Boolean).length, n = r.length; return s + (2 * yes * (n - yes)) / (n * (n - 1)) }, 0)
+  return Number((d / usable.length).toFixed(3))
+}
+
+const report = { probeId: protocol.probeId, meta: existsSync(join(out, 'run-meta.json')) ? JSON.parse(readFileSync(join(out, 'run-meta.json'), 'utf8')) : null, conditions: {} }
+for (const condition of conditions) {
+  const perCell = cells.map((cell) => {
+    const results = Array.from({ length: repeats }, (_, i) => resultPath({ cell, condition, rep: i + 1 })).filter(existsSync).map((p) => JSON.parse(readFileSync(p, 'utf8')))
+    return { cell, results, accepted: results.map((r) => r.accepted) }
+  }).filter((c) => c.results.length)
+  if (!perCell.length) continue
+  const acc = perCell.map((c) => c.accepted)
+  const byTask = {}
+  for (const c of perCell) {
+    const t = (byTask[c.cell.task] ??= { cells: 0, split: 0, dimensionCounts: {}, dims: {} })
+    t.cells += 1
+    if (new Set(c.accepted).size > 1) t.split += 1
+    for (const r of c.results) { const k = r.verdict?.dimensions?.length ?? 'unparsed'; t.dimensionCounts[k] = (t.dimensionCounts[k] ?? 0) + 1 }
+    for (const id of fixedIds(c.cell.acceptance)) (t.dims[id] ??= []).push(c.results.map((r) => r.verdict?.dimensions?.find((d) => d.id === id)?.pass))
+  }
+  for (const t of Object.values(byTask)) {
+    t.dims = Object.fromEntries(Object.entries(t.dims).map(([id, rows]) => {
+      const complete = rows.filter((r) => r.every((v) => typeof v === 'boolean'))
+      return [id, { reported: complete.length, of: rows.length, unanimous: complete.filter((r) => new Set(r).size === 1).length, kappa: fleiss(complete) }]
+    }))
+  }
+  const attempts = perCell.flatMap((c) => c.results.flatMap((r) => r.attempts))
+  const u = (k) => attempts.reduce((s, a) => s + (a.usage?.[k] ?? 0), 0)
+  const unchanged = perCell.filter((c) => ['v1', 'v2'].includes(c.cell.run) && c.cell.arm !== 'context')
+  const entry = {
+    cells: perCell.length, reviews: acc.flat().length,
+    acceptance: { unanimousCells: acc.filter((r) => new Set(r).size === 1).length, pairwiseDisagreement: pairwiseDisagreement(acc), kappa: fleiss(acc),
+      unchangedArmsV1V2: { answers: unchanged.length, pairwiseDisagreement: pairwiseDisagreement(unchanged.map((c) => c.accepted)) } },
+    byTask,
+    usage: { costUsd: Number(attempts.reduce((s, a) => s + (a.cost ?? 0), 0).toFixed(2)), input: u('input_tokens') + u('cache_creation_input_tokens') + u('cache_read_input_tokens'),
+      uncached: u('input_tokens') + u('cache_creation_input_tokens'), output: u('output_tokens'), seconds: Math.round(attempts.reduce((s, a) => s + a.seconds, 0)), unparsedAttempts: attempts.filter((a) => !a.parsed).length },
+  }
+  if (condition === 'replay') {
+    const withRecorded = perCell.map((c) => [c.cell.recorded.accepted, ...c.accepted])
+    entry.acceptance.withRecorded = { unanimousCells: withRecorded.filter((r) => new Set(r).size === 1).length, recordedOutvoted: perCell.filter((c) => c.accepted.filter((a) => a !== c.cell.recorded.accepted).length > c.accepted.length / 2).length }
+  }
+  report.conditions[condition] = entry
+}
+// Cross-run flips on unchanged arms: v1 against v2, recorded verdicts only.
+const recordedAccepted = new Map(cells.map((c) => [`${c.run}|${c.task}|${c.arm}`, c.recorded.accepted]))
+const pairs = cells.filter((c) => c.run === 'v1' && c.arm !== 'context' && recordedAccepted.has(`v2|${c.task}|${c.arm}`))
+report.crossRunV1V2UnchangedArms = { pairs: pairs.length, flipped: pairs.filter((c) => c.recorded.accepted !== recordedAccepted.get(`v2|${c.task}|${c.arm}`)).length }
+mkdirSync(out, { recursive: true })
+writeFileSync(join(out, 'agreement-summary.json'), JSON.stringify(report, null, 2))
+process.stdout.write(`${JSON.stringify(report, null, 2)}\n`)
diff --git a/docs/experiments/reviewer-agreement-20260923/protocol.json b/docs/experiments/reviewer-agreement-20260923/protocol.json
new file mode 100644
index 0000000..de62d96
--- /dev/null
+++ b/docs/experiments/reviewer-agreement-20260923/protocol.json
@@ -0,0 +1,42 @@
+{
+  "probeId": "reviewer-agreement-20260923",
+  "lockedAt": null,
+  "purpose": "Measure how often the blind acceptance reviewer returns the same verdict for the same recorded answer, so that reviewer variance can be separated from executor variance before any further executor run. No executor runs.",
+  "sample": {
+    "runs": ["v1", "v2", "s5", "v3"],
+    "cells": "every run/task/arm directory holding receipt.json with a reviewer verdict and reviewer.prompt.txt: 80 cells (v1, v2 and S5: 8 tasks x 3 arms; v3: 8 tasks x Context arm)",
+    "note": "S5 executors were DeepSeek V4 Pro but its reviewer was the same Sonnet 5 configuration, so its answers are valid review inputs."
+  },
+  "conditions": {
+    "replay": "The recorded reviewer.prompt.txt, byte for byte (SHA-256 checked against reviewer.argv.json), with the recorded review schema. Measures agreement under the conditions that produced every verdict so far.",
+    "fixed": "The same recorded prompt with one block inserted before its final line naming the exact dimension ids to report, once each, in order, and a schema that restricts ids to that list with exactly that many items. Structured tasks use the rubric ids; code tasks use b1..bN for the N required behaviours plus scope. Measures whether a fixed dimension list removes the dimension-set drift and reduces verdict churn."
+  },
+  "repeats": 3,
+  "reviewer": {
+    "client": "claude-code (version recorded per run; the recorded reviews used 2.1.278)",
+    "model": "claude-sonnet-5",
+    "effort": "high",
+    "maxTurns": 3,
+    "tools": "none: the executor and reviewer disallowed-tool lists of graphify-20260922/protocol.json, empty strict MCP config, agents-md plugin disabled, no session persistence",
+    "invocation": "Headless print mode with --output-format json in a per-cell output directory; the recorded reviews used stream-json in the arm directory. Neither affects what the model receives. The reviewer never had tools, so the recorded PATH setting is not reproduced.",
+    "retry": "one further attempt only when the verdict does not parse; a provider or client failure (rate limit, quota, authentication, overload) stops the whole probe without retry"
+  },
+  "acceptedRule": "As in the recorded runs: deterministic checker passed (taken from the receipt), verdict.accepted true and no material issues.",
+  "measures": {
+    "acceptance": "Per condition: share of cells whose three fresh verdicts agree; mean pairwise disagreement (probability two independent reviews of the same answer differ); Fleiss' kappa over cells. The replay condition also reports agreement of the three fresh verdicts with the recorded one.",
+    "dimensions": "Per condition and task: distribution of dimension counts; per dimension id, share of cells with unanimous pass/fail and pooled Fleiss' kappa.",
+    "attribution": "Cross-run disagreement between v1 and v2 on the arms whose tools and executor model did not change (plain and Graphify, 16 task/arm pairs, 5 flipped; the Context arm changed between runs and flipped once more) compared with the replay condition's mean pairwise disagreement on the same 32 answers.",
+    "cost": "Sum of client-reported total_cost_usd, input (uncached and cached), output and seconds per condition."
+  },
+  "decisionRule": {
+    "reviewerStable": "Replay condition: mean pairwise acceptance disagreement at most 0.10 and no task with more than one split cell. If met, whole-task acceptance from one review stays the measure for the repeated-run design.",
+    "fixedListAdopted": "If the replay condition is not stable and the fixed condition is (same thresholds), the fixed list replaces the recorded reviewer prompt in all later runs, and earlier runs are not re-scored with it.",
+    "neitherStable": "If neither condition is stable, whole-task acceptance from a single review is not used in later comparisons; the next step is a majority of three reviews or deterministic rubric checks, costed and approved separately.",
+    "attribution": "If replay pairwise disagreement on the 32 unchanged-arm answers is at least half the v1-v2 cross-run flip rate on those arms (5 of 16, 0.3125), reviewer variance is reported as a major share of the recorded churn."
+  },
+  "cost": {
+    "basis": "The 80 recorded reviews: client-reported $10.81 in total ($0.110 code change, $0.122 impact, $0.149 diagnosis, $0.159 orientation per review on average), 2.30M input of which 1.27M uncached, 0.55M output, 5,344 seconds.",
+    "estimate": "Both conditions x 3 repeats = 480 reviews: about $65 client-reported, 13.8M input (7.6M uncached), 3.3M output, about 8.9 hours of review time, about 3 hours at concurrency 3. Replay only: 240 reviews, about $32. Per-review cost can exceed the average when a verdict needs its second attempt.",
+    "billing": "total_cost_usd is the client's API-price estimate. On a subscription the reviews draw on the Sonnet weekly quota instead; an earlier acceptance run was blocked by that quota."
+  }
+}

From 2ccec2ce3de1c7504eaaf240e6d2b89864c61d76 Mon Sep 17 00:00:00 2001
From: TheCryptoDonkey <TheCryptoDonkey@users.noreply.github.com>
Date: Wed, 23 Sep 2026 00:25:47 +0100
Subject: [PATCH 2/3] docs: measure reviewer consistency from recorded reviews
 without model calls

---
 .../reviewer-agreement-20260923/README.md     |   2 +-
 .../reviewer-evidence-20260923/RESULTS.md     |  71 +++++++++++
 .../reviewer-evidence-20260923/analyse.mjs    | 114 ++++++++++++++++++
 3 files changed, 186 insertions(+), 1 deletion(-)
 create mode 100644 docs/experiments/reviewer-evidence-20260923/RESULTS.md
 create mode 100644 docs/experiments/reviewer-evidence-20260923/analyse.mjs

diff --git a/docs/experiments/reviewer-agreement-20260923/README.md b/docs/experiments/reviewer-agreement-20260923/README.md
index b0bedff..a0f9759 100644
--- a/docs/experiments/reviewer-agreement-20260923/README.md
+++ b/docs/experiments/reviewer-agreement-20260923/README.md
@@ -39,4 +39,4 @@ The client's cost is an API-price estimate; on a subscription the reviews draw
 on the Sonnet weekly quota instead. The pipeline was exercised end to end
 against a stub reviewer (no model calls) before this was committed.
 
-Results, when present, are in `RESULTS.md`.
+Results, when present, are in `RESULTS.md`. `../reviewer-evidence-20260923/RESULTS.md` already shows, without model calls, that at least three of the five v1 to v2 flips on unchanged arms came from the reviewer.
diff --git a/docs/experiments/reviewer-evidence-20260923/RESULTS.md b/docs/experiments/reviewer-evidence-20260923/RESULTS.md
new file mode 100644
index 0000000..d206c1c
--- /dev/null
+++ b/docs/experiments/reviewer-evidence-20260923/RESULTS.md
@@ -0,0 +1,71 @@
+# Reviewer consistency from recorded reviews (no model calls)
+
+Source: the 80 recorded reviews of v1, v2, S5 and v3 (Sonnet 5, high effort),
+their answers, diffs and deterministic checker results. `analyse.mjs` computes
+the mechanical measures; the side-by-side readings were made by hand from the
+same files. Nothing was re-reviewed.
+
+## The five flips on unchanged arms
+
+Between v1 and v2 the plain and Graphify arms did not change (same tools,
+same Sonnet 5 executor), and 5 of their 16 task verdicts flipped. Reading each
+pair side by side:
+
+| Task / arm | v1 to v2 | Answers | Cause |
+|------------|----------|---------|-------|
+| code-change-context / Graphify | rejected to accepted | same one-line code change, only the comment differs | reviewer |
+| diagnosis-context / plain | rejected to accepted | same claims on all five findings (repair "already present" at commit; test "kept") | reviewer: four dimensions judged oppositely |
+| orientation-context / plain | accepted to rejected | both omit "do not grant authority" and "generation-and-term bound" | reviewer: v1 notes the omissions and passes, v2 fails them |
+| diagnosis-kithmoot / plain | accepted to rejected | v1 names the unsafe cast (`as any`), v2 does not | executor; reviewer consistent |
+| diagnosis-kithmoot / Graphify | rejected to accepted | v2 names the cast (`as ContextVaultOptions`), v1 does not | executor; reviewer consistent |
+
+At least three of the five flips come from the reviewer judging equivalent
+answers differently, and two from real differences between answers. The
+reviewer is consistent where the rubric names a concrete, checkable element
+(the unsafe cast) and inconsistent where passing depends on whether a framing
+counts as equivalent ("re-verified before commit" against "generation made
+stale").
+
+## Identical code, different verdicts
+
+Nine of the ten code-change-context diffs make the same code change (delete
+the early `this.cursors.delete(options.cursor)`), differing only in a comment.
+All nine passed the deterministic checker, which runs the four behaviour
+probes. The reviewer accepted eight and rejected one (v1 Graphify) for
+material issues that apply equally to all nine ("no test file changes"). For
+the same code that is a disagreement probability of 0.22 between two reviews.
+All 20 code-change diffs passed the checker; the reviewer added no correct
+rejection on them.
+
+## Other mechanical signals
+
+- **Dimension ids.** 6 of 20 code-change reviews ignored the prompt's
+  `b1..bN` plus `scope` instruction (9 or 11 dimensions for 5 expected); 1 of
+  60 structured reviews departed from the rubric ids.
+- **Self-contradiction.** None: every verdict's `accepted` agrees with its
+  dimensions and material issues.
+- **Similar wording, different verdict.** For the same task dimension, pairs
+  of findings disagree 33 percent of the time at low word overlap (Jaccard
+  below 0.2) and still 12.5 percent at 0.5 or above (7 of 56 pairs). Examples
+  read by hand include orientation-context freshness (v2 Context passed by
+  crediting another finding; S5 Context, with nearly the same text, failed).
+- **Lexical rubric coverage** predicts the reviewer's pass only weakly
+  (AUC 0.69 over 320 dimension rows), so a word-overlap check cannot stand
+  in for the reviewer on structured tasks.
+
+## Consequences
+
+- Reviewer variance accounts for a large share of the recorded churn, so
+  single-review whole-task acceptance cannot distinguish arms that differ by
+  one or two tasks. The paid replay condition of
+  `reviewer-agreement-20260923` would mainly put a number on this; it is not
+  needed to establish it.
+- For code-change tasks the deterministic checker already carries the
+  behaviour; the reviewer's contribution on these 20 diffs was noise. Accepting
+  on the checker plus a deterministic scope check (only the expected files
+  changed, no test weakened) needs no model.
+- For structured tasks the unstable dimensions are those whose rubric text
+  admits more than one reading. Rewriting each such dimension as explicit,
+  checkable claims (as the diagnosis-kithmoot regression dimension already
+  is) is the model-free step; whether it steadies the reviewer can only be
+  measured by re-reviewing.
diff --git a/docs/experiments/reviewer-evidence-20260923/analyse.mjs b/docs/experiments/reviewer-evidence-20260923/analyse.mjs
new file mode 100644
index 0000000..6013ded
--- /dev/null
+++ b/docs/experiments/reviewer-evidence-20260923/analyse.mjs
@@ -0,0 +1,114 @@
+#!/usr/bin/env node
+// What the recorded reviews show about reviewer consistency, without any model call.
+// Usage: node analyse.mjs NAME=DIR [...] > report.json
+import { createHash } from 'node:crypto'
+import { existsSync, readFileSync, readdirSync } from 'node:fs'
+import { dirname, join, resolve } from 'node:path'
+import { fileURLToPath } from 'node:url'
+
+const packDir = resolve(dirname(fileURLToPath(import.meta.url)), '../d5-20260921')
+const runs = process.argv.slice(2).map((arg) => arg.split('='))
+if (!runs.length) throw new Error('usage: analyse.mjs NAME=DIR [...]')
+const read = (p) => JSON.parse(readFileSync(p, 'utf8'))
+const stop = new Set('a an the and or of to in on for is are be by as at it its that this with from not no must before after when than then only same any which into can'.split(' '))
+const words = (text) => new Set((text ?? '').toLowerCase().replace(/([a-z])([A-Z])/g, '$1 $2').split(/[^a-z0-9]+/).filter((w) => w.length > 2 && !stop.has(w)).map((w) => w.replace(/(ing|ed|es|s)$/, '')))
+const jaccard = (a, b) => { const i = [...a].filter((w) => b.has(w)).length; return a.size + b.size ? i / (a.size + b.size - i) : 1 }
+const coverage = (rubric, text) => { const r = words(rubric), t = words(text); return r.size ? [...r].filter((w) => t.has(w)).length / r.size : null }
+const mean = (xs) => (xs.length ? xs.reduce((s, x) => s + x, 0) / xs.length : null)
+const round = (x) => (x === null ? null : Number(x.toFixed(3)))
+
+const cells = []
+for (const [run, dir] of runs) {
+  for (const task of readdirSync(dir).sort()) {
+    const ap = join(packDir, 'acceptance', `${task}.json`)
+    if (!existsSync(ap)) continue
+    const acceptance = read(ap)
+    for (const arm of ['plain', 'graphify', 'context']) {
+      const armDir = join(dir, task, arm)
+      if (!existsSync(join(armDir, 'receipt.json'))) continue
+      const receipt = read(join(armDir, 'receipt.json'))
+      const verdict = receipt.reviewerRun?.verdict
+      if (!verdict) continue
+      let answer = null
+      try { answer = read(join(armDir, 'workspace', 'answer.json')) } catch {}
+      const diff = existsSync(join(armDir, 'diff.patch')) ? readFileSync(join(armDir, 'diff.patch'), 'utf8') : null
+      cells.push({ run, task, arm, kind: acceptance.kind, acceptance, verdict, answer, diff, checker: Boolean(receipt.checker?.passed), accepted: Boolean(receipt.accepted) })
+    }
+  }
+}
+
+// 1. Verdicts that contradict themselves.
+const selfInconsistent = cells.filter((c) => {
+  const allPass = c.verdict.dimensions.every((d) => d.pass)
+  return (c.verdict.accepted && (!allPass || c.verdict.materialIssues.length)) || (!c.verdict.accepted && allPass && !c.verdict.materialIssues.length)
+}).map((c) => ({ run: c.run, task: c.task, arm: c.arm, accepted: c.verdict.accepted, failed: c.verdict.dimensions.filter((d) => !d.pass).map((d) => d.id), materialIssues: c.verdict.materialIssues.length }))
+
+// 2. Dimension ids reported against the ids the rubric defines.
+const expectedIds = (c) => (c.kind === 'structured' ? Object.keys(c.acceptance.reviewerRubric) : [...c.acceptance.behaviour.map((_, i) => `b${i + 1}`), 'scope'])
+const idDrift = cells.map((c) => { const exp = expectedIds(c), got = c.verdict.dimensions.map((d) => d.id); return { cell: `${c.run}/${c.task}/${c.arm}`, kind: c.kind, expected: exp.length, reported: got.length, missing: exp.filter((i) => !got.includes(i)), extra: got.filter((i) => !exp.includes(i)) } }).filter((d) => d.missing.length || d.extra.length)
+
+// 3. Code tasks: the same diff reviewed more than once.
+const norm = (d) => (d ?? '').split('\n').filter((l) => /^[+-]/.test(l) && !/^(\+\+\+|---)/.test(l)).map((l) => l.replace(/\s+/g, ' ').trim()).join('\n')
+const byDiff = {}
+for (const c of cells.filter((c) => c.kind === 'code')) (byDiff[`${c.task}|${createHash('sha256').update(norm(c.diff)).digest('hex').slice(0, 12)}`] ??= []).push(c)
+const sameDiff = Object.entries(byDiff).filter(([, cs]) => cs.length > 1).map(([key, cs]) => ({ key, cells: cs.map((c) => `${c.run}/${c.arm}`), checker: cs.map((c) => c.checker), accepted: cs.map((c) => c.accepted), dims: cs.map((c) => c.verdict.dimensions.length) }))
+const codePairs = []
+const code = cells.filter((c) => c.kind === 'code')
+for (let i = 0; i < code.length; i += 1) for (let j = i + 1; j < code.length; j += 1) {
+  const a = code[i], b = code[j]
+  if (a.task !== b.task || !a.checker || !b.checker) continue
+  codePairs.push({ sim: jaccard(words(norm(a.diff)), words(norm(b.diff))), agree: a.accepted === b.accepted, a: `${a.run}/${a.arm}`, b: `${b.run}/${b.arm}`, task: a.task })
+}
+
+// 4. Structured tasks, per dimension: rubric-term coverage of the finding against the reviewer's pass,
+//    and pairs of findings for the same dimension that are near-identical in wording yet judged differently.
+const rows = []
+for (const c of cells.filter((c) => c.kind === 'structured' && c.answer)) {
+  for (const [id, rubric] of Object.entries(c.acceptance.reviewerRubric)) {
+    const finding = c.answer.findings?.find((f) => f.id === id)?.value ?? ''
+    const judged = c.verdict.dimensions.find((d) => d.id === id)
+    if (!judged) continue
+    rows.push({ cell: `${c.run}/${c.task}/${c.arm}`, task: c.task, id, pass: judged.pass, note: judged.note, cov: coverage(rubric, finding), finding, words: words(finding) })
+  }
+}
+function auc(xs) {
+  const pos = xs.filter((r) => r.pass), neg = xs.filter((r) => !r.pass)
+  if (!pos.length || !neg.length) return null
+  let w = 0
+  for (const p of pos) for (const n of neg) w += p.cov > n.cov ? 1 : p.cov === n.cov ? 0.5 : 0
+  return w / (pos.length * neg.length)
+}
+const byDim = {}
+for (const r of rows) (byDim[`${r.task}|${r.id}`] ??= []).push(r)
+const dimensions = Object.entries(byDim).map(([k, rs]) => ({ dim: k, n: rs.length, passed: rs.filter((r) => r.pass).length, coverageAuc: round(auc(rs)),
+  meanCovPass: round(mean(rs.filter((r) => r.pass).map((r) => r.cov))), meanCovFail: round(mean(rs.filter((r) => !r.pass).map((r) => r.cov))) }))
+const pairs = []
+for (const rs of Object.values(byDim)) for (let i = 0; i < rs.length; i += 1) for (let j = i + 1; j < rs.length; j += 1) pairs.push({ sim: jaccard(rs[i].words, rs[j].words), agree: rs[i].pass === rs[j].pass, a: rs[i], b: rs[j] })
+const band = (lo, hi) => { const p = pairs.filter((x) => x.sim >= lo && x.sim < hi); return { band: `${lo}-${hi}`, pairs: p.length, disagree: p.filter((x) => !x.agree).length, rate: round(p.length ? p.filter((x) => !x.agree).length / p.length : null) } }
+const similarDisagree = pairs.filter((p) => !p.agree && p.sim >= 0.5).sort((x, y) => y.sim - x.sim).slice(0, 12)
+  .map((p) => ({ dim: `${p.a.task}|${p.a.id}`, sim: round(p.sim), pass: p.a.pass ? p.a.cell : p.b.cell, fail: p.a.pass ? p.b.cell : p.a.cell, failNote: (p.a.pass ? p.b : p.a).note.slice(0, 300) }))
+
+// 5. Unchanged arms v1 against v2: for each flipped verdict, which dimensions moved and how similar the findings were.
+const get = (run, task, arm) => cells.find((c) => c.run === run && c.task === task && c.arm === arm)
+const flips = []
+for (const a of cells.filter((c) => c.run === 'v1' && c.arm !== 'context')) {
+  const b = get('v2', a.task, a.arm)
+  if (!b) continue
+  const moved = a.kind === 'structured'
+    ? Object.keys(a.acceptance.reviewerRubric).map((id) => {
+        const pa = a.verdict.dimensions.find((d) => d.id === id)?.pass, pb = b.verdict.dimensions.find((d) => d.id === id)?.pass
+        const fa = a.answer?.findings?.find((f) => f.id === id)?.value ?? '', fb = b.answer?.findings?.find((f) => f.id === id)?.value ?? ''
+        return { id, v1: pa, v2: pb, findingSim: round(jaccard(words(fa), words(fb))), covV1: round(coverage(a.acceptance.reviewerRubric[id], fa)), covV2: round(coverage(a.acceptance.reviewerRubric[id], fb)) }
+      }).filter((m) => m.v1 !== m.v2)
+    : { diffSim: round(jaccard(words(norm(a.diff)), words(norm(b.diff)))), checker: [a.checker, b.checker], dims: [a.verdict.dimensions.map((d) => `${d.id}:${d.pass}`), b.verdict.dimensions.map((d) => `${d.id}:${d.pass}`)], issues: [a.verdict.materialIssues, b.verdict.materialIssues] }
+  flips.push({ task: a.task, arm: a.arm, v1: a.accepted, v2: b.accepted, flipped: a.accepted !== b.accepted, moved })
+}
+
+process.stdout.write(`${JSON.stringify({
+  cells: cells.length,
+  selfInconsistent,
+  idDrift: { cells: idDrift.length, byKind: { code: idDrift.filter((d) => d.kind === 'code').length, structured: idDrift.filter((d) => d.kind === 'structured').length }, examples: idDrift.slice(0, 8) },
+  code: { sameDiff, pairsCheckerPassed: codePairs.length, highSimilarityDisagree: codePairs.filter((p) => p.sim >= 0.8 && !p.agree) },
+  structured: { dimensionRows: rows.length, overallCoverageAuc: round(auc(rows)), dimensions, similarityBands: [band(0, 0.2), band(0.2, 0.35), band(0.35, 0.5), band(0.5, 1.01)], similarDisagree },
+  unchangedArmsV1V2: flips,
+}, null, 1)}\n`)

From a14cd95d2272843e0cde4098e58f15939a899488 Mon Sep 17 00:00:00 2001
From: TheCryptoDonkey <TheCryptoDonkey@users.noreply.github.com>
Date: Wed, 23 Sep 2026 00:28:54 +0100
Subject: [PATCH 3/3] feat: accept code-change tasks on checker plus
 deterministic scope check

---
 docs/experiments/REPEATED-RUNS.md             |  3 +
 .../code-acceptance-20260923/README.md        | 46 ++++++++++++++
 .../code-acceptance-20260923/rescore.mjs      | 37 +++++++++++
 .../code-acceptance-20260923/scope.mjs        | 62 +++++++++++++++++++
 docs/experiments/graphify-20260922/run.mjs    | 14 ++++-
 .../graphify-20260922/summarise.mjs           |  4 +-
 6 files changed, 162 insertions(+), 4 deletions(-)
 create mode 100644 docs/experiments/code-acceptance-20260923/README.md
 create mode 100644 docs/experiments/code-acceptance-20260923/rescore.mjs
 create mode 100644 docs/experiments/code-acceptance-20260923/scope.mjs

diff --git a/docs/experiments/REPEATED-RUNS.md b/docs/experiments/REPEATED-RUNS.md
index 55cadb6..5d3d2c0 100644
--- a/docs/experiments/REPEATED-RUNS.md
+++ b/docs/experiments/REPEATED-RUNS.md
@@ -17,6 +17,9 @@ the changed tool was never called. Every later comparison uses this design.
 - **Executor.** One model per run, recorded with effort and client version.
   The next run uses `claude-opus-5-5`, with the reviewer unchanged
   (`claude-sonnet-5`, high) so verdicts stay comparable with earlier runs.
+- **Code-change acceptance.** Checker plus the deterministic scope check
+  (`"codeAcceptance": "checker-and-scope"`, see `code-acceptance-20260923/`);
+  the model reviewer judges structured answers only.
 - **Arms.** Plain tools, Graphify, and Context at its current build, which
   includes `repository_explore` and `repository_coverage` with the exact-quote
   check. A category checklist in the instructions, if tested, is a fourth arm
diff --git a/docs/experiments/code-acceptance-20260923/README.md b/docs/experiments/code-acceptance-20260923/README.md
new file mode 100644
index 0000000..669c427
--- /dev/null
+++ b/docs/experiments/code-acceptance-20260923/README.md
@@ -0,0 +1,46 @@
+# Code-change acceptance without a model reviewer
+
+On code-change tasks the pack's checker (`d5-20260921/accept.mjs`) already
+builds, runs the focused tests and probes each required behaviour. The recorded
+reviews added nothing correct on top: all 20 recorded code-change diffs passed
+the checker, and the reviewer's one rejection was of a diff whose code is
+identical to eight it accepted (`../reviewer-evidence-20260923/RESULTS.md`).
+
+`scope.mjs` replaces the reviewer's `scope` dimension with a deterministic
+check of the workspace against its base commit:
+
+- every changed path is inside the task's selection policy;
+- no package manifest, lockfile, TypeScript, Vite, Vitest or Jest config,
+  `.gitignore`, `.npmrc` or `.github/` file changed;
+- no test file deleted, no test declaration removed, no `skip`, `only`, `todo`,
+  `xit` or `xdescribe` added;
+- at least one file changed.
+
+Assertion counts are reported but not judged: a behaviour change legitimately
+rewrites assertions (S5 Graphify on code-change-kithmoot replaced two
+assertions of the old behaviour with one looped assertion over seven invalid
+inputs), and the checker runs its own behaviour probes regardless of the
+agent's tests.
+
+A task is accepted when the checker and the scope check both pass. The harness
+uses this rule only when a protocol sets `"codeAcceptance": "checker-and-scope"`;
+the locked v1, v2, S5 and v3 protocols do not, so their runs replay unchanged.
+
+## Recorded runs re-scored
+
+`node rescore.mjs v1=DIR v2=DIR s5=DIR v3=DIR` (no model calls):
+
+| Run | Arms | Recorded accepted | Checker and scope |
+|-----|------|------------------:|------------------:|
+| v1 | plain, Graphify, Context | 2, 1, 2 of 2 | 2, 2, 2 of 2 |
+| v2 | plain, Graphify, Context | 2, 2, 2 of 2 | 2, 2, 2 of 2 |
+| S5 | plain, Graphify, Context | 2, 2, 2 of 2 | 2, 2, 2 of 2 |
+| v3 | Context | 2 of 2 | 2 of 2 |
+
+The only change is v1 Graphify on code-change-context, rejected by the
+reviewer for the same code the other eight arms were accepted for. All 20
+cells pass the scope check. The rule would have avoided 20 reviews: 1,199
+seconds and $2.20 client-reported.
+
+These two tasks discriminate no arm under either rule; they measure cost, not
+acceptance.
diff --git a/docs/experiments/code-acceptance-20260923/rescore.mjs b/docs/experiments/code-acceptance-20260923/rescore.mjs
new file mode 100644
index 0000000..d534136
--- /dev/null
+++ b/docs/experiments/code-acceptance-20260923/rescore.mjs
@@ -0,0 +1,37 @@
+#!/usr/bin/env node
+// Re-score recorded code-change cells with checker plus scope check instead of the model reviewer.
+// No model calls. Usage: node rescore.mjs NAME=DIR [...]
+import { existsSync, readFileSync, readdirSync } from 'node:fs'
+import { dirname, join, resolve } from 'node:path'
+import { fileURLToPath } from 'node:url'
+import { checkScope } from './scope.mjs'
+
+const packDir = resolve(dirname(fileURLToPath(import.meta.url)), '../d5-20260921')
+const runs = process.argv.slice(2).map((arg) => arg.split('='))
+if (!runs.length) throw new Error('usage: rescore.mjs NAME=DIR [...]')
+const cells = []
+for (const [run, dir] of runs) {
+  for (const taskId of readdirSync(dir).sort()) {
+    const taskPath = join(packDir, 'tasks', `${taskId}.json`)
+    if (!existsSync(taskPath)) continue
+    const task = JSON.parse(readFileSync(taskPath, 'utf8'))
+    if (task.answerSchema) continue
+    for (const arm of ['plain', 'graphify', 'context']) {
+      const armDir = join(dir, taskId, arm)
+      if (!existsSync(join(armDir, 'receipt.json'))) continue
+      const receipt = JSON.parse(readFileSync(join(armDir, 'receipt.json'), 'utf8'))
+      const scope = checkScope({ workspace: join(armDir, 'workspace'), include: task.selectionPolicy.include })
+      const checker = Boolean(receipt.checker?.passed)
+      cells.push({ run, task: taskId, arm, checker, scope: scope.passed, scopeReasons: scope.reasons, recordedAccepted: Boolean(receipt.accepted), accepted: checker && scope.passed,
+        reviewerSeconds: receipt.reviewerRun?.seconds ?? null, reviewerCostUsd: receipt.reviewerRun?.totalCostUsd ?? null })
+    }
+  }
+}
+const changed = cells.filter((c) => c.accepted !== c.recordedAccepted)
+const byRun = {}
+for (const c of cells) {
+  const r = (byRun[`${c.run}|${c.arm}`] ??= { run: c.run, arm: c.arm, tasks: 0, recordedAccepted: 0, accepted: 0 })
+  r.tasks += 1; r.recordedAccepted += Number(c.recordedAccepted); r.accepted += Number(c.accepted)
+}
+const sum = (k) => Number(cells.reduce((s, c) => s + (c[k] ?? 0), 0).toFixed(2))
+process.stdout.write(`${JSON.stringify({ cells: cells.length, changed, byRunAndArm: Object.values(byRun), reviewerAvoided: { seconds: sum('reviewerSeconds'), costUsd: sum('reviewerCostUsd') }, all: cells }, null, 1)}\n`)
diff --git a/docs/experiments/code-acceptance-20260923/scope.mjs b/docs/experiments/code-acceptance-20260923/scope.mjs
new file mode 100644
index 0000000..07b7ef1
--- /dev/null
+++ b/docs/experiments/code-acceptance-20260923/scope.mjs
@@ -0,0 +1,62 @@
+#!/usr/bin/env node
+// Deterministic scope check for code-change tasks: the agent's changes stay inside the task's
+// selection policy, leave build and test configuration alone, and do not delete or disable tests.
+// Assertion counts are reported but not judged: a behaviour change legitimately rewrites assertions
+// (one looped expect can replace several), and the pack checker runs its own behaviour probes.
+// With the pack's behaviour checker it replaces the model reviewer for code tasks.
+// Usage: node scope.mjs --workspace DIR --task ID    (prints the result; exit 1 when it fails)
+import { spawnSync } from 'node:child_process'
+import { readFileSync } from 'node:fs'
+import { dirname, join, resolve } from 'node:path'
+import { fileURLToPath } from 'node:url'
+
+const ignored = [/^graphify-out\//, /^answer\.json$/]
+const protectedConfig = /(^|\/)(package(-lock)?\.json|npm-shrinkwrap\.json|tsconfig[^/]*\.json|(vitest|vite|jest)\.config\.[cm]?[jt]s|\.gitignore|\.npmrc)$|^\.github\//
+const testFile = /\.(test|spec)\.[cm]?[jt]sx?$|(^|\/)(test|tests|__tests__)\//
+const declarations = /\b(it|test)(\.each\([^)]*\))?\s*\(/g
+const assertions = /\bexpect\s*\(|\bassert(\.\w+)?\s*\(/g
+const disabled = /\b(it|test|describe)\.(skip|only|todo)\s*\(|\bx(it|describe)\s*\(/g
+
+function git(workspace, args, env) {
+  const r = spawnSync('git', args, { cwd: workspace, encoding: 'utf8', env, maxBuffer: 64 * 1024 * 1024 })
+  if (r.status !== 0) throw new Error(`git ${args.join(' ')} failed: ${r.stderr}`)
+  return r.stdout
+}
+const count = (text, pattern) => (text.match(pattern) ?? []).length
+
+export function checkScope({ workspace, include, env = process.env }) {
+  const reasons = []
+  const changes = git(workspace, ['status', '--porcelain=v1', '-z', '-uall'], env).split('\0').filter(Boolean)
+  const changed = []
+  for (let i = 0; i < changes.length; i += 1) {
+    const status = changes[i].slice(0, 2), path = changes[i].slice(3)
+    // With -z a rename or copy is followed by its source path.
+    const from = status.includes('R') || status.includes('C') ? changes[(i += 1)] : path
+    if (!ignored.some((re) => re.test(path))) changed.push({ status, path, from })
+  }
+  const tests = []
+  for (const { status, path, from } of changed) {
+    if (!include.some((prefix) => path === prefix || path.startsWith(`${prefix.replace(/\/$/, '')}/`))) reasons.push(`outside selection policy: ${path}`)
+    if (protectedConfig.test(path)) reasons.push(`build or test configuration changed: ${path}`)
+    if (!testFile.test(path)) continue
+    if (status.includes('D')) { reasons.push(`test file deleted: ${path}`); continue }
+    const before = status.includes('?') || status.includes('A') ? '' : git(workspace, ['show', `HEAD:${from}`], env)
+    const after = readFileSync(join(workspace, path), 'utf8')
+    const t = { path, tests: [count(before, declarations), count(after, declarations)], assertions: [count(before, assertions), count(after, assertions)], disabled: [count(before, disabled), count(after, disabled)] }
+    tests.push(t)
+    if (t.tests[1] < t.tests[0]) reasons.push(`fewer tests in ${path}: ${t.tests[0]} to ${t.tests[1]}`)
+    if (t.disabled[1] > t.disabled[0]) reasons.push(`skip, only or todo added in ${path}`)
+  }
+  if (!changed.length) reasons.push('no source change')
+  return { passed: reasons.length === 0, reasons, changed, tests }
+}
+
+if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
+  const arg = (key) => { const i = process.argv.indexOf(`--${key}`); return i >= 0 ? process.argv[i + 1] : undefined }
+  if (!arg('workspace') || !arg('task')) throw new Error('usage: scope.mjs --workspace DIR --task ID')
+  const packDir = resolve(dirname(fileURLToPath(import.meta.url)), '../d5-20260921')
+  const task = JSON.parse(readFileSync(join(packDir, 'tasks', `${arg('task')}.json`), 'utf8'))
+  const result = checkScope({ workspace: resolve(arg('workspace')), include: task.selectionPolicy.include })
+  process.stdout.write(`${JSON.stringify(result, null, 2)}\n`)
+  process.exit(result.passed ? 0 : 1)
+}
diff --git a/docs/experiments/graphify-20260922/run.mjs b/docs/experiments/graphify-20260922/run.mjs
index 99bb085..66ac050 100644
--- a/docs/experiments/graphify-20260922/run.mjs
+++ b/docs/experiments/graphify-20260922/run.mjs
@@ -2,12 +2,14 @@
 // Three-way retrieval comparison runner: plain tools, Graphify, Context.
 // Usage: node run.mjs --local /private/local.json [--protocol DIR] [--task ID | --all] [--arms plain,graphify,context] [--skip-review]
 // --protocol selects a directory holding protocol.json and context-instructions.txt (default: this directory).
+// protocol.json may set "codeAcceptance": "checker-and-scope" to accept code tasks without the model reviewer.
 // local.json (private, machine-specific): { evidence, roots: { context, kithmoot }, node, contextCli, graphifyBin, graphifyAlwaysOn }
 import { spawn, spawnSync } from 'node:child_process'
 import { createHash } from 'node:crypto'
 import { appendFileSync, existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'
 import { dirname, join, resolve } from 'node:path'
 import { fileURLToPath } from 'node:url'
+import { checkScope } from '../code-acceptance-20260923/scope.mjs'
 
 const here = dirname(fileURLToPath(import.meta.url))
 const packDir = join(here, '..', 'd5-20260921')
@@ -310,7 +312,14 @@ async function runArm({ taskId, arm, orderIndex, local, evidence, skipReview })
   log(evidence, `${taskId}/${arm}: checker ${checker.passed ? 'passed' : 'failed'}`)
   write()
 
-  if (!skipReview) {
+  // protocol.codeAcceptance 'checker-and-scope' (optional) accepts code tasks on the behaviour checker plus a
+  // deterministic scope check, with no model reviewer; protocols without it keep the reviewer.
+  const deterministicCode = acceptance.kind === 'code' && protocol.codeAcceptance === 'checker-and-scope'
+  if (deterministicCode) {
+    receipt.scope = checkScope({ workspace, include: task.selectionPolicy.include, env: { ...cleanEnv(), ...gitEnv } })
+    log(evidence, `${taskId}/${arm}: scope ${receipt.scope.passed ? 'passed' : `failed (${receipt.scope.reasons.join('; ')})`}`)
+  }
+  if (!skipReview && !deterministicCode) {
     const rprompt = reviewerPrompt({ task, acceptance, workspace, answer, checker, diff })
     const attempts = []
     for (let attempt = 1; attempt <= 2; attempt += 1) {
@@ -326,7 +335,8 @@ async function runArm({ taskId, arm, orderIndex, local, evidence, skipReview })
     const last = attempts[attempts.length - 1]
     receipt.reviewerRun = { ...last, attempts: attempts.length, seconds: attempts.reduce((a, r) => a + r.seconds, 0), inputTotal: attempts.reduce((a, r) => a + (r.inputTotal ?? 0), 0), output: attempts.reduce((a, r) => a + (r.output ?? 0), 0), allAttempts: attempts.map(r => ({ subtype: r.subtype, inputTotal: r.inputTotal, output: r.output, seconds: r.seconds, parsed: Boolean(r.verdict) })) }
   }
-  receipt.accepted = checker.passed && receipt.reviewerRun?.verdict?.accepted === true && (receipt.reviewerRun?.verdict?.materialIssues?.length ?? 1) === 0
+  receipt.acceptanceRule = deterministicCode ? 'checker-and-scope' : 'checker-and-reviewer'
+  receipt.accepted = deterministicCode ? checker.passed && receipt.scope.passed : checker.passed && receipt.reviewerRun?.verdict?.accepted === true && (receipt.reviewerRun?.verdict?.materialIssues?.length ?? 1) === 0
   receipt.armSeconds = (receipt.setup.graphify?.seconds ?? 0) + exec.seconds + checker.seconds + (receipt.reviewerRun?.seconds ?? 0)
   receipt.finishedAt = now()
   writeFileSync(receiptPath, JSON.stringify(receipt, null, 2))
diff --git a/docs/experiments/graphify-20260922/summarise.mjs b/docs/experiments/graphify-20260922/summarise.mjs
index 2ae18fb..4a8036e 100644
--- a/docs/experiments/graphify-20260922/summarise.mjs
+++ b/docs/experiments/graphify-20260922/summarise.mjs
@@ -16,7 +16,7 @@ for (const task of readdirSync(evidence, { withFileTypes: true }).filter(d => d.
     const r = JSON.parse(readFileSync(path, 'utf8'))
     const e = r.executorRun ?? {}, v = r.reviewerRun ?? {}
     rows.push({
-      task, arm, order: r.orderIndex, accepted: r.accepted, checker: r.checker?.passed ?? null, reviewer: v.verdict?.accepted ?? null,
+      task, arm, order: r.orderIndex, accepted: r.accepted, checker: r.checker?.passed ?? null, reviewer: v.verdict?.accepted ?? null, scope: r.scope ? r.scope.passed : null,
       subtype: e.subtype, turns: e.numTurns, toolCalls: e.toolCallsTotal, toolCallsByName: e.toolCalls,
       inputTotal: e.inputTotal, inputUncached: e.inputUncached, cacheRead: e.usage?.cache_read_input_tokens ?? null, output: e.output,
       costUsd: e.totalCostUsd, executorSeconds: e.seconds, reviewerInput: v.inputTotal, reviewerOutput: v.output, reviewerSeconds: v.seconds,
@@ -26,7 +26,7 @@ for (const task of readdirSync(evidence, { withFileTypes: true }).filter(d => d.
 }
 const fmt = (n) => n === null || n === undefined ? '?' : typeof n === 'number' ? (Number.isInteger(n) ? n.toLocaleString('en-GB') : n.toFixed(1)) : String(n)
 const lines = ['| Task | Arm | Order | Accepted | Checker | Reviewer | Turns | Tool calls | Input total | Uncached input | Output | Cost est. USD | Executor s | Reviewer in/out | Arm s |', '| --- | --- | ---: | --- | --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |']
-for (const r of rows) lines.push(`| ${r.task} | ${r.arm} | ${r.order} | ${r.accepted ? 'yes' : 'no'} | ${r.checker ? 'pass' : 'fail'} | ${r.reviewer === null ? '?' : r.reviewer ? 'accept' : 'reject'} | ${fmt(r.turns)} | ${fmt(r.toolCalls)} | ${fmt(r.inputTotal)} | ${fmt(r.inputUncached)} | ${fmt(r.output)} | ${r.costUsd === null ? '?' : r.costUsd.toFixed(2)} | ${fmt(r.executorSeconds)} | ${fmt(r.reviewerInput)}/${fmt(r.reviewerOutput)} | ${fmt(r.armSeconds)} |`)
+for (const r of rows) lines.push(`| ${r.task} | ${r.arm} | ${r.order} | ${r.accepted ? 'yes' : 'no'} | ${r.checker ? 'pass' : 'fail'} | ${r.scope !== null ? (r.scope ? 'scope ok' : 'scope fail') : r.reviewer === null ? '?' : r.reviewer ? 'accept' : 'reject'} | ${fmt(r.turns)} | ${fmt(r.toolCalls)} | ${fmt(r.inputTotal)} | ${fmt(r.inputUncached)} | ${fmt(r.output)} | ${r.costUsd === null ? '?' : r.costUsd.toFixed(2)} | ${fmt(r.executorSeconds)} | ${fmt(r.reviewerInput)}/${fmt(r.reviewerOutput)} | ${fmt(r.armSeconds)} |`)
 const agg = {}
 for (const arm of arms) {
   const set = rows.filter(r => r.arm === arm)