From c0e8bc8ffd7945159190eb9c0fa3fae1d3b0ec45 Mon Sep 17 00:00:00 2001 From: TheCryptoDonkey Date: Wed, 23 Sep 2026 07:26:48 +0100 Subject: [PATCH 1/2] docs: lock a single-arm cost screen of Context 0.4.0 on DeepSeek V4 Pro --- .../context-instructions.txt | 9 + .../screen-pro-040-20260923/protocol.json | 187 ++++++++++++++++++ .../screen-pro-040-20260923/screen.mjs | 34 ++++ .../screen-pro-040-20260923/summarise.mjs | 68 +++++++ 4 files changed, 298 insertions(+) create mode 100644 docs/experiments/screen-pro-040-20260923/context-instructions.txt create mode 100644 docs/experiments/screen-pro-040-20260923/protocol.json create mode 100644 docs/experiments/screen-pro-040-20260923/screen.mjs create mode 100644 docs/experiments/screen-pro-040-20260923/summarise.mjs diff --git a/docs/experiments/screen-pro-040-20260923/context-instructions.txt b/docs/experiments/screen-pro-040-20260923/context-instructions.txt new file mode 100644 index 0000000..5f4a971 --- /dev/null +++ b/docs/experiments/screen-pro-040-20260923/context-instructions.txt @@ -0,0 +1,9 @@ +Use the configured z1p-repository tools for source discovery. First check that +repository_status.root is this exact checkout or worktree; stop using a +mismatched binding. Explore each symbol once, fetch the blocks you rely on with +repository_packet and cite those lines. Run repository_coverage on the draft and +fix what it reports. Read files directly only for evidence the tools cannot +supply; tiny known-file edits need no scan. Refresh and re-fetch after edits, +branch switches, pulls, merges or rebases; reconnect after changing the binding +or server build. A shell directory change does not retarget Context. Keep the +project's existing models, tests and review standards. diff --git a/docs/experiments/screen-pro-040-20260923/protocol.json b/docs/experiments/screen-pro-040-20260923/protocol.json new file mode 100644 index 0000000..f6a673b --- /dev/null +++ b/docs/experiments/screen-pro-040-20260923/protocol.json @@ -0,0 +1,187 @@ +{ + "version": 1, + "experimentId": "context-040-screen-deepseek-v4-pro-20260923", + "lockedAt": "2026-09-23T06:26:15.145Z", + "shellPathRule": "The client's Bash tool rebuilds PATH from the login profile, so every arm receives PATH through the settings env: the Node 24 bin directory plus the runner's PATH. The graphify arm additionally prepends the isolated virtualenv bin so `graphify` resolves. Verified by a probe before lock.", + "purpose": "A single-arm cost screen: the Context arm alone on the 0.4.0 release build, on the eight locked tasks with three repetitions, compared against the plain and Graphify cells recorded by repeated-three-way-deepseek-v4-pro-20260923 under the same executor, instructions and task pack. It asks whether the first-use build, whole-file packets and range merging remove the extra turns that run measured. Structured answers are not reviewed (no reviewer spend); code tasks are still accepted by checker and scope.", + "taskPack": { + "directory": "../d5-20260921", + "reused": [ + "tasks/*.json", + "setup/*.patch", + "acceptance/*.json", + "prepare-arm.mjs", + "accept.mjs" + ], + "note": "The eight locked v1 task definitions, seeded patches, frozen archives and deterministic checker are reused unchanged. v1, v2 and v3 results are not pooled with this experiment." + }, + "tasks": [ + "orientation-context", + "orientation-kithmoot", + "diagnosis-context", + "diagnosis-kithmoot", + "impact-context", + "impact-kithmoot", + "code-change-context", + "code-change-kithmoot" + ], + "retrievalInstructionRule": "Each arm's task prompt begins with one retrieval instruction of similar length and force, as the locked v1 measurement definition prescribed. Pilot 3 showed that the tools' own always-on text alone was ignored by the model in every arm, which measures nothing about the tools.", + "arms": { + "plain": { + "description": "Built-in Read, Edit, Write and Bash only. No MCP servers, no Graphify on PATH.", + "systemPromptAppendix": "Retrieval: use the built-in Read, Grep, Glob and Bash tools for source discovery.", + "retrievalInstruction": "you must use only bounded grep or rg and exact file reads for source discovery, confined to the task selection policy." + }, + "graphify": { + "description": "Built-in tools plus a Graphify graph prebuilt in graphify-out/ with `graphify update ` (no model, no API key) and the graphify CLI on PATH. The system prompt appendix is Graphify's own always-on instruction text (always_on/claude-md.md) verbatim. graphify-out/ is hidden from Git through an environment-supplied core.excludesFile so no hashed file changes.", + "retrievalInstruction": "a Graphify knowledge graph is prebuilt in graphify-out/ and the graphify CLI is on PATH. Before any grep, find, cat or file read you must run `graphify query \"\"` and `graphify explain \"\"` (and `graphify path \"\" \"\"` for relationships) for the concepts in this task, then read only the file locations they return, confined to the task selection policy.", + "package": "graphifyy", + "packageVersion": "0.9.65", + "build": "graphify update ", + "instructionSource": "graphify/always_on/claude-md.md", + "instructionSha256": "5a599e9f148ebbf0a8786c4508c3954daf95d0d74b88223d890dbf37aad23ec6", + "graphOutExcludedFromGit": true + }, + "context": { + "description": "Built-in tools plus the z1p-repository stdio MCP server (repository_status, repository_refresh, repository_explore, repository_coverage, repository_search, repository_packet) bound to the arm workspace, with the task selection policy in .z1p-navigation.json. Search and packet responses render as compact text; repository_explore answers one symbol per call. The system prompt appendix is the project instruction text from docs/GETTING-STARTED.md step 3 verbatim (context-instructions.txt). repository_coverage is a deterministic pre-submit check that lists explored files a draft answer leaves uncited.", + "package": "@forgesworn/context-tools", + "packageVersion": "0.4.0 release build (tag v0.4.0, e581a40), identified by dist digests below", + "server": "encrypted-context.mjs navigate ", + "instructionSource": "docs/GETTING-STARTED.md step 3 (context-instructions.txt in this directory)", + "retrievalInstruction": "the z1p-repository MCP tools are bound to this workspace. Before any grep, find, cat or file read you must call repository_status (and repository_refresh if it is unavailable, stale or unknown), then call repository_explore for each identifier in this task, use repository_search with pathPrefix only for literals, and obtain the source and tests you cite with repository_packet using the current expectedGeneration, reading files directly only for evidence those tools cannot supply, confined to the task selection policy. Before writing answer.json, call repository_coverage with your draft answer and the task's identifiers, then address each missing file or state why it does not bear on the task.", + "instructionSha256": "cb7f696ea84c42f5923cfc0470a5416b1b6053f4df29bb162e80bb8795b9f934", + "build": { + "headCommit": "e581a40", + "workingTreeClean": true, + "dist": { + "repository-coverage.js": "45416989a1e158b70409665d49813f7cc6c122514db6fb4b33f359041fd63ae9", + "repository-explore.js": "54269956eee2297c8318c5fd430c2bdd0417ac16298027b52ce8840fbb9e3249", + "repository-navigation.js": "1e11b6bed5d18e35ed1045c2914579948aa1c960dbd76a9c5aebb1cf2abca893", + "repository-navigation-mcp.js": "0e8bf94b635c651f06920af9f3c7ef8dee7750a8c09477b361b55e0df8569643", + "source-packet.mjs": "d0b61caa608a1306c1e3a59b2a0d39ce45a3b6b6b6ff6bfbd204f4e94954e4ad" + } + } + } + }, + "armOrders": [ + [ + "plain", + "graphify", + "context" + ], + [ + "graphify", + "context", + "plain" + ], + [ + "context", + "plain", + "graphify" + ] + ], + "armOrderRule": "Repetition r of task i runs the Context arm alone, recording the position it held in armOrders[(i + r - 1) mod 3] with --order-index.", + "executor": { + "client": "claude-code", + "clientVersion": "2.1.280", + "model": "deepseek-v4-pro:cloud", + "effort": "medium", + "maxTurns": 80, + "maxBudgetUsd": null, + "settingSources": "project", + "settingsOverride": { + "enabledPlugins": { + "agents-md@builtin": false + } + }, + "settingsOverrideReason": "A probe showed the built-in agents-md plugin injects the frozen tree's AGENTS.md into every arm; the Context revision's AGENTS.md instructs use of z1p-repository tools, so it is disabled for all arms. A second probe confirmed no project instruction text remains in context.", + "disallowedTools": [ + "WebFetch", + "WebSearch", + "Agent", + "ToolSearch", + "Workflow", + "EnterWorktree", + "ExitWorktree", + "RemoteTrigger", + "CronCreate", + "CronDelete", + "CronList", + "ScheduleWakeup", + "SendMessage", + "PushNotification", + "Monitor", + "DesignSync", + "ListAgents", + "TaskStop", + "NotebookEdit" + ], + "builtInToolsObserved": "This Claude Code build exposes Bash, Read, Edit and Write for source work; Grep and Glob are not separate tools, so plain-arm search runs through Bash.", + "permissions": "dangerously-skip-permissions inside a disposable frozen workspace", + "skills": "disabled", + "mcp": "strict; only the arm's --mcp-config", + "sessionPersistence": false, + "promptDelivery": "identical task prompt on stdin for every arm; the only difference is the arm's system prompt appendix, PATH and MCP config", + "modelRoute": "Claude Code 2.1.280 with ANTHROPIC_BASE_URL pointing at the local Ollama 0.34.2 daemon (Anthropic-compatible /v1/messages), which proxies the cloud tag to ollama.com; verified by a smoke test with Bash and MCP tool calls", + "env": { + "ANTHROPIC_BASE_URL": "http://127.0.0.1:11435", + "ANTHROPIC_API_KEY": "ollama", + "ANTHROPIC_AUTH_TOKEN": "ollama" + }, + "costNote": "The client reports a list-price estimate for an unrecognised model; it is meaningless here and no budget cap is applied. Provider usage counters (input, cache read, output) are recorded; Ollama Cloud credit consumption is read from the account, not from receipts.", + "effortNote": "--effort medium is accepted by the client; whether the route honours it is unknown. The smoke test reported zero thinking tokens." + }, + "reviewer": { + "client": "claude-code", + "model": "claude-sonnet-5", + "effort": "high", + "maxTurns": 3, + "disallowedTools": [ + "Bash", + "Read", + "Edit", + "Write", + "Grep", + "Glob", + "TodoWrite" + ], + "retry": "one further attempt if the structured verdict does not parse; both attempts are recorded and their usage and time are summed", + "tools": "none (Bash, Read, Edit, Write, Grep and Glob disallowed as well); receives the answer or diff, the private rubric, the deterministic checker result and bounded excerpts of the cited frozen source", + "blind": "does not see the arm name or any other arm's output", + "skipped": "structured answers are not reviewed in this screen (--skip-review); acceptance is known for the code tasks only" + }, + "measures": { + "accepted": "code tasks: deterministic checker and scope check passed (no model reviewer); structured tasks: checker passed and the blind reviewer accepted against rubric version 3 with no material issue", + "executorToolCalls": "count of tool_use blocks by tool name from the stream", + "executorInputTokens": "input_tokens + cache_creation_input_tokens + cache_read_input_tokens from the result usage; uncached = input_tokens + cache_creation_input_tokens", + "executorOutputTokens": "output_tokens from the result usage", + "costEstimateUsd": "Claude Code total_cost_usd, a list-price estimate, not subscription billing", + "executorSeconds": "wall time of the headless session", + "armSeconds": "graphify build (arm-specific setup) + executor + checker + reviewer wall time", + "reviewerTokens": "reviewer session usage", + "setupSeconds": "dependency installation, shared by all arms and excluded", + "orderIndex": "position of the arm in its rotated order, passed to run.mjs with --order-index" + }, + "decisionRule": "Screen, on this locked set only, cost only. For each task, the ratio of the 0.4.0 Context arm's median executor input (three repetitions) to each recorded comparator's median. The screen passes against a comparator if the geometric mean of the eight ratios is at most 0.8 and the upper end of a 90 percent bootstrap interval (resampling repetitions within task and arm, 10,000 draws, seed 20260923) is below 1. Also reported: the same ratio against the earlier Context cells, median turns, tool errors and code-task acceptance. Passing is a screen for a later held-out comparison, not a saving: acceptance of structured answers is not measured, and the tasks were used to find the fixes.", + "knownAsymmetries": [ + "Graphify indexes the whole frozen workspace; Context indexes only the task selection policy. Graphify is not disadvantaged by this.", + "Graphify's arm-specific build time is counted in armSeconds; Context's refresh happens inside the session and is counted as a tool call.", + "Arm instructions are delivered by system prompt appendix rather than a CLAUDE.md file so the frozen tree stays identical across arms.", + "Four tasks ask about the navigation code the Context arm runs. Its tools print local-source-unsigned in their metadata and its instruction names the status states, both of which bear on orientation-context answers. Results are reported for the Context-repository and KithMoot tasks separately.", + "The Context instruction (the product's documented project instruction) is longer than the other arms' instructions.", + "Comparator cells come from an earlier run on the same day; provider-side variation between runs is not controlled.", + "The fixes were found on this task set, so a pass may overstate the effect on other tasks." + ], + "derivedFrom": "repeated-three-way-deepseek-v4-pro-20260923", + "codeAcceptance": "checker-and-scope", + "rubricDir": "../rubric-v3-20260923", + "repetitions": 3, + "failureRule": "The driver stops at the first executor, checker or reviewer failure that is not a verdict (provider error, quota, refusal, non-success result) and does not retry; finished cells are kept and a rerun resumes at the first cell without receipt.json.", + "armsRun": [ + "context" + ], + "comparators": { + "evidence": "the plain and graphify cells of repeated-three-way-deepseek-v4-pro-20260923 attempt 3 (RESULTS.md in ../repeated-pro-20260923/)", + "note": "Comparator cells are reused, not rerun; they ran on the same executor, client version and day. The Context arm's own earlier cells are reported alongside for the before and after comparison." + } +} diff --git a/docs/experiments/screen-pro-040-20260923/screen.mjs b/docs/experiments/screen-pro-040-20260923/screen.mjs new file mode 100644 index 0000000..db6ce74 --- /dev/null +++ b/docs/experiments/screen-pro-040-20260923/screen.mjs @@ -0,0 +1,34 @@ +#!/usr/bin/env node +// Runs the Context arm alone for every repetition, without structured review. +// Resumes by skipping cells with receipt.json; stops at the first failed run. +// Usage: node screen.mjs --local /private/local.json (local.evidence is the parent; repetitions go in rep1..repN) +import { spawnSync } from 'node:child_process' +import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs' +import { dirname, join, resolve } from 'node:path' +import { fileURLToPath } from 'node:url' + +const here = dirname(fileURLToPath(import.meta.url)) +const protocol = JSON.parse(readFileSync(join(here, 'protocol.json'), 'utf8')) +const flag = process.argv.indexOf('--local') +if (flag < 0) throw new Error('usage: screen.mjs --local /private/local.json') +const local = JSON.parse(readFileSync(resolve(process.argv[flag + 1]), 'utf8')) +const failed = (run) => run && !['success', 'error_max_turns'].includes(run.subtype) + +for (let rep = 1; rep <= protocol.repetitions; rep += 1) { + const evidence = join(local.evidence, `rep${rep}`) + mkdirSync(evidence, { recursive: true }) + const repLocal = join(evidence, 'local.json') + writeFileSync(repLocal, JSON.stringify({ ...local, evidence }, null, 2)) + protocol.tasks.forEach((task, i) => { + const position = protocol.armOrders[(i + rep - 1) % protocol.armOrders.length].indexOf('context') + 1 + const receipt = join(evidence, task, 'context', 'receipt.json') + if (!existsSync(receipt)) { + const r = spawnSync(process.execPath, [join(here, '../graphify-20260922/run.mjs'), '--local', repLocal, '--protocol', here, '--task', task, '--arms', 'context', '--order-index', String(position), '--skip-review'], { stdio: 'inherit' }) + if (r.status !== 0) { console.log(`STOPPED rep${rep} ${task}: run.mjs exited ${r.status}`); process.exit(1) } + } + const done = JSON.parse(readFileSync(receipt, 'utf8')) + if (failed(done.executorRun)) { console.log(`STOPPED rep${rep} ${task}: executor did not finish successfully`); process.exit(1) } + console.log(`CELL rep${rep} ${task}/context input=${done.executorRun.inputTotal} turns=${done.executorRun.numTurns}`) + }) +} +console.log('SCREEN-DONE') diff --git a/docs/experiments/screen-pro-040-20260923/summarise.mjs b/docs/experiments/screen-pro-040-20260923/summarise.mjs new file mode 100644 index 0000000..42b304f --- /dev/null +++ b/docs/experiments/screen-pro-040-20260923/summarise.mjs @@ -0,0 +1,68 @@ +#!/usr/bin/env node +// Applies the screen rule. Written before the first screen cell ran. +// Usage: node summarise.mjs +import { existsSync, readFileSync } from 'node:fs' +import { dirname, join } from 'node:path' +import { fileURLToPath } from 'node:url' + +const here = dirname(fileURLToPath(import.meta.url)) +const protocol = JSON.parse(readFileSync(join(here, 'protocol.json'), 'utf8')) +const [screen, baseline] = process.argv.slice(2) +if (!screen || !baseline) throw new Error('usage: summarise.mjs ') +const reps = Array.from({ length: protocol.repetitions }, (_, i) => i + 1) +const median = (xs) => { const s = [...xs].sort((a, b) => a - b); const m = s.length >> 1; return s.length % 2 ? s[m] : (s[m - 1] + s[m]) / 2 } +const geomean = (xs) => Math.exp(xs.reduce((a, x) => a + Math.log(x), 0) / xs.length) +function rng(seed) { let a = seed >>> 0; return () => { a = (a + 0x6d2b79f5) >>> 0; let t = a; t = Math.imul(t ^ (t >>> 15), t | 1); t ^= t + Math.imul(t ^ (t >>> 7), t | 61); return ((t ^ (t >>> 14)) >>> 0) / 4294967296 } } +function errors(stream) { + let n = 0; let total = 0 + for (const line of readFileSync(stream, 'utf8').split('\n')) { + if (!line.trim()) continue + const e = JSON.parse(line) + if (e.type !== 'user' || !Array.isArray(e.message?.content)) continue + for (const p of e.message.content) if (p.type === 'tool_result') { total += 1; if (p.is_error) n += 1 } + } + return [n, total] +} +const load = (parent, task, arm) => reps.map((rep) => { + const dir = join(parent, `rep${rep}`, task, arm) + if (!existsSync(join(dir, 'receipt.json'))) return null + const r = JSON.parse(readFileSync(join(dir, 'receipt.json'), 'utf8')) + return { input: r.executorRun.inputTotal, turns: r.executorRun.numTurns, accepted: r.accepted, errors: errors(join(dir, 'executor.stream.jsonl')) } +}) +const cells = {} +for (const task of protocol.tasks) { + cells[`${task}/new`] = load(screen, task, 'context') + for (const arm of ['context', 'plain', 'graphify']) cells[`${task}/${arm}`] = load(baseline, task, arm) +} +const complete = Object.values(cells).every((runs) => runs.every(Boolean)) +const med = (task, g, runs = cells[`${task}/${g}`]) => median(runs.filter(Boolean).map((c) => c.input)) +const ratio = (g, pick = med) => geomean(protocol.tasks.map((t) => pick(t, 'new') / pick(t, g))) +function bootstrap(g, draws = 10000, seed = 20260923) { + const random = rng(seed); const values = [] + for (let i = 0; i < draws; i += 1) { + const s = {} + for (const t of protocol.tasks) for (const k of ['new', g]) { const ok = cells[`${t}/${k}`].filter(Boolean); s[`${t}/${k}`] = ok.map(() => ok[Math.floor(random() * ok.length)]) } + values.push(ratio(g, (t, k) => median(s[`${t}/${k}`].map((c) => c.input)))) + } + values.sort((a, b) => a - b) + return [values[Math.floor(0.05 * draws)], values[Math.floor(0.95 * draws) - 1]] +} +const out = [`Cells complete: ${complete ? 'yes' : 'no'}\n`, '| Task | 0.4.0 median input | earlier Context | plain | graphify | 0.4.0 median turns | earlier Context turns | 0.4.0 tool errors | 0.4.0 code accepted |', '| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |'] +const fmt = (n) => Math.round(n).toLocaleString('en-GB') +for (const t of protocol.tasks) { + const n = cells[`${t}/new`].filter(Boolean) + if (!n.length) continue + const e = n.reduce((a, c) => [a[0] + c.errors[0], a[1] + c.errors[1]], [0, 0]) + out.push(`| ${t} | ${fmt(med(t, 'new'))} | ${fmt(med(t, 'context'))} | ${fmt(med(t, 'plain'))} | ${fmt(med(t, 'graphify'))} | ${median(n.map((c) => c.turns))} | ${median(cells[`${t}/context`].map((c) => c.turns))} | ${e[0]} / ${e[1]} | ${t.startsWith('code-change') ? `${n.filter((c) => c.accepted).length} of ${n.length}` : 'not reviewed'} |`) +} +if (complete) { + out.push('\n| Against | Geometric mean ratio (0.4.0 / against) | 90% interval | Screen |', '| --- | ---: | --- | --- |') + const verdict = [] + for (const g of ['plain', 'graphify', 'context']) { + const r = ratio(g); const [lo, hi] = bootstrap(g); const pass = r <= 0.8 && hi < 1 + if (g !== 'context') verdict.push(pass) + out.push(`| ${g === 'context' ? 'earlier Context (reported only)' : g} | ${r.toFixed(3)} | ${lo.toFixed(3)} to ${hi.toFixed(3)} | ${g === 'context' ? '-' : pass ? 'passes' : 'does not pass'} |`) + } + out.push(`\nScreen: ${verdict.every(Boolean) ? 'passes against both comparators' : 'does not pass against both comparators'}.`) +} +process.stdout.write(`${out.join('\n')}\n`) From c873a3bb7c4fdb8c180fa2b9b1703aff4c65bd0d Mon Sep 17 00:00:00 2001 From: TheCryptoDonkey Date: Wed, 23 Sep 2026 09:25:01 +0100 Subject: [PATCH 2/2] docs: record the Context 0.4.0 cost screen result (does not pass) --- .../screen-pro-040-20260923/RESULTS.md | 65 +++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 docs/experiments/screen-pro-040-20260923/RESULTS.md diff --git a/docs/experiments/screen-pro-040-20260923/RESULTS.md b/docs/experiments/screen-pro-040-20260923/RESULTS.md new file mode 100644 index 0000000..398d468 --- /dev/null +++ b/docs/experiments/screen-pro-040-20260923/RESULTS.md @@ -0,0 +1,65 @@ +# Context 0.4.0 cost screen on a DeepSeek V4 Pro executor + +Locked protocol `context-040-screen-deepseek-v4-pro-20260923` (locked +2026-09-23T06:26:15Z), run 06:27 to 08:24 UTC on 23 September 2026 with +Claude Code 2.1.280. The Context arm alone ran the 0.4.0 release build +(tag v0.4.0, e581a40, dist digests in `protocol.json`) on the eight locked +tasks, three repetitions: 24 cells, all completed, no provider failure, no +retry. The plain and Graphify comparators are the recorded cells of +`../repeated-pro-20260923/`. Structured answers were not reviewed; code tasks +were accepted by checker and scope. `summarise.mjs`, committed before the +first cell, applies the decision rule. + +## Result: the screen does not pass + +| Against | Geometric mean input ratio, 0.4.0 / comparator | 90% interval | Screen | +| --- | ---: | --- | --- | +| plain | 1.086 | 0.831 to 1.271 | does not pass | +| graphify | 1.219 | 1.007 to 1.536 | does not pass | +| earlier Context (reported only) | 0.929 | 0.831 to 1.237 | - | + +The rule needed at most 0.8 with the interval's upper end below 1. On this +executor and task set, 0.4.0 is not cheaper than plain or Graphify. Against the +earlier Context cells it is slightly lower in the point estimate, with an +interval that includes no change. + +## Per task + +| Task | 0.4.0 median input | Earlier Context | Plain | Graphify | 0.4.0 median turns | Earlier Context turns | 0.4.0 tool errors | Code accepted | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| orientation-context | 380,269 | 250,133 | 395,186 | 144,836 | 22 | 19 | 3 / 62 | not reviewed | +| orientation-kithmoot | 303,587 | 329,316 | 502,732 | 316,911 | 22 | 24 | 1 / 62 | not reviewed | +| diagnosis-context | 4,380,206 | 2,509,745 | 1,434,506 | 2,332,533 | 74 | 43 | 5 / 215 | not reviewed | +| diagnosis-kithmoot | 201,232 | 204,273 | 236,355 | 228,206 | 15 | 18 | 0 / 45 | not reviewed | +| impact-context | 279,439 | 277,023 | 366,147 | 412,812 | 19 | 23 | 1 / 55 | not reviewed | +| impact-kithmoot | 137,355 | 185,361 | 174,419 | 116,087 | 16 | 20 | 0 / 44 | not reviewed | +| code-change-context | 480,117 | 1,535,828 | 407,972 | 437,707 | 22 | 42 | 2 / 64 | 3 of 3 | +| code-change-kithmoot | 333,332 | 339,157 | 184,583 | 250,162 | 24 | 26 | 1 / 72 | 3 of 3 | + +Executor input across all 24 cells was 19,911,272 (earlier Context arm: +15,198,553). Input totals include cache reads; the route's cache accounting is +inconsistent, so only totals are compared. Credit consumption was not read. + +## What changed and what did not + +- Tool errors fell from 29 of 578 tool results in the earlier Context arm to + 13 of 619. +- Median turns fell on six of eight tasks. code-change-context fell from 42 to + 22 turns and from 1.54M to 0.48M median input, and all six code cells were + accepted. +- diagnosis-context rose to 4.90M, 4.20M and 4.38M input (80, 70 and 74 + turns; the first hit the 80-turn cap and still returned success). Those + sessions used the navigation tools (10 to 14 packets, up to 6 explores) and + then read files directly as well (up to 24 `Read` calls), so the tools added + turns rather than replacing reads. This task alone accounts for 13.5M of the + 19.9M total and keeps the ratio against plain above 1. +- The KithMoot tasks, which do not ask about Context's own code, were at or + below the earlier Context arm on every task, but still above plain on the + code-change task. + +## Limits + +Structured answers were not reviewed, so acceptance is unknown for six tasks. +Comparator cells come from an earlier run on the same day. The fixes were found +on this task set. A pass would only have justified a held-out comparison; the +failure means 0.4.0 shows no measured cost benefit on this executor.