From 4b27cfc287553eaed5a71d197b8bba6dddad690a Mon Sep 17 00:00:00 2001 From: TheCryptoDonkey Date: Wed, 23 Sep 2026 01:21:51 +0100 Subject: [PATCH 1/4] docs: rubric version 3 limits required points to what each prompt asks --- .../context-instructions.txt | 9 + .../repeated-pro-20260923/protocol.json | 188 ++++++++++++++++++ .../repeated-pro-20260923/repeat.mjs | 38 ++++ docs/experiments/rubric-v3-20260923/README.md | 40 ++++ .../acceptance/diagnosis-context.json | 40 ++++ .../acceptance/diagnosis-kithmoot.json | 40 ++++ .../acceptance/impact-context.json | 50 +++++ .../acceptance/impact-kithmoot.json | 50 +++++ .../acceptance/orientation-context.json | 44 ++++ .../acceptance/orientation-kithmoot.json | 48 +++++ docs/experiments/rubric-v3-20260923/build.mjs | 21 ++ .../rubric-v3-20260923/rubrics.mjs | 56 ++++++ 12 files changed, 624 insertions(+) create mode 100644 docs/experiments/repeated-pro-20260923/context-instructions.txt create mode 100644 docs/experiments/repeated-pro-20260923/protocol.json create mode 100644 docs/experiments/repeated-pro-20260923/repeat.mjs create mode 100644 docs/experiments/rubric-v3-20260923/README.md create mode 100644 docs/experiments/rubric-v3-20260923/acceptance/diagnosis-context.json create mode 100644 docs/experiments/rubric-v3-20260923/acceptance/diagnosis-kithmoot.json create mode 100644 docs/experiments/rubric-v3-20260923/acceptance/impact-context.json create mode 100644 docs/experiments/rubric-v3-20260923/acceptance/impact-kithmoot.json create mode 100644 docs/experiments/rubric-v3-20260923/acceptance/orientation-context.json create mode 100644 docs/experiments/rubric-v3-20260923/acceptance/orientation-kithmoot.json create mode 100644 docs/experiments/rubric-v3-20260923/build.mjs create mode 100644 docs/experiments/rubric-v3-20260923/rubrics.mjs diff --git a/docs/experiments/repeated-pro-20260923/context-instructions.txt b/docs/experiments/repeated-pro-20260923/context-instructions.txt new file mode 100644 index 0000000..5f4a971 --- /dev/null +++ b/docs/experiments/repeated-pro-20260923/context-instructions.txt @@ -0,0 +1,9 @@ +Use the configured z1p-repository tools for source discovery. First check that +repository_status.root is this exact checkout or worktree; stop using a +mismatched binding. Explore each symbol once, fetch the blocks you rely on with +repository_packet and cite those lines. Run repository_coverage on the draft and +fix what it reports. Read files directly only for evidence the tools cannot +supply; tiny known-file edits need no scan. Refresh and re-fetch after edits, +branch switches, pulls, merges or rebases; reconnect after changing the binding +or server build. A shell directory change does not retarget Context. Keep the +project's existing models, tests and review standards. diff --git a/docs/experiments/repeated-pro-20260923/protocol.json b/docs/experiments/repeated-pro-20260923/protocol.json new file mode 100644 index 0000000..2a85048 --- /dev/null +++ b/docs/experiments/repeated-pro-20260923/protocol.json @@ -0,0 +1,188 @@ +{ + "version": 1, + "experimentId": "repeated-three-way-deepseek-v4-pro-20260923", + "lockedAt": "2026-09-23T00:21:44.863Z", + "shellPathRule": "The client's Bash tool rebuilds PATH from the login profile, so every arm receives PATH through the settings env: the Node 24 bin directory plus the runner's PATH. The graphify arm additionally prepends the isolated virtualenv bin so `graphify` resolves. Verified by a probe before lock.", + "purpose": "The repeated-run design of REPEATED-RUNS.md on the eight locked tasks: three independent repetitions of every task in every arm on a DeepSeek V4 Pro executor, so per-arm differences can be read against run-to-run spread. Held-out tasks are not included and are reported separately when written.", + "taskPack": { + "directory": "../d5-20260921", + "reused": [ + "tasks/*.json", + "setup/*.patch", + "acceptance/*.json", + "prepare-arm.mjs", + "accept.mjs" + ], + "note": "The eight locked v1 task definitions, seeded patches, frozen archives and deterministic checker are reused unchanged. v1, v2 and v3 results are not pooled with this experiment." + }, + "tasks": [ + "orientation-context", + "orientation-kithmoot", + "diagnosis-context", + "diagnosis-kithmoot", + "impact-context", + "impact-kithmoot", + "code-change-context", + "code-change-kithmoot" + ], + "retrievalInstructionRule": "Each arm's task prompt begins with one retrieval instruction of similar length and force, as the locked v1 measurement definition prescribed. Pilot 3 showed that the tools' own always-on text alone was ignored by the model in every arm, which measures nothing about the tools.", + "arms": { + "plain": { + "description": "Built-in Read, Edit, Write and Bash only. No MCP servers, no Graphify on PATH.", + "systemPromptAppendix": "Retrieval: use the built-in Read, Grep, Glob and Bash tools for source discovery.", + "retrievalInstruction": "you must use only bounded grep or rg and exact file reads for source discovery, confined to the task selection policy." + }, + "graphify": { + "description": "Built-in tools plus a Graphify graph prebuilt in graphify-out/ with `graphify update ` (no model, no API key) and the graphify CLI on PATH. The system prompt appendix is Graphify's own always-on instruction text (always_on/claude-md.md) verbatim. graphify-out/ is hidden from Git through an environment-supplied core.excludesFile so no hashed file changes.", + "retrievalInstruction": "a Graphify knowledge graph is prebuilt in graphify-out/ and the graphify CLI is on PATH. Before any grep, find, cat or file read you must run `graphify query \"\"` and `graphify explain \"\"` (and `graphify path \"\" \"\"` for relationships) for the concepts in this task, then read only the file locations they return, confined to the task selection policy.", + "package": "graphifyy", + "packageVersion": "0.9.65", + "build": "graphify update ", + "instructionSource": "graphify/always_on/claude-md.md", + "instructionSha256": "5a599e9f148ebbf0a8786c4508c3954daf95d0d74b88223d890dbf37aad23ec6", + "graphOutExcludedFromGit": true + }, + "context": { + "description": "Built-in tools plus the z1p-repository stdio MCP server (repository_status, repository_refresh, repository_explore, repository_coverage, repository_search, repository_packet) bound to the arm workspace, with the task selection policy in .z1p-navigation.json. Search and packet responses render as compact text; repository_explore answers one symbol per call. The system prompt appendix is the project instruction text from docs/GETTING-STARTED.md step 3 verbatim (context-instructions.txt). repository_coverage is a deterministic pre-submit check that lists explored files a draft answer leaves uncited.", + "package": "@forgesworn/context-tools", + "packageVersion": "source build at the recorded commit: repository_explore, repository_coverage (exact-quote check, symbols beyond eight returned as not checked), literal search, queued packets, packet length errors; clean working tree, identified by dist digests below", + "server": "encrypted-context.mjs navigate ", + "instructionSource": "docs/GETTING-STARTED.md step 3 (context-instructions.txt in this directory)", + "retrievalInstruction": "the z1p-repository MCP tools are bound to this workspace. Before any grep, find, cat or file read you must call repository_status (and repository_refresh if it is unavailable, stale or unknown), then call repository_explore for each identifier in this task, use repository_search with pathPrefix only for literals, and obtain the source and tests you cite with repository_packet using the current expectedGeneration, reading files directly only for evidence those tools cannot supply, confined to the task selection policy. Before writing answer.json, call repository_coverage with your draft answer and the task's identifiers, then address each missing file or state why it does not bear on the task.", + "instructionSha256": "cb7f696ea84c42f5923cfc0470a5416b1b6053f4df29bb162e80bb8795b9f934", + "build": { + "headCommit": "eb5fbff", + "workingTreeClean": true, + "dist": { + "repository-coverage.js": "45416989a1e158b70409665d49813f7cc6c122514db6fb4b33f359041fd63ae9", + "repository-explore.js": "54269956eee2297c8318c5fd430c2bdd0417ac16298027b52ce8840fbb9e3249", + "repository-navigation.js": "09ef3a2e92081c77f67eeb0b22a4d56c5f93559e33069dd24ea9639c30445a2b", + "repository-navigation-mcp.js": "4bee8f3cf1e28f51c2ad3d0d83c50de4843ae8bd41a06bfbdd0a20cb23a9febb", + "source-packet.mjs": "95335ae695966c23faa7de4e3c7fecce820a0fe90714787103c6cfa41b1e5298" + } + } + } + }, + "armOrders": [ + [ + "plain", + "graphify", + "context" + ], + [ + "graphify", + "context", + "plain" + ], + [ + "context", + "plain", + "graphify" + ] + ], + "armOrderRule": "Repetition r (1 to 3) of task i uses armOrders[(i + r - 1) mod 3], so arm order rotates per repetition as well as per task. Each repetition writes its own evidence directory; every arm has its own prepared workspace, fresh headless session and, for the context arm, its own server process.", + "executor": { + "client": "claude-code", + "clientVersion": "2.1.280", + "model": "deepseek-v4-pro:cloud", + "effort": "medium", + "maxTurns": 80, + "maxBudgetUsd": null, + "settingSources": "project", + "settingsOverride": { + "enabledPlugins": { + "agents-md@builtin": false + } + }, + "settingsOverrideReason": "A probe showed the built-in agents-md plugin injects the frozen tree's AGENTS.md into every arm; the Context revision's AGENTS.md instructs use of z1p-repository tools, so it is disabled for all arms. A second probe confirmed no project instruction text remains in context.", + "disallowedTools": [ + "WebFetch", + "WebSearch", + "Agent", + "ToolSearch", + "Workflow", + "EnterWorktree", + "ExitWorktree", + "RemoteTrigger", + "CronCreate", + "CronDelete", + "CronList", + "ScheduleWakeup", + "SendMessage", + "PushNotification", + "Monitor", + "DesignSync", + "ListAgents", + "TaskStop", + "NotebookEdit" + ], + "builtInToolsObserved": "This Claude Code build exposes Bash, Read, Edit and Write for source work; Grep and Glob are not separate tools, so plain-arm search runs through Bash.", + "permissions": "dangerously-skip-permissions inside a disposable frozen workspace", + "skills": "disabled", + "mcp": "strict; only the arm's --mcp-config", + "sessionPersistence": false, + "promptDelivery": "identical task prompt on stdin for every arm; the only difference is the arm's system prompt appendix, PATH and MCP config", + "modelRoute": "Claude Code 2.1.280 with ANTHROPIC_BASE_URL pointing at the local Ollama 0.34.2 daemon (Anthropic-compatible /v1/messages), which proxies the cloud tag to ollama.com; verified by a smoke test with Bash and MCP tool calls", + "env": { + "ANTHROPIC_BASE_URL": "http://127.0.0.1:11435", + "ANTHROPIC_API_KEY": "ollama", + "ANTHROPIC_AUTH_TOKEN": "ollama" + }, + "costNote": "The client reports a list-price estimate for an unrecognised model; it is meaningless here and no budget cap is applied. Provider usage counters (input, cache read, output) are recorded; Ollama Cloud credit consumption is read from the account, not from receipts.", + "effortNote": "--effort medium is accepted by the client; whether the route honours it is unknown. The smoke test reported zero thinking tokens." + }, + "reviewer": { + "client": "claude-code", + "model": "claude-sonnet-5", + "effort": "high", + "maxTurns": 3, + "disallowedTools": [ + "Bash", + "Read", + "Edit", + "Write", + "Grep", + "Glob", + "TodoWrite" + ], + "retry": "one further attempt if the structured verdict does not parse; both attempts are recorded and their usage and time are summed", + "tools": "none (Bash, Read, Edit, Write, Grep and Glob disallowed as well); receives the answer or diff, the private rubric, the deterministic checker result and bounded excerpts of the cited frozen source", + "blind": "does not see the arm name or any other arm's output" + }, + "measures": { + "accepted": "code tasks: deterministic checker and scope check passed (no model reviewer); structured tasks: checker passed and the blind reviewer accepted against rubric version 3 with no material issue", + "executorToolCalls": "count of tool_use blocks by tool name from the stream", + "executorInputTokens": "input_tokens + cache_creation_input_tokens + cache_read_input_tokens from the result usage; uncached = input_tokens + cache_creation_input_tokens", + "executorOutputTokens": "output_tokens from the result usage", + "costEstimateUsd": "Claude Code total_cost_usd, a list-price estimate, not subscription billing", + "executorSeconds": "wall time of the headless session", + "armSeconds": "graphify build (arm-specific setup) + executor + checker + reviewer wall time", + "reviewerTokens": "reviewer session usage", + "setupSeconds": "dependency installation, shared by all arms and excluded" + }, + "decisionRule": "As REPEATED-RUNS.md, on this locked set only: Context is better only if it accepts at least as many tasks (two of three repetitions) as each comparator and its sum of per-task median executor input is at least 20 percent lower than each comparator, with no higher median reviewer time. A difference of one task is inconclusive. Input totals only (the route cache accounting is inconsistent). The held-out half of the rule is not evaluated by this run.", + "knownAsymmetries": [ + "Graphify indexes the whole frozen workspace; Context indexes only the task selection policy. Graphify is not disadvantaged by this.", + "Graphify's arm-specific build time is counted in armSeconds; Context's refresh happens inside the session and is counted as a tool call.", + "Arm instructions are delivered by system prompt appendix rather than a CLAUDE.md file so the frozen tree stays identical across arms." + ], + "derivedFrom": "three-way-deepseek-v4.1-flash-smoke-20260923", + "codeAcceptance": "checker-and-scope", + "rubricDir": "../rubric-v3-20260923", + "repetitions": 3, + "failureRule": "The driver stops at the first executor, checker or reviewer failure that is not a verdict (provider error, quota, refusal, non-success result) and does not retry; finished cells are kept and a rerun resumes at the first cell without receipt.json.", + "changesFromS5": [ + "Three repetitions with rotated arm order; eight locked tasks.", + "Code acceptance by checker plus scope check; structured rubrics version 3.", + "Context arm at the recorded build with the Flash smoke fixes; instructions docs/GETTING-STARTED.md step 3." + ], + "attempts": [ + { + "lockedAt": "2026-09-23T00:10:23.306Z", + "rubricDir": "../rubric-v2-20260923", + "stoppedAt": "after 2026-09-23T00:19:31. (last logged event)", + "cells": "rep1 orientation-context, all three arms, rejected; rep1 orientation-kithmoot plain interrupted", + "reason": "All three arms failed orientation-context on rubric points the task prompt does not ask for; rubric version 3 limits required points to the prompt. Evidence of this attempt is kept and not reported with the run." + } + ] +} diff --git a/docs/experiments/repeated-pro-20260923/repeat.mjs b/docs/experiments/repeated-pro-20260923/repeat.mjs new file mode 100644 index 0000000..a2ba329 --- /dev/null +++ b/docs/experiments/repeated-pro-20260923/repeat.mjs @@ -0,0 +1,38 @@ +#!/usr/bin/env node +// Runs every repetition of the protocol through graphify-20260922/run.mjs, one task and arm order at a time. +// Arm order rotates per repetition; cells with a finished receipt.json are skipped, so a rerun resumes. +// Stops at the first failed run.mjs or a receipt whose executor or reviewer did not finish successfully. +// Usage: node repeat.mjs --local /private/local.json (local.evidence is the parent; repetitions go in rep1..repN) +import { spawnSync } from 'node:child_process' +import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs' +import { dirname, join, resolve } from 'node:path' +import { fileURLToPath } from 'node:url' + +const here = dirname(fileURLToPath(import.meta.url)) +const protocol = JSON.parse(readFileSync(join(here, 'protocol.json'), 'utf8')) +const flag = process.argv.indexOf('--local') +if (flag < 0) throw new Error('usage: repeat.mjs --local /private/local.json') +const local = JSON.parse(readFileSync(resolve(process.argv[flag + 1]), 'utf8')) +// Hitting the turn cap is an outcome of the cell, not a failure of the run. +const failed = (run) => run && !['success', 'error_max_turns'].includes(run.subtype) + +for (let rep = 1; rep <= protocol.repetitions; rep += 1) { + const evidence = join(local.evidence, `rep${rep}`) + mkdirSync(evidence, { recursive: true }) + const repLocal = join(evidence, 'local.json') + writeFileSync(repLocal, JSON.stringify({ ...local, evidence }, null, 2)) + protocol.tasks.forEach((task, i) => { + const order = protocol.armOrders[(i + rep - 1) % protocol.armOrders.length] + for (const arm of order) { + const receipt = join(evidence, task, arm, 'receipt.json') + if (!existsSync(receipt)) { + const r = spawnSync(process.execPath, [join(here, '../graphify-20260922/run.mjs'), '--local', repLocal, '--protocol', here, '--task', task, '--arms', arm], { stdio: 'inherit' }) + if (r.status !== 0) { console.log(`STOPPED rep${rep} ${task}/${arm}: run.mjs exited ${r.status}`); process.exit(1) } + } + const done = JSON.parse(readFileSync(receipt, 'utf8')) + if (failed(done.executorRun) || failed(done.reviewerRun)) { console.log(`STOPPED rep${rep} ${task}/${arm}: executor or reviewer did not finish successfully`); process.exit(1) } + console.log(`CELL rep${rep} ${task}/${arm} accepted=${done.accepted}`) + } + }) +} +console.log('REPEATS-DONE') diff --git a/docs/experiments/rubric-v3-20260923/README.md b/docs/experiments/rubric-v3-20260923/README.md new file mode 100644 index 0000000..597976f --- /dev/null +++ b/docs/experiments/rubric-v3-20260923/README.md @@ -0,0 +1,40 @@ +# Structured rubrics, version 3 + +Version 2 made each rubric point explicit, which exposed a second fault: some +required points were never asked for by the task prompt. In the first +repetition of the Pro run (`../repeated-pro-20260923/`) all three arms were +rejected on orientation-context, mostly for the same omissions: none said a +cursor is bound to its search term, when the prompt asks only "how continuation +state is bounded", and two did not say retrieved source grants no authority, +when the prompt asks only "which evidence remains unsigned". When every arm +misses a point the prompt did not request, the task measures guessing the +author's rubric, not retrieval, and cannot separate the arms. + +Version 3 keeps the prompts, dimension ids, required evidence, review rules and +checker, and limits each required point to what the prompt asks for. A point +the prompt does not ask for is marked "not required"; an answer that states it +wrongly can still fail through the source-contradiction rule. `node build.mjs` +writes `acceptance/*.json` with `version: 3` and refuses to build if any id +differs from version 1. + +## Changes (7 of 30 dimensions) + +| Task / dimension | Version 2 required | Version 3 required | Why | +| --- | --- | --- | --- | +| orientation-context / freshness | search re-inspects source and policy | search re-checks the policy before returning results or consuming a cursor | the prompt asks when search is blocked; source freshness labels results but never blocks | +| orientation-context / continuation | term and generation binding plus three bounds | the three bounds: single use, five minutes, 128 live | the prompt asks how continuation state is bounded | +| orientation-context / trust | unsigned marker, and data not instructions granting no authority | the unsigned marker, distinct from signed records | the prompt asks which evidence remains unsigned | +| diagnosis-kithmoot / regression | the unsafe cast must be named | a hostile verifier in the caller options, shown not to be used | the cast is an implementation detail of the test the prompt asks for | +| impact-context / response | eight response properties named unchanged | only: no existing field changes meaning | the prompt does not ask about response fields | +| impact-context / coverage | five named test cases | default unchanged, exact-case filtering, cross-mode cursor rejection | the three behaviours the prompt names | +| impact-kithmoot / tests | all six invalid-input classes named | valid input and at least three invalid classes | the prompt asks for tests, not a fixed list | + +orientation-kithmoot and diagnosis-context are unchanged: every required point +answers something their prompts ask. + +## What this does not show + +Whether version 3 separates the arms is what the repeated run measures. It may +make some tasks easy enough that every arm passes, which would also leave them +unable to discriminate; the per-task results will show that. Version 3 verdicts +are not comparable with version 1 or 2 verdicts. diff --git a/docs/experiments/rubric-v3-20260923/acceptance/diagnosis-context.json b/docs/experiments/rubric-v3-20260923/acceptance/diagnosis-context.json new file mode 100644 index 0000000..95114be --- /dev/null +++ b/docs/experiments/rubric-v3-20260923/acceptance/diagnosis-context.json @@ -0,0 +1,40 @@ +{ + "version": 3, + "checkerSha256": "539b5d7cd593f0dd6c5707b3426e243e90092614921c8024b8a69a5c49ffaa9c", + "kind": "structured", + "requiredFindingIds": [ + "root-cause", + "lost-invariant", + "why-first-check-fails", + "repair", + "regression" + ], + "reviewerRubric": { + "root-cause": "Required: inspectFreshness no longer re-inspects the policy after buildManifest has read the eligible source bytes, so it can report policy \"current\" after a change made during those reads. Fails if: the root cause is placed only in search() or its commit phase, or the answer says there is no source defect.", + "lost-invariant": "Required: a policy change during manifest construction (the freshness source reads) must make freshness inspection report the generation stale, or unknown, before search proceeds with old indexed results. Accepted alternative: freshness inspection must re-verify the policy after the source reads before reporting it current. Fails if: the invariant is placed only at search commit time; the commit-time re-check in search() is intact in the seeded tree, so that invariant was not lost.", + "why-first-check-fails": "Required: the discovery policy comparison in inspectFreshness runs before the source reads, so it cannot observe a policy change made during them.", + "repair": "Required: restore a re-inspection of the same policy directories (reinspectPolicy over discovery.policyDirectories) inside inspectFreshness after buildManifest, returning stale when the revision differs and unknown when re-inspection fails, before the manifest is used. Fails if: the existing commit-phase reinspectPolicy in search() is named as the repair, or the answer says no source change is needed; that check is present in the seeded tree and does not correct status().", + "regression": "Required: a test that fails on the seeded tree: tighten .gitignore while a source file handle is opened during freshness inspection and assert that status() (or inspectFreshness) reports the policy as stale or unknown, not current. Fails if: the only test proposed is the existing \"blocks a policy change made during freshness source reads without consuming the cursor\" search test, which passes on the seeded tree because the commit-time re-check still blocks the search. Keeping that test as well is fine." + }, + "requiredEvidence": [ + { + "path": "packages/context-tools/src/repository-navigation.ts", + "token": "buildManifest(discovery, signal)" + }, + { + "path": "packages/context-tools/src/repository-navigation.ts", + "token": "reinspectPolicy" + }, + { + "path": "packages/context-tools/src/repository-navigation.test.ts", + "token": "blocks a policy change made during freshness source reads" + } + ], + "basedOn": "d5-20260921/acceptance/diagnosis-context.json", + "reviewerRules": [ + "Judge each dimension against the whole answer: the summary and every finding. A required point stated under a different finding id counts; never fail a dimension for placement alone.", + "A required point passes only when the answer states it explicitly, in words a reader could quote. Different wording is fine; a point that is only implied by other statements does not pass.", + "Where a dimension lists items, each listed item must be named. A generic phrase such as \"edge cases\" or \"all semantics unchanged\" does not cover named items.", + "A dimension passes when every required point is stated, no \"fails if\" condition applies and nothing it states contradicts the frozen source excerpts. Points marked \"not required\" never cause a failure. Accepted alternatives pass equally." + ] +} diff --git a/docs/experiments/rubric-v3-20260923/acceptance/diagnosis-kithmoot.json b/docs/experiments/rubric-v3-20260923/acceptance/diagnosis-kithmoot.json new file mode 100644 index 0000000..a2fe16f --- /dev/null +++ b/docs/experiments/rubric-v3-20260923/acceptance/diagnosis-kithmoot.json @@ -0,0 +1,40 @@ +{ + "version": 3, + "checkerSha256": "539b5d7cd593f0dd6c5707b3426e243e90092614921c8024b8a69a5c49ffaa9c", + "kind": "structured", + "requiredFindingIds": [ + "root-cause", + "trust-impact", + "repair-type", + "repair-merge", + "regression" + ], + "reviewerRubric": { + "root-cause": "Required, both: (a) the seeded ContextVaultOptions is PortableOptions and so admits verifyDelegation; (b) kithmootContextOptions spreads caller options after installing KithMoot's verifier ({ verifyDelegation, ...options }), so a caller-supplied verifier overwrites it.", + "trust-impact": "Required: a caller-supplied verifier can approve forged or unrelated agent ownership, so grants that should be refused are accepted and the room adapter's grant boundary is weakened.", + "repair-type": "Required: ContextVaultOptions becomes Omit again.", + "repair-merge": "Required: kithmootContextOptions spreads caller options first and installs KithMoot's verifyDelegation last ({ ...options, verifyDelegation }).", + "regression": "Required, both: (a) the test supplies a hostile verifyDelegation in the caller options; (b) it shows KithMoot's verifier is used rather than the hostile one, either by asserting what kithmootContextOptions returns or by showing that a forged proof the hostile verifier would approve is still rejected. Not required: naming the unsafe cast the repaired type makes necessary." + }, + "requiredEvidence": [ + { + "path": "src/context.ts", + "token": "kithmootContextOptions" + }, + { + "path": "src/context.ts", + "token": "verifyAgentOwnership" + }, + { + "path": "src/context.test.ts", + "token": "rejects forged ownership" + } + ], + "basedOn": "d5-20260921/acceptance/diagnosis-kithmoot.json", + "reviewerRules": [ + "Judge each dimension against the whole answer: the summary and every finding. A required point stated under a different finding id counts; never fail a dimension for placement alone.", + "A required point passes only when the answer states it explicitly, in words a reader could quote. Different wording is fine; a point that is only implied by other statements does not pass.", + "Where a dimension lists items, each listed item must be named. A generic phrase such as \"edge cases\" or \"all semantics unchanged\" does not cover named items.", + "A dimension passes when every required point is stated, no \"fails if\" condition applies and nothing it states contradicts the frozen source excerpts. Points marked \"not required\" never cause a failure. Accepted alternatives pass equally." + ] +} diff --git a/docs/experiments/rubric-v3-20260923/acceptance/impact-context.json b/docs/experiments/rubric-v3-20260923/acceptance/impact-context.json new file mode 100644 index 0000000..bde535e --- /dev/null +++ b/docs/experiments/rubric-v3-20260923/acceptance/impact-context.json @@ -0,0 +1,50 @@ +{ + "version": 3, + "checkerSha256": "539b5d7cd593f0dd6c5707b3426e243e90092614921c8024b8a69a5c49ffaa9c", + "kind": "structured", + "requiredFindingIds": [ + "request-surface", + "default", + "filter", + "cursor", + "response", + "coverage" + ], + "reviewerRubric": { + "request-surface": "Required, both: optional caseSensitive is added to (a) the search options type (NavigationSearchOptions in repository-navigation.ts) and (b) the repository_search MCP input schema.", + "default": "Required: omitted or false keeps today's behaviour: the term is normalised by normalizeTerm (lower-cased) and looked up in the case-insensitive postings, so existing callers see no change.", + "filter": "Required: the postings are lower-case, so case-sensitive mode must filter candidate source lines for the exact-case token. Accepted alternatives: reuse the lower-case postings and post-filter the candidate lines, or add a separate case-preserving index, provided exact-case filtering of results is stated.", + "cursor": "Required, both: (a) caseSensitive is stored in the Cursor; (b) continuing a cursor with a different mode is rejected.", + "response": "Required: no point beyond the fails-if condition; the prompt does not ask about response fields, so an answer silent on them passes. Adding a separate field that reports the mode is allowed. Fails if: the answer proposes changing what an existing response field reports (for example echoing the raw-case term in term).", + "coverage": "Required, all three named: (a) default matching stays case-insensitive; (b) case-sensitive mode returns only exact-case matches; (c) continuing a cursor in the other mode is rejected. Not required: MCP schema validation or pagination within one mode." + }, + "requiredEvidence": [ + { + "path": "packages/context-tools/src/repository-navigation.ts", + "token": "export interface NavigationSearchOptions" + }, + { + "path": "packages/context-tools/src/repository-navigation.ts", + "token": "interface Cursor" + }, + { + "path": "packages/context-tools/src/repository-navigation-mcp.ts", + "token": "repository_search" + }, + { + "path": "packages/context-tools/src/repository-navigation.test.ts", + "token": "wrong-term, stale, and unknown cursors" + }, + { + "path": "docs/LOCAL-NAVIGATION.md", + "token": "case-insensitive" + } + ], + "basedOn": "d5-20260921/acceptance/impact-context.json", + "reviewerRules": [ + "Judge each dimension against the whole answer: the summary and every finding. A required point stated under a different finding id counts; never fail a dimension for placement alone.", + "A required point passes only when the answer states it explicitly, in words a reader could quote. Different wording is fine; a point that is only implied by other statements does not pass.", + "Where a dimension lists items, each listed item must be named. A generic phrase such as \"edge cases\" or \"all semantics unchanged\" does not cover named items.", + "A dimension passes when every required point is stated, no \"fails if\" condition applies and nothing it states contradicts the frozen source excerpts. Points marked \"not required\" never cause a failure. Accepted alternatives pass equally." + ] +} diff --git a/docs/experiments/rubric-v3-20260923/acceptance/impact-kithmoot.json b/docs/experiments/rubric-v3-20260923/acceptance/impact-kithmoot.json new file mode 100644 index 0000000..64f3f0f --- /dev/null +++ b/docs/experiments/rubric-v3-20260923/acceptance/impact-kithmoot.json @@ -0,0 +1,50 @@ +{ + "version": 3, + "checkerSha256": "539b5d7cd593f0dd6c5707b3426e243e90092614921c8024b8a69a5c49ffaa9c", + "kind": "structured", + "requiredFindingIds": [ + "contract", + "callers", + "privacy", + "tests", + "guard", + "compatibility" + ], + "reviewerRubric": { + "contract": "Required: shortId returns the first eight characters only for exactly 64 lower-case hexadecimal characters, and [invalid-id] for every other input.", + "callers": "Required: server/forwarder.mjs passes config.roomId and config.pubkey through shortId before startup logging.", + "privacy": "Required, both: (a) the invalid marker is a constant; stating that the literal [invalid-id] is returned for all invalid input satisfies this; (b) it echoes no part of the input, including no prefix.", + "tests": "Required, both: (a) the existing permissive expectations (\"never returns more than it was given\", which returns 'abcd' and '' unchanged) must be replaced; (b) new cases cover valid canonical input and invalid input of at least three named classes (for example empty, short, long, upper-case, non-hex, Unicode). Not required: all six classes.", + "guard": "Required, both: (a) the log-redaction scan (test/log-redaction-scan.test.ts) is retained and still rejects direct full-identifier logging; (b) the forwarder startup logging assertion is retained. Fails if: either is dismissed as unrelated or not mentioned.", + "compatibility": "Required, both: (a) valid canonical 64-character lower-case hex identifiers keep the same eight-character display; (b) malformed configured identifiers now display as [invalid-id], visibly invalid, instead of a prefix." + }, + "requiredEvidence": [ + { + "path": "src/log-redact.ts", + "token": "export function shortId" + }, + { + "path": "server/forwarder.mjs", + "token": "shortId(config.roomId)" + }, + { + "path": "server/forwarder.mjs", + "token": "shortId(config.pubkey)" + }, + { + "path": "src/log-redact.test.ts", + "token": "never returns more than it was given" + }, + { + "path": "test/log-redaction-scan.test.ts", + "token": "shortId" + } + ], + "basedOn": "d5-20260921/acceptance/impact-kithmoot.json", + "reviewerRules": [ + "Judge each dimension against the whole answer: the summary and every finding. A required point stated under a different finding id counts; never fail a dimension for placement alone.", + "A required point passes only when the answer states it explicitly, in words a reader could quote. Different wording is fine; a point that is only implied by other statements does not pass.", + "Where a dimension lists items, each listed item must be named. A generic phrase such as \"edge cases\" or \"all semantics unchanged\" does not cover named items.", + "A dimension passes when every required point is stated, no \"fails if\" condition applies and nothing it states contradicts the frozen source excerpts. Points marked \"not required\" never cause a failure. Accepted alternatives pass equally." + ] +} diff --git a/docs/experiments/rubric-v3-20260923/acceptance/orientation-context.json b/docs/experiments/rubric-v3-20260923/acceptance/orientation-context.json new file mode 100644 index 0000000..5da77b5 --- /dev/null +++ b/docs/experiments/rubric-v3-20260923/acceptance/orientation-context.json @@ -0,0 +1,44 @@ +{ + "version": 3, + "checkerSha256": "539b5d7cd593f0dd6c5707b3426e243e90092614921c8024b8a69a5c49ffaa9c", + "kind": "structured", + "requiredFindingIds": [ + "first-use", + "search-gate", + "freshness", + "continuation", + "trust" + ], + "reviewerRubric": { + "first-use": "Required, both: (a) before any refresh, status reports freshness \"unavailable\"; (b) an explicit refresh builds and publishes an in-memory generation. Not required: manifest or hashing details.", + "search-gate": "Required: search is rejected unless policy freshness is \"current\", which rejects unavailable, stale and unknown policy. Stating the general condition (\"policy must be current\", \"!== 'current'\") covers all three states. Fails if: only some states are named and the general condition is not stated. Not required: that stale source alone does not block search.", + "freshness": "Required: search re-checks the policy before it returns results or consumes a cursor, so a policy change made during a search blocks it. Not required: that search also re-inspects source freshness to label results, or that stale source alone does not block search. Fails if: the answer says a policy change during a search goes undetected.", + "continuation": "Required, all three bounds: (a) a cursor is single-use, consumed only when a continuation succeeds; (b) it expires after five minutes; (c) at most 128 cursors are live. Not required: that a cursor is bound to its generation and search term, or that refresh clears cursors.", + "trust": "Required: results are marked local-source-unsigned, as unsigned local evidence distinct from signed context records. Not required: that source is data, not instructions, or grants no authority. Fails if: the answer calls navigation results signed or treats them as signed evidence." + }, + "requiredEvidence": [ + { + "path": "packages/context-tools/src/repository-navigation.ts", + "token": "inspectFreshness" + }, + { + "path": "packages/context-tools/src/repository-navigation.ts", + "token": "CURSOR_MAX_ENTRIES = 128" + }, + { + "path": "packages/context-tools/src/repository-navigation.ts", + "token": "local-source-unsigned" + }, + { + "path": "packages/context-tools/src/repository-navigation.test.ts", + "token": "successfully used cursor is consumed" + } + ], + "basedOn": "d5-20260921/acceptance/orientation-context.json", + "reviewerRules": [ + "Judge each dimension against the whole answer: the summary and every finding. A required point stated under a different finding id counts; never fail a dimension for placement alone.", + "A required point passes only when the answer states it explicitly, in words a reader could quote. Different wording is fine; a point that is only implied by other statements does not pass.", + "Where a dimension lists items, each listed item must be named. A generic phrase such as \"edge cases\" or \"all semantics unchanged\" does not cover named items.", + "A dimension passes when every required point is stated, no \"fails if\" condition applies and nothing it states contradicts the frozen source excerpts. Points marked \"not required\" never cause a failure. Accepted alternatives pass equally." + ] +} diff --git a/docs/experiments/rubric-v3-20260923/acceptance/orientation-kithmoot.json b/docs/experiments/rubric-v3-20260923/acceptance/orientation-kithmoot.json new file mode 100644 index 0000000..7fa5cd2 --- /dev/null +++ b/docs/experiments/rubric-v3-20260923/acceptance/orientation-kithmoot.json @@ -0,0 +1,48 @@ +{ + "version": 3, + "checkerSha256": "539b5d7cd593f0dd6c5707b3426e243e90092614921c8024b8a69a5c49ffaa9c", + "kind": "structured", + "requiredFindingIds": [ + "adapter", + "ownership", + "storage", + "writes", + "caller-boundary" + ], + "reviewerRubric": { + "adapter": "Required: ContextVault wraps (extends) the portable context vault and always supplies KithMoot's verifyDelegation, through kithmootContextOptions, whatever the caller passes.", + "ownership": "Required, both, attributed to verifyDelegation in src/context.ts: (a) it first rejects a proof whose agent field does not equal options.agent; (b) it then calls verifyAgentOwnership. Fails if: only verifyAgentOwnership's internal checks are described, or the agent check is attributed only to verifyAgentOwnership.", + "storage": "Required: ContextFileStore in src/node/context-store.ts extends or wraps the portable file store and constructs it with kithmootContextOptions(options).", + "writes": "Required: all five tools are named as entering store.run with the write flag: context_create, context_append, context_import, context_upload and context_set_grants. Fails if: any of the five is missing. Not required: read-only tools.", + "caller-boundary": "Required: ContextVaultOptions is PortableOptions with verifyDelegation omitted (Omit), so callers cannot pass or select the room's trust policy." + }, + "requiredEvidence": [ + { + "path": "src/context.ts", + "token": "Omit" + }, + { + "path": "src/context.ts", + "token": "return { ...options, verifyDelegation }" + }, + { + "path": "src/node/context-store.ts", + "token": "kithmootContextOptions(options)" + }, + { + "path": "packages/context-tools/src/context-mcp.ts", + "token": "store.run(v => v.append" + }, + { + "path": "src/context.test.ts", + "token": "rejects forged ownership" + } + ], + "basedOn": "d5-20260921/acceptance/orientation-kithmoot.json", + "reviewerRules": [ + "Judge each dimension against the whole answer: the summary and every finding. A required point stated under a different finding id counts; never fail a dimension for placement alone.", + "A required point passes only when the answer states it explicitly, in words a reader could quote. Different wording is fine; a point that is only implied by other statements does not pass.", + "Where a dimension lists items, each listed item must be named. A generic phrase such as \"edge cases\" or \"all semantics unchanged\" does not cover named items.", + "A dimension passes when every required point is stated, no \"fails if\" condition applies and nothing it states contradicts the frozen source excerpts. Points marked \"not required\" never cause a failure. Accepted alternatives pass equally." + ] +} diff --git a/docs/experiments/rubric-v3-20260923/build.mjs b/docs/experiments/rubric-v3-20260923/build.mjs new file mode 100644 index 0000000..566e1b8 --- /dev/null +++ b/docs/experiments/rubric-v3-20260923/build.mjs @@ -0,0 +1,21 @@ +#!/usr/bin/env node +// Write acceptance/*.json (version 3) from rubrics.mjs and the locked version 1 files, and check that +// everything except the rubric text is unchanged. Usage: node build.mjs +import { readFileSync, writeFileSync } from 'node:fs' +import { dirname, join, resolve } from 'node:path' +import { fileURLToPath } from 'node:url' +import { reviewerRules, rubrics } from './rubrics.mjs' + +const here = dirname(fileURLToPath(import.meta.url)) +const pack = resolve(here, '../d5-20260921') +for (const [id, rubric] of Object.entries(rubrics)) { + const v1 = JSON.parse(readFileSync(join(pack, 'acceptance', `${id}.json`), 'utf8')) + const task = JSON.parse(readFileSync(join(pack, 'tasks', `${id}.json`), 'utf8')) + const ids = Object.keys(rubric) + if (v1.kind !== 'structured') throw new Error(`${id}: not a structured task`) + if (JSON.stringify(ids) !== JSON.stringify(task.requiredFindingIds) || JSON.stringify(ids) !== JSON.stringify(Object.keys(v1.reviewerRubric))) throw new Error(`${id}: dimension ids differ from version 1`) + for (const [dim, text] of Object.entries(rubric)) if (!text.startsWith('Required')) throw new Error(`${id}/${dim}: rubric text must start with its required points`) + const v3 = { ...v1, version: 3, basedOn: `d5-20260921/acceptance/${id}.json`, reviewerRules, reviewerRubric: rubric } + writeFileSync(join(here, 'acceptance', `${id}.json`), `${JSON.stringify(v3, null, 2)}\n`) +} +process.stdout.write(`wrote ${Object.keys(rubrics).length} acceptance files\n`) diff --git a/docs/experiments/rubric-v3-20260923/rubrics.mjs b/docs/experiments/rubric-v3-20260923/rubrics.mjs new file mode 100644 index 0000000..a149b9a --- /dev/null +++ b/docs/experiments/rubric-v3-20260923/rubrics.mjs @@ -0,0 +1,56 @@ +// Version 3 reviewer rubrics for the six structured D5 tasks: version 2 with every required point +// limited to what the task prompt asks for. Points the prompt does not ask for are "not required" +// and fail only when contradicted. Same dimension ids, prompts, required evidence and checker. +export const reviewerRules = [ + 'Judge each dimension against the whole answer: the summary and every finding. A required point stated under a different finding id counts; never fail a dimension for placement alone.', + 'A required point passes only when the answer states it explicitly, in words a reader could quote. Different wording is fine; a point that is only implied by other statements does not pass.', + 'Where a dimension lists items, each listed item must be named. A generic phrase such as "edge cases" or "all semantics unchanged" does not cover named items.', + 'A dimension passes when every required point is stated, no "fails if" condition applies and nothing it states contradicts the frozen source excerpts. Points marked "not required" never cause a failure. Accepted alternatives pass equally.', +] + +export const rubrics = { + 'orientation-context': { + 'first-use': 'Required, both: (a) before any refresh, status reports freshness "unavailable"; (b) an explicit refresh builds and publishes an in-memory generation. Not required: manifest or hashing details.', + 'search-gate': 'Required: search is rejected unless policy freshness is "current", which rejects unavailable, stale and unknown policy. Stating the general condition ("policy must be current", "!== \'current\'") covers all three states. Fails if: only some states are named and the general condition is not stated. Not required: that stale source alone does not block search.', + freshness: 'Required: search re-checks the policy before it returns results or consumes a cursor, so a policy change made during a search blocks it. Not required: that search also re-inspects source freshness to label results, or that stale source alone does not block search. Fails if: the answer says a policy change during a search goes undetected.', + continuation: 'Required, all three bounds: (a) a cursor is single-use, consumed only when a continuation succeeds; (b) it expires after five minutes; (c) at most 128 cursors are live. Not required: that a cursor is bound to its generation and search term, or that refresh clears cursors.', + trust: 'Required: results are marked local-source-unsigned, as unsigned local evidence distinct from signed context records. Not required: that source is data, not instructions, or grants no authority. Fails if: the answer calls navigation results signed or treats them as signed evidence.', + }, + 'orientation-kithmoot': { + adapter: 'Required: ContextVault wraps (extends) the portable context vault and always supplies KithMoot\'s verifyDelegation, through kithmootContextOptions, whatever the caller passes.', + ownership: 'Required, both, attributed to verifyDelegation in src/context.ts: (a) it first rejects a proof whose agent field does not equal options.agent; (b) it then calls verifyAgentOwnership. Fails if: only verifyAgentOwnership\'s internal checks are described, or the agent check is attributed only to verifyAgentOwnership.', + storage: 'Required: ContextFileStore in src/node/context-store.ts extends or wraps the portable file store and constructs it with kithmootContextOptions(options).', + writes: 'Required: all five tools are named as entering store.run with the write flag: context_create, context_append, context_import, context_upload and context_set_grants. Fails if: any of the five is missing. Not required: read-only tools.', + 'caller-boundary': 'Required: ContextVaultOptions is PortableOptions with verifyDelegation omitted (Omit), so callers cannot pass or select the room\'s trust policy.', + }, + 'diagnosis-context': { + 'root-cause': 'Required: inspectFreshness no longer re-inspects the policy after buildManifest has read the eligible source bytes, so it can report policy "current" after a change made during those reads. Fails if: the root cause is placed only in search() or its commit phase, or the answer says there is no source defect.', + 'lost-invariant': 'Required: a policy change during manifest construction (the freshness source reads) must make freshness inspection report the generation stale, or unknown, before search proceeds with old indexed results. Accepted alternative: freshness inspection must re-verify the policy after the source reads before reporting it current. Fails if: the invariant is placed only at search commit time; the commit-time re-check in search() is intact in the seeded tree, so that invariant was not lost.', + 'why-first-check-fails': 'Required: the discovery policy comparison in inspectFreshness runs before the source reads, so it cannot observe a policy change made during them.', + repair: 'Required: restore a re-inspection of the same policy directories (reinspectPolicy over discovery.policyDirectories) inside inspectFreshness after buildManifest, returning stale when the revision differs and unknown when re-inspection fails, before the manifest is used. Fails if: the existing commit-phase reinspectPolicy in search() is named as the repair, or the answer says no source change is needed; that check is present in the seeded tree and does not correct status().', + regression: 'Required: a test that fails on the seeded tree: tighten .gitignore while a source file handle is opened during freshness inspection and assert that status() (or inspectFreshness) reports the policy as stale or unknown, not current. Fails if: the only test proposed is the existing "blocks a policy change made during freshness source reads without consuming the cursor" search test, which passes on the seeded tree because the commit-time re-check still blocks the search. Keeping that test as well is fine.', + }, + 'diagnosis-kithmoot': { + 'root-cause': 'Required, both: (a) the seeded ContextVaultOptions is PortableOptions and so admits verifyDelegation; (b) kithmootContextOptions spreads caller options after installing KithMoot\'s verifier ({ verifyDelegation, ...options }), so a caller-supplied verifier overwrites it.', + 'trust-impact': 'Required: a caller-supplied verifier can approve forged or unrelated agent ownership, so grants that should be refused are accepted and the room adapter\'s grant boundary is weakened.', + 'repair-type': 'Required: ContextVaultOptions becomes Omit again.', + 'repair-merge': 'Required: kithmootContextOptions spreads caller options first and installs KithMoot\'s verifyDelegation last ({ ...options, verifyDelegation }).', + regression: 'Required, both: (a) the test supplies a hostile verifyDelegation in the caller options; (b) it shows KithMoot\'s verifier is used rather than the hostile one, either by asserting what kithmootContextOptions returns or by showing that a forged proof the hostile verifier would approve is still rejected. Not required: naming the unsafe cast the repaired type makes necessary.', + }, + 'impact-context': { + 'request-surface': 'Required, both: optional caseSensitive is added to (a) the search options type (NavigationSearchOptions in repository-navigation.ts) and (b) the repository_search MCP input schema.', + default: 'Required: omitted or false keeps today\'s behaviour: the term is normalised by normalizeTerm (lower-cased) and looked up in the case-insensitive postings, so existing callers see no change.', + filter: 'Required: the postings are lower-case, so case-sensitive mode must filter candidate source lines for the exact-case token. Accepted alternatives: reuse the lower-case postings and post-filter the candidate lines, or add a separate case-preserving index, provided exact-case filtering of results is stated.', + cursor: 'Required, both: (a) caseSensitive is stored in the Cursor; (b) continuing a cursor with a different mode is rejected.', + response: 'Required: no point beyond the fails-if condition; the prompt does not ask about response fields, so an answer silent on them passes. Adding a separate field that reports the mode is allowed. Fails if: the answer proposes changing what an existing response field reports (for example echoing the raw-case term in term).', + coverage: 'Required, all three named: (a) default matching stays case-insensitive; (b) case-sensitive mode returns only exact-case matches; (c) continuing a cursor in the other mode is rejected. Not required: MCP schema validation or pagination within one mode.', + }, + 'impact-kithmoot': { + contract: 'Required: shortId returns the first eight characters only for exactly 64 lower-case hexadecimal characters, and [invalid-id] for every other input.', + callers: 'Required: server/forwarder.mjs passes config.roomId and config.pubkey through shortId before startup logging.', + privacy: 'Required, both: (a) the invalid marker is a constant; stating that the literal [invalid-id] is returned for all invalid input satisfies this; (b) it echoes no part of the input, including no prefix.', + tests: 'Required, both: (a) the existing permissive expectations ("never returns more than it was given", which returns \'abcd\' and \'\' unchanged) must be replaced; (b) new cases cover valid canonical input and invalid input of at least three named classes (for example empty, short, long, upper-case, non-hex, Unicode). Not required: all six classes.', + guard: 'Required, both: (a) the log-redaction scan (test/log-redaction-scan.test.ts) is retained and still rejects direct full-identifier logging; (b) the forwarder startup logging assertion is retained. Fails if: either is dismissed as unrelated or not mentioned.', + compatibility: 'Required, both: (a) valid canonical 64-character lower-case hex identifiers keep the same eight-character display; (b) malformed configured identifiers now display as [invalid-id], visibly invalid, instead of a prefix.', + }, +} From 8d7954fdd099660d6d0ce0de3ff965c9b3c71157 Mon Sep 17 00:00:00 2001 From: TheCryptoDonkey Date: Wed, 23 Sep 2026 01:39:55 +0100 Subject: [PATCH 2/4] docs: cost-first decision rule and updated build for the repeated Pro run --- docs/experiments/graphify-20260922/run.mjs | 2 +- .../repeated-pro-20260923/protocol.json | 31 +++++++++++++------ .../repeated-pro-20260923/repeat.mjs | 4 +-- 3 files changed, 24 insertions(+), 13 deletions(-) diff --git a/docs/experiments/graphify-20260922/run.mjs b/docs/experiments/graphify-20260922/run.mjs index 3b00f8b..8512edc 100644 --- a/docs/experiments/graphify-20260922/run.mjs +++ b/docs/experiments/graphify-20260922/run.mjs @@ -367,7 +367,7 @@ for (const taskId of tasks) { const armDir = join(evidence, taskId, order[i]) mkdirSync(armDir, { recursive: true }) writeFileSync(join(armDir, 'mcp-empty.json'), '{"mcpServers":{}}') - await runArm({ taskId, arm: order[i], orderIndex: i + 1, local, evidence, skipReview: Boolean(a['skip-review']) }) + await runArm({ taskId, arm: order[i], orderIndex: a['order-index'] ? Number(a['order-index']) : i + 1, local, evidence, skipReview: Boolean(a['skip-review']) }) } } log(evidence, 'done') diff --git a/docs/experiments/repeated-pro-20260923/protocol.json b/docs/experiments/repeated-pro-20260923/protocol.json index 2a85048..befebf0 100644 --- a/docs/experiments/repeated-pro-20260923/protocol.json +++ b/docs/experiments/repeated-pro-20260923/protocol.json @@ -1,7 +1,7 @@ { "version": 1, "experimentId": "repeated-three-way-deepseek-v4-pro-20260923", - "lockedAt": "2026-09-23T00:21:44.863Z", + "lockedAt": "2026-09-23T00:39:45.578Z", "shellPathRule": "The client's Bash tool rebuilds PATH from the login profile, so every arm receives PATH through the settings env: the Node 24 bin directory plus the runner's PATH. The graphify arm additionally prepends the isolated virtualenv bin so `graphify` resolves. Verified by a probe before lock.", "purpose": "The repeated-run design of REPEATED-RUNS.md on the eight locked tasks: three independent repetitions of every task in every arm on a DeepSeek V4 Pro executor, so per-arm differences can be read against run-to-run spread. Held-out tasks are not included and are reported separately when written.", "taskPack": { @@ -45,20 +45,20 @@ "context": { "description": "Built-in tools plus the z1p-repository stdio MCP server (repository_status, repository_refresh, repository_explore, repository_coverage, repository_search, repository_packet) bound to the arm workspace, with the task selection policy in .z1p-navigation.json. Search and packet responses render as compact text; repository_explore answers one symbol per call. The system prompt appendix is the project instruction text from docs/GETTING-STARTED.md step 3 verbatim (context-instructions.txt). repository_coverage is a deterministic pre-submit check that lists explored files a draft answer leaves uncited.", "package": "@forgesworn/context-tools", - "packageVersion": "source build at the recorded commit: repository_explore, repository_coverage (exact-quote check, symbols beyond eight returned as not checked), literal search, queued packets, packet length errors; clean working tree, identified by dist digests below", + "packageVersion": "source build at the recorded commit: repository_explore, repository_coverage (exact-quote check, symbols beyond eight returned as not checked), literal search, queued packets, packet ranges clamped to the end of file, optional packet handoff metadata; clean working tree, identified by dist digests below", "server": "encrypted-context.mjs navigate ", "instructionSource": "docs/GETTING-STARTED.md step 3 (context-instructions.txt in this directory)", "retrievalInstruction": "the z1p-repository MCP tools are bound to this workspace. Before any grep, find, cat or file read you must call repository_status (and repository_refresh if it is unavailable, stale or unknown), then call repository_explore for each identifier in this task, use repository_search with pathPrefix only for literals, and obtain the source and tests you cite with repository_packet using the current expectedGeneration, reading files directly only for evidence those tools cannot supply, confined to the task selection policy. Before writing answer.json, call repository_coverage with your draft answer and the task's identifiers, then address each missing file or state why it does not bear on the task.", "instructionSha256": "cb7f696ea84c42f5923cfc0470a5416b1b6053f4df29bb162e80bb8795b9f934", "build": { - "headCommit": "eb5fbff", + "headCommit": "3e1cd6a", "workingTreeClean": true, "dist": { "repository-coverage.js": "45416989a1e158b70409665d49813f7cc6c122514db6fb4b33f359041fd63ae9", "repository-explore.js": "54269956eee2297c8318c5fd430c2bdd0417ac16298027b52ce8840fbb9e3249", "repository-navigation.js": "09ef3a2e92081c77f67eeb0b22a4d56c5f93559e33069dd24ea9639c30445a2b", - "repository-navigation-mcp.js": "4bee8f3cf1e28f51c2ad3d0d83c50de4843ae8bd41a06bfbdd0a20cb23a9febb", - "source-packet.mjs": "95335ae695966c23faa7de4e3c7fecce820a0fe90714787103c6cfa41b1e5298" + "repository-navigation-mcp.js": "0e7a532448edba00c7668fca5d528d36418afd0489fac6f04ad79b81fb175946", + "source-packet.mjs": "674e00406782396839c3f578cd3f0684260aa584a22d3ad888fcb5b661e687eb" } } } @@ -158,13 +158,16 @@ "executorSeconds": "wall time of the headless session", "armSeconds": "graphify build (arm-specific setup) + executor + checker + reviewer wall time", "reviewerTokens": "reviewer session usage", - "setupSeconds": "dependency installation, shared by all arms and excluded" + "setupSeconds": "dependency installation, shared by all arms and excluded", + "orderIndex": "position of the arm in its rotated order, passed to run.mjs with --order-index" }, - "decisionRule": "As REPEATED-RUNS.md, on this locked set only: Context is better only if it accepts at least as many tasks (two of three repetitions) as each comparator and its sum of per-task median executor input is at least 20 percent lower than each comparator, with no higher median reviewer time. A difference of one task is inconclusive. Input totals only (the route cache accounting is inconsistent). The held-out half of the rule is not evaluated by this run.", + "decisionRule": "Cost first, quality as a guard, on this locked set only. Primary: for each task, the ratio of Context's median executor input (three repetitions) to each comparator's; Context is cheaper than a comparator if the geometric mean of the eight ratios is at most 0.8 and the upper end of a 90 percent bootstrap interval (resampling repetitions within task and arm, 10,000 draws, seed 20260923) is below 1. Guard: a task counts as accepted for an arm when two of three repetitions are accepted; Context must accept no fewer tasks than each comparator. Context is better only if both hold against both comparators. Reported, not decisive: per-task medians and spread, tool-result bytes, tool errors, turns, reviewer time, and the Context-repository and KithMoot tasks separately. Input totals only (the route cache accounting is inconsistent). The held-out half of REPEATED-RUNS.md is not evaluated.", "knownAsymmetries": [ "Graphify indexes the whole frozen workspace; Context indexes only the task selection policy. Graphify is not disadvantaged by this.", "Graphify's arm-specific build time is counted in armSeconds; Context's refresh happens inside the session and is counted as a tool call.", - "Arm instructions are delivered by system prompt appendix rather than a CLAUDE.md file so the frozen tree stays identical across arms." + "Arm instructions are delivered by system prompt appendix rather than a CLAUDE.md file so the frozen tree stays identical across arms.", + "Four tasks ask about the navigation code the Context arm runs. Its tools print local-source-unsigned in their metadata and its instruction names the status states, both of which bear on orientation-context answers. Results are reported for the Context-repository and KithMoot tasks separately.", + "The Context instruction (the product's documented project instruction) is longer than the other arms' instructions." ], "derivedFrom": "three-way-deepseek-v4.1-flash-smoke-20260923", "codeAcceptance": "checker-and-scope", @@ -174,7 +177,7 @@ "changesFromS5": [ "Three repetitions with rotated arm order; eight locked tasks.", "Code acceptance by checker plus scope check; structured rubrics version 3.", - "Context arm at the recorded build with the Flash smoke fixes; instructions docs/GETTING-STARTED.md step 3." + "Context arm at the recorded build with the Flash smoke and retrieval-analysis fixes; instructions docs/GETTING-STARTED.md step 3." ], "attempts": [ { @@ -183,6 +186,14 @@ "stoppedAt": "after 2026-09-23T00:19:31. (last logged event)", "cells": "rep1 orientation-context, all three arms, rejected; rep1 orientation-kithmoot plain interrupted", "reason": "All three arms failed orientation-context on rubric points the task prompt does not ask for; rubric version 3 limits required points to the prompt. Evidence of this attempt is kept and not reported with the run." + }, + { + "lockedAt": "2026-09-23T00:21:44.863Z", + "rubricDir": "../rubric-v3-20260923", + "stoppedAt": "before the first cell", + "cells": "none", + "reason": "An independent methodology review and a model-free retrieval analysis (../retrieval-recall-20260923/) found that every arm usually retrieves the required evidence and that the Context arm's friction was tool errors. The build was updated with those fixes (#27), the decision rule was made cost-first with acceptance as a quality guard, and receipts now record the rotated arm position." } - ] + ], + "decisionRuleChange": "Changed before any cell of this attempt ran. The previous rule required a 20 percent lower sum of per-task medians, which one task (diagnosis-context, about 37 percent of input) dominated, and no higher median reviewer time, which times the reviewer rather than the tool. The model-free analysis showed acceptance on these tasks reflects synthesis rather than retrieval, so acceptance is kept as a guard and cost is the measure." } diff --git a/docs/experiments/repeated-pro-20260923/repeat.mjs b/docs/experiments/repeated-pro-20260923/repeat.mjs index a2ba329..505a960 100644 --- a/docs/experiments/repeated-pro-20260923/repeat.mjs +++ b/docs/experiments/repeated-pro-20260923/repeat.mjs @@ -23,10 +23,10 @@ for (let rep = 1; rep <= protocol.repetitions; rep += 1) { writeFileSync(repLocal, JSON.stringify({ ...local, evidence }, null, 2)) protocol.tasks.forEach((task, i) => { const order = protocol.armOrders[(i + rep - 1) % protocol.armOrders.length] - for (const arm of order) { + for (const [position, arm] of order.entries()) { const receipt = join(evidence, task, arm, 'receipt.json') if (!existsSync(receipt)) { - const r = spawnSync(process.execPath, [join(here, '../graphify-20260922/run.mjs'), '--local', repLocal, '--protocol', here, '--task', task, '--arms', arm], { stdio: 'inherit' }) + const r = spawnSync(process.execPath, [join(here, '../graphify-20260922/run.mjs'), '--local', repLocal, '--protocol', here, '--task', task, '--arms', arm, '--order-index', String(position + 1)], { stdio: 'inherit' }) if (r.status !== 0) { console.log(`STOPPED rep${rep} ${task}/${arm}: run.mjs exited ${r.status}`); process.exit(1) } } const done = JSON.parse(readFileSync(receipt, 'utf8')) From a885a912a7fb2cce14d43bfd5c432c780b21586d Mon Sep 17 00:00:00 2001 From: TheCryptoDonkey Date: Wed, 23 Sep 2026 01:41:22 +0100 Subject: [PATCH 3/4] docs: summariser for the repeated Pro run decision rule, written before results --- .../repeated-pro-20260923/summarise.mjs | 92 +++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 docs/experiments/repeated-pro-20260923/summarise.mjs diff --git a/docs/experiments/repeated-pro-20260923/summarise.mjs b/docs/experiments/repeated-pro-20260923/summarise.mjs new file mode 100644 index 0000000..392a44c --- /dev/null +++ b/docs/experiments/repeated-pro-20260923/summarise.mjs @@ -0,0 +1,92 @@ +#!/usr/bin/env node +// Applies protocol.json's decision rule to rep1..repN receipts. Written before any cell of attempt 3 ran. +// Usage: node summarise.mjs (prints markdown tables and the verdict) +import { existsSync, readFileSync } from 'node:fs' +import { dirname, join } from 'node:path' +import { fileURLToPath } from 'node:url' + +const here = dirname(fileURLToPath(import.meta.url)) +const protocol = JSON.parse(readFileSync(join(here, 'protocol.json'), 'utf8')) +const parent = process.argv[2] +if (!parent) throw new Error('usage: summarise.mjs ') +const arms = ['plain', 'graphify', 'context'] +const reps = Array.from({ length: protocol.repetitions }, (_, i) => i + 1) + +const median = (xs) => { const s = [...xs].sort((a, b) => a - b); const m = s.length >> 1; return s.length % 2 ? s[m] : (s[m - 1] + s[m]) / 2 } +const geomean = (xs) => Math.exp(xs.reduce((a, x) => a + Math.log(x), 0) / xs.length) +// Deterministic PRNG (mulberry32) so the interval is reproducible from the seed. +function rng(seed) { let a = seed >>> 0; return () => { a = (a + 0x6d2b79f5) >>> 0; let t = a; t = Math.imul(t ^ (t >>> 15), t | 1); t ^= t + Math.imul(t ^ (t >>> 7), t | 61); return ((t ^ (t >>> 14)) >>> 0) / 4294967296 } } + +function toolStats(streamPath) { + let bytes = 0; let errors = 0; let results = 0 + if (!existsSync(streamPath)) return { bytes: null, errors: null, results: null } + for (const line of readFileSync(streamPath, 'utf8').split('\n')) { + if (!line.trim()) continue + const event = JSON.parse(line) + if (event.type !== 'user' || !Array.isArray(event.message?.content)) continue + for (const part of event.message.content) { + if (part.type !== 'tool_result') continue + results += 1 + if (part.is_error) errors += 1 + const text = typeof part.content === 'string' ? part.content : (part.content ?? []).map((p) => p.text ?? '').join('\n') + bytes += Buffer.byteLength(text) + } + } + return { bytes, errors, results } +} + +const cells = {} +for (const task of protocol.tasks) for (const arm of arms) { + cells[`${task}/${arm}`] = reps.map((rep) => { + const dir = join(parent, `rep${rep}`, task, arm) + const path = join(dir, 'receipt.json') + if (!existsSync(path)) return null + const r = JSON.parse(readFileSync(path, 'utf8')) + return { accepted: Boolean(r.accepted), input: r.executorRun.inputTotal, turns: r.executorRun.numTurns, reviewerSeconds: r.reviewerRun?.seconds ?? null, ...toolStats(join(dir, 'executor.stream.jsonl')) } + }) +} +const complete = Object.values(cells).every((runs) => runs.every(Boolean)) + +const acceptedTasks = (arm) => protocol.tasks.filter((t) => cells[`${t}/${arm}`].filter((c) => c?.accepted).length >= 2).length +const medInput = (task, arm, runs = cells[`${task}/${arm}`]) => median(runs.filter(Boolean).map((c) => c.input)) +const ratio = (comparator, pick = (t, a) => medInput(t, a)) => geomean(protocol.tasks.map((t) => pick(t, 'context') / pick(t, comparator))) + +function bootstrap(comparator, draws = 10000, seed = 20260923) { + const random = rng(seed) + const resample = (runs) => { const ok = runs.filter(Boolean); return ok.map(() => ok[Math.floor(random() * ok.length)]) } + const values = [] + for (let i = 0; i < draws; i += 1) { + const sampled = {} + for (const t of protocol.tasks) for (const a of ['context', comparator]) sampled[`${t}/${a}`] = resample(cells[`${t}/${a}`]) + values.push(ratio(comparator, (t, a) => median(sampled[`${t}/${a}`].map((c) => c.input)))) + } + values.sort((a, b) => a - b) + return [values[Math.floor(0.05 * draws)], values[Math.floor(0.95 * draws) - 1]] +} + +const out = [] +out.push(`Cells complete: ${complete ? 'yes' : 'no'}\n`) +out.push('| Task | Arm | Accepted | Median input | Input range | Median tool bytes | Tool errors | Median turns |') +out.push('| --- | --- | ---: | ---: | --- | ---: | ---: | ---: |') +for (const task of protocol.tasks) for (const arm of arms) { + const runs = cells[`${task}/${arm}`].filter(Boolean) + if (!runs.length) continue + const inputs = runs.map((c) => c.input) + out.push(`| ${task} | ${arm} | ${runs.filter((c) => c.accepted).length} of ${runs.length} | ${Math.round(median(inputs)).toLocaleString('en-GB')} | ${Math.min(...inputs).toLocaleString('en-GB')} to ${Math.max(...inputs).toLocaleString('en-GB')} | ${Math.round(median(runs.map((c) => c.bytes))).toLocaleString('en-GB')} | ${runs.reduce((a, c) => a + c.errors, 0)} / ${runs.reduce((a, c) => a + c.results, 0)} | ${median(runs.map((c) => c.turns))} |`) +} +if (complete) { + out.push('\n| Comparator | Geometric mean ratio (context / comparator) | 90% interval | Context-repo tasks | KithMoot tasks | Accepted tasks (context vs comparator) | Cost | Guard |') + out.push('| --- | ---: | --- | ---: | ---: | --- | --- | --- |') + const verdicts = [] + for (const comparator of ['plain', 'graphify']) { + const g = ratio(comparator) + const [lo, hi] = bootstrap(comparator) + const sub = (repo) => geomean(protocol.tasks.filter((t) => t.endsWith(`-${repo}`)).map((t) => medInput(t, 'context') / medInput(t, comparator))) + const cost = g <= 0.8 && hi < 1 + const guard = acceptedTasks('context') >= acceptedTasks(comparator) + verdicts.push(cost && guard) + out.push(`| ${comparator} | ${g.toFixed(3)} | ${lo.toFixed(3)} to ${hi.toFixed(3)} | ${sub('context').toFixed(3)} | ${sub('kithmoot').toFixed(3)} | ${acceptedTasks('context')} vs ${acceptedTasks(comparator)} | ${cost ? 'met' : 'not met'} | ${guard ? 'met' : 'not met'} |`) + } + out.push(`\nDecision rule: ${verdicts.every(Boolean) ? 'met' : 'not met'}. Graphify accepted ${acceptedTasks('graphify')}, plain ${acceptedTasks('plain')}, Context ${acceptedTasks('context')} of ${protocol.tasks.length} tasks (two of three repetitions).`) +} +process.stdout.write(`${out.join('\n')}\n`) From 79fe6641e6c7e9920ac2e817485a8c88e9ec965a Mon Sep 17 00:00:00 2001 From: TheCryptoDonkey Date: Wed, 23 Sep 2026 06:57:17 +0100 Subject: [PATCH 4/4] docs: repeated Pro run results; decision rule not met --- .../repeated-pro-20260923/RESULTS.md | 91 +++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 docs/experiments/repeated-pro-20260923/RESULTS.md diff --git a/docs/experiments/repeated-pro-20260923/RESULTS.md b/docs/experiments/repeated-pro-20260923/RESULTS.md new file mode 100644 index 0000000..ef4c59d --- /dev/null +++ b/docs/experiments/repeated-pro-20260923/RESULTS.md @@ -0,0 +1,91 @@ +# Repeated three-way run on a DeepSeek V4 Pro executor + +Locked protocol `repeated-three-way-deepseek-v4-pro-20260923` (attempt 3, locked +2026-09-23T00:39:45Z), run 00:40 to 05:56 UTC on 23 September 2026 with +Claude Code 2.1.280. Every executor ran `deepseek-v4-pro:cloud` at medium +effort through the local Ollama route; structured answers were reviewed by +`claude-sonnet-5` at high effort against rubric version 3; code tasks were +accepted by checker and scope. Eight locked tasks, three arms, three +repetitions with rotated arm order: 72 cells, all completed, no provider +failure, no retry. `summarise.mjs`, committed before the first result, applies +the decision rule. + +## Result: the decision rule is not met + +| Comparator | Geometric mean input ratio, Context / comparator | 90% interval | Context-repo tasks | KithMoot tasks | Tasks accepted, Context vs comparator | +| --- | ---: | --- | ---: | ---: | --- | +| plain | 1.169 | 0.793 to 1.355 | 1.333 | 1.025 | 3 vs 5 | +| graphify | 1.313 | 0.935 to 1.639 | 1.446 | 1.191 | 3 vs 5 | + +A ratio above 1 means Context used more executor input. The rule needed at +most 0.8 with the interval below 1, and no fewer accepted tasks; both parts +fail against both comparators. On this executor and task set, Context was +neither cheaper nor more often accepted. + +| Arm | Executor input, all 24 cells | Output | Median turns | Accepted cells | Executor minutes | +| --- | ---: | ---: | ---: | ---: | ---: | +| plain | 15,694,617 | 628,448 | 17.5 | 15 of 24 | 93 | +| graphify | 12,823,047 | 419,252 | 17.5 | 13 of 24 | 70 | +| context | 15,198,553 | 726,119 | 22.0 | 11 of 24 | 94 | + +Input totals include cache reads; the route's cache accounting is +inconsistent, so only totals are compared. Credit consumption was not read. + +## Per task + +| Task | Arm | Accepted | Median input | Input range | Median tool bytes | Tool errors | Median turns | +| --- | --- | ---: | ---: | --- | ---: | ---: | ---: | +| orientation-context | plain | 1 of 3 | 395,186 | 323,744 to 445,020 | 116,640 | 0 / 47 | 18 | +| orientation-context | graphify | 0 of 3 | 144,836 | 124,716 to 288,798 | 111,471 | 2 / 31 | 9 | +| orientation-context | context | 1 of 3 | 250,133 | 248,043 to 345,592 | 87,023 | 6 / 55 | 19 | +| orientation-kithmoot | plain | 1 of 3 | 502,732 | 478,081 to 622,940 | 107,982 | 0 / 77 | 25 | +| orientation-kithmoot | graphify | 0 of 3 | 316,911 | 262,293 to 324,383 | 94,660 | 1 / 60 | 22 | +| orientation-kithmoot | context | 0 of 3 | 329,316 | 328,618 to 353,720 | 107,651 | 3 / 64 | 24 | +| diagnosis-context | plain | 0 of 3 | 1,434,506 | 1,216,138 to 5,941,060 | 122,225 | 5 / 131 | 28 | +| diagnosis-context | graphify | 0 of 3 | 2,332,533 | 501,826 to 4,067,275 | 143,528 | 1 / 131 | 49 | +| diagnosis-context | context | 0 of 3 | 2,509,745 | 1,596,367 to 2,644,144 | 159,385 | 7 / 122 | 43 | +| diagnosis-kithmoot | plain | 3 of 3 | 236,355 | 192,110 to 367,797 | 57,582 | 0 / 46 | 16 | +| diagnosis-kithmoot | graphify | 3 of 3 | 228,206 | 214,408 to 351,316 | 95,634 | 0 / 45 | 15 | +| diagnosis-kithmoot | context | 3 of 3 | 204,273 | 192,691 to 277,170 | 47,799 | 2 / 51 | 18 | +| impact-context | plain | 2 of 3 | 366,147 | 308,469 to 408,217 | 149,565 | 0 / 50 | 17 | +| impact-context | graphify | 2 of 3 | 412,812 | 294,386 to 415,340 | 78,443 | 2 / 65 | 23 | +| impact-context | context | 1 of 3 | 277,023 | 208,943 to 344,248 | 75,886 | 0 / 60 | 23 | +| impact-kithmoot | plain | 2 of 3 | 174,419 | 92,415 to 177,550 | 68,637 | 1 / 40 | 13 | +| impact-kithmoot | graphify | 2 of 3 | 116,087 | 104,861 to 156,080 | 73,923 | 0 / 29 | 10 | +| impact-kithmoot | context | 0 of 3 | 185,361 | 170,463 to 239,603 | 32,897 | 2 / 54 | 20 | +| code-change-context | plain | 3 of 3 | 407,972 | 213,401 to 756,290 | 112,931 | 0 / 52 | 17 | +| code-change-context | graphify | 3 of 3 | 437,707 | 407,044 to 437,962 | 105,063 | 0 / 53 | 16 | +| code-change-context | context | 3 of 3 | 1,535,828 | 249,795 to 1,783,696 | 100,576 | 3 / 100 | 42 | +| code-change-kithmoot | plain | 3 of 3 | 184,583 | 147,967 to 301,518 | 43,215 | 2 / 45 | 16 | +| code-change-kithmoot | graphify | 3 of 3 | 250,162 | 234,101 to 399,004 | 76,832 | 2 / 51 | 15 | +| code-change-kithmoot | context | 3 of 3 | 339,157 | 241,678 to 342,946 | 32,649 | 6 / 72 | 26 | + +## What drives the result + +- **Turns, not bytes.** Context returned fewer tool-result bytes than plain in + seven of eight tasks, but took more turns in seven. Each turn resends the + conversation, so input follows turns. The Context arm averaged 24 tool calls + per session against about 20; its documented workflow alone adds about 3.4 + (status, refresh and coverage), before explore (3.8) and packets (6.2). +- **One task is noisy.** code-change-context Context sessions took 42, 46 and + 15 turns (1.54M, 1.78M and 0.25M input). Two long sessions account for most + of that task's ratio. +- **Remaining tool errors** (29 in 578 Context tool results, against about + one in five on S5): 13 packets gave only a `path`, meaning "the whole file"; + 4 overlapping ranges; 2 over the 64 KiB response cap; 2 stale after the + model's own edit; the rest in Bash (counts by message are approximate). +- **Acceptance is set by synthesis.** Most rejections in every arm are single + rubric points: for example, in the first repetition, all three diagnosis-context arms named the right + repair but not "return unknown when re-inspection fails", and all three + orientation-kithmoot arms missed the same `verifyDelegation` ordering point. + The retrieval analysis (`../retrieval-recall-20260923/`) found the required + evidence was usually retrieved in every arm. + +## What this does and does not show + +It shows that on this locked set, with DeepSeek V4 Pro through Claude Code, +Context 0.4.0-pre did not reduce executor input and was accepted less often +than either comparator. It does not measure a Claude executor, held-out tasks, +repositories outside this ecosystem, provider credit consumption or developer +time, and one repetition set of 72 cells cannot rule out effects smaller than +the spread shown above. No saving is claimed.