diff --git a/docs/experiments/REPEATED-RUNS.md b/docs/experiments/REPEATED-RUNS.md index 5d3d2c0..65e4e75 100644 --- a/docs/experiments/REPEATED-RUNS.md +++ b/docs/experiments/REPEATED-RUNS.md @@ -20,6 +20,8 @@ the changed tool was never called. Every later comparison uses this design. - **Code-change acceptance.** Checker plus the deterministic scope check (`"codeAcceptance": "checker-and-scope"`, see `code-acceptance-20260923/`); the model reviewer judges structured answers only. +- **Structured rubrics.** Version 2 (`"rubricDir": "../rubric-v2-20260923"`), with + explicit required points and review rules; not comparable with earlier verdicts. - **Arms.** Plain tools, Graphify, and Context at its current build, which includes `repository_explore` and `repository_coverage` with the exact-quote check. A category checklist in the instructions, if tested, is a fourth arm diff --git a/docs/experiments/graphify-20260922/run.mjs b/docs/experiments/graphify-20260922/run.mjs index 66ac050..3b00f8b 100644 --- a/docs/experiments/graphify-20260922/run.mjs +++ b/docs/experiments/graphify-20260922/run.mjs @@ -2,7 +2,8 @@ // Three-way retrieval comparison runner: plain tools, Graphify, Context. // Usage: node run.mjs --local /private/local.json [--protocol DIR] [--task ID | --all] [--arms plain,graphify,context] [--skip-review] // --protocol selects a directory holding protocol.json and context-instructions.txt (default: this directory). -// protocol.json may set "codeAcceptance": "checker-and-scope" to accept code tasks without the model reviewer. +// protocol.json may set "codeAcceptance": "checker-and-scope" to accept code tasks without the model reviewer, +// and "rubricDir" to review structured tasks against later rubrics (for example ../rubric-v2-20260923). // local.json (private, machine-specific): { evidence, roots: { context, kithmoot }, node, contextCli, graphifyBin, graphifyAlwaysOn } import { spawn, spawnSync } from 'node:child_process' import { createHash } from 'node:crypto' @@ -46,7 +47,9 @@ function cleanEnv(extraPath) { function loadTask(id) { const task = JSON.parse(readFileSync(join(packDir, 'tasks', `${id}.json`), 'utf8')) - const acceptance = JSON.parse(readFileSync(join(packDir, 'acceptance', `${id}.json`), 'utf8')) + // protocol.rubricDir (optional, relative to the protocol directory) supplies later structured rubrics; the checker is unchanged. + const override = protocol.rubricDir ? join(protocolDir, protocol.rubricDir, 'acceptance', `${id}.json`) : null + const acceptance = JSON.parse(readFileSync(override && existsSync(override) ? override : join(packDir, 'acceptance', `${id}.json`), 'utf8')) return { task, acceptance } } @@ -196,6 +199,11 @@ function reviewerPrompt({ task, acceptance, workspace, answer, checker, diff }) '', ] if (acceptance.kind === 'structured') { + if (acceptance.reviewerRules) { + lines.push('Review rules:') + for (const rule of acceptance.reviewerRules) lines.push(`- ${rule}`) + lines.push('') + } lines.push('Private rubric (dimension id: expected substance):') for (const [id, text] of Object.entries(acceptance.reviewerRubric)) lines.push(`- ${id}: ${text}`) lines.push('', 'Reference evidence the rubric author expected (absence is not automatic failure if the claim is supported by other cited source):') diff --git a/docs/experiments/rubric-v2-20260923/README.md b/docs/experiments/rubric-v2-20260923/README.md new file mode 100644 index 0000000..ec22047 --- /dev/null +++ b/docs/experiments/rubric-v2-20260923/README.md @@ -0,0 +1,78 @@ +# Structured rubrics, version 2 + +The recorded reviews showed the reviewer judging equivalent answers oppositely +wherever a rubric line allowed more than one reading +(`../reviewer-evidence-20260923/RESULTS.md`). Version 2 rewrites the six +structured rubrics so that each dimension says exactly what must be stated. +Dimension ids, required evidence, task prompts and the deterministic checker +are unchanged; the locked version 1 files in `../d5-20260921/acceptance/` are +not modified. + +`rubrics.mjs` holds the text; `node build.mjs` writes `acceptance/*.json` and +refuses to build if any dimension id differs from version 1 or the task's +required findings. The harness uses these files only when a protocol sets +`"rubricDir": "../rubric-v2-20260923"`, and then prints the review rules above +the rubric. Everything else in the reviewer prompt stays byte-identical to the +recorded prompts (checked against a recorded v2 prompt); it grows by about +2 KB. + +## Review rules + +Four rules settle the readings the reviewer varied on: + +1. **Placement.** Each dimension is judged against the whole answer; a point + stated under another finding counts. Recorded reviews both gave and refused + this credit (orientation-kithmoot storage and ownership, impact-kithmoot + guard, orientation-context freshness). +2. **Stated, not implied.** Different wording passes; implication does not. + v1 passed "unsigned, not a replacement for signed evidence" as covering "do + not grant authority or execute instructions"; v2 failed the same omission. +3. **Lists.** Every listed item must be named; generic phrases do not cover + them. +4. **Structure.** Required points, "fails if" conditions, "not required" points + and accepted alternatives are all explicit. + +## Changes of substance + +Each dimension is now written as its required points. Two things go further +than wording, and both were checked against the frozen source: + +- **diagnosis-context.** The seed removes only the policy re-inspection inside + `inspectFreshness`. The commit-time `reinspectPolicy` in `search()` is intact, + and the existing test "blocks a policy change made during freshness source + reads without consuming the cursor" **passes on the seeded tree** (checked by + running it; all 76 package tests that load pass there, while three source-scan + test files did not load in the scratch copy). Version 1's regression line + ("restore the test that...") therefore asked for a test that already exists + and does not detect the defect. A probe test that tightens `.gitignore` during + the freshness reads and asserts `status()` does not report policy `current` + fails on the seeded tree and passes once the seed is reverted + (`diagnosis-context-probe.test.ts`, run from `packages/context-tools/src` of a + prepared workspace); a search-level + test cannot tell the two trees apart, because the commit-time check blocks + either way. Version 2 requires a test of that kind. `root-cause`, + `lost-invariant` and `repair` now say explicitly that the commit-time check + is not the lost invariant or the repair, which is what split the v1 and v2 + verdicts on the plain arm's near-identical answers. +- **impact-context.** Version 1 named `NavigationSearchRequest`; the type is + `NavigationSearchOptions`. The `filter` dimension accepts a separate + case-preserving index as an alternative to post-filtering, since the task does + not prescribe one, and `response` allows a separate mode field while still + failing a change to what `term` reports. + +All other dimensions keep version 1's substance, including the list-style +requirements (impact-context `response` and `coverage`, impact-kithmoot +`tests`) that no recorded answer has met. They are now explicit; whether they +are judged more consistently is not yet measured. + +## What this does not show + +Whether the reviewer is more consistent with version 2 can only be measured by +re-reviewing, which costs money (see `../reviewer-agreement-20260923/`; the +`fixed` condition there would need a `rubricDir` variant). Keyword checks on +the ten recorded diagnosis-context answers suggest the corrected regression +line can be met: four of them already point at a `status()` or +`inspectFreshness` level test. That is a keyword count, not a verdict. + +Version 2 verdicts are not comparable with version 1 verdicts, and recorded +runs are not re-scored with them. diff --git a/docs/experiments/rubric-v2-20260923/acceptance/diagnosis-context.json b/docs/experiments/rubric-v2-20260923/acceptance/diagnosis-context.json new file mode 100644 index 0000000..d3d6e3f --- /dev/null +++ b/docs/experiments/rubric-v2-20260923/acceptance/diagnosis-context.json @@ -0,0 +1,40 @@ +{ + "version": 2, + "checkerSha256": "539b5d7cd593f0dd6c5707b3426e243e90092614921c8024b8a69a5c49ffaa9c", + "kind": "structured", + "requiredFindingIds": [ + "root-cause", + "lost-invariant", + "why-first-check-fails", + "repair", + "regression" + ], + "reviewerRubric": { + "root-cause": "Required: inspectFreshness no longer re-inspects the policy after buildManifest has read the eligible source bytes, so it can report policy \"current\" after a change made during those reads. Fails if: the root cause is placed only in search() or its commit phase, or the answer says there is no source defect.", + "lost-invariant": "Required: a policy change during manifest construction (the freshness source reads) must make freshness inspection report the generation stale, or unknown, before search proceeds with old indexed results. Accepted alternative: freshness inspection must re-verify the policy after the source reads before reporting it current. Fails if: the invariant is placed only at search commit time; the commit-time re-check in search() is intact in the seeded tree, so that invariant was not lost.", + "why-first-check-fails": "Required: the discovery policy comparison in inspectFreshness runs before the source reads, so it cannot observe a policy change made during them.", + "repair": "Required: restore a re-inspection of the same policy directories (reinspectPolicy over discovery.policyDirectories) inside inspectFreshness after buildManifest, returning stale when the revision differs and unknown when re-inspection fails, before the manifest is used. Fails if: the existing commit-phase reinspectPolicy in search() is named as the repair, or the answer says no source change is needed; that check is present in the seeded tree and does not correct status().", + "regression": "Required: a test that fails on the seeded tree: tighten .gitignore while a source file handle is opened during freshness inspection and assert that status() (or inspectFreshness) reports the policy as stale or unknown, not current. Fails if: the only test proposed is the existing \"blocks a policy change made during freshness source reads without consuming the cursor\" search test, which passes on the seeded tree because the commit-time re-check still blocks the search. Keeping that test as well is fine." + }, + "requiredEvidence": [ + { + "path": "packages/context-tools/src/repository-navigation.ts", + "token": "buildManifest(discovery, signal)" + }, + { + "path": "packages/context-tools/src/repository-navigation.ts", + "token": "reinspectPolicy" + }, + { + "path": "packages/context-tools/src/repository-navigation.test.ts", + "token": "blocks a policy change made during freshness source reads" + } + ], + "basedOn": "d5-20260921/acceptance/diagnosis-context.json", + "reviewerRules": [ + "Judge each dimension against the whole answer: the summary and every finding. A required point stated under a different finding id counts; never fail a dimension for placement alone.", + "A required point passes only when the answer states it explicitly, in words a reader could quote. Different wording is fine; a point that is only implied by other statements does not pass.", + "Where a dimension lists items, each listed item must be named. A generic phrase such as \"edge cases\" or \"all semantics unchanged\" does not cover named items.", + "A dimension passes when every required point is stated, no \"fails if\" condition applies and nothing it states contradicts the frozen source excerpts. Points marked \"not required\" never cause a failure. Accepted alternatives pass equally." + ] +} diff --git a/docs/experiments/rubric-v2-20260923/acceptance/diagnosis-kithmoot.json b/docs/experiments/rubric-v2-20260923/acceptance/diagnosis-kithmoot.json new file mode 100644 index 0000000..a47cf34 --- /dev/null +++ b/docs/experiments/rubric-v2-20260923/acceptance/diagnosis-kithmoot.json @@ -0,0 +1,40 @@ +{ + "version": 2, + "checkerSha256": "539b5d7cd593f0dd6c5707b3426e243e90092614921c8024b8a69a5c49ffaa9c", + "kind": "structured", + "requiredFindingIds": [ + "root-cause", + "trust-impact", + "repair-type", + "repair-merge", + "regression" + ], + "reviewerRubric": { + "root-cause": "Required, both: (a) the seeded ContextVaultOptions is PortableOptions and so admits verifyDelegation; (b) kithmootContextOptions spreads caller options after installing KithMoot's verifier ({ verifyDelegation, ...options }), so a caller-supplied verifier overwrites it.", + "trust-impact": "Required: a caller-supplied verifier can approve forged or unrelated agent ownership, so grants that should be refused are accepted and the room adapter's grant boundary is weakened.", + "repair-type": "Required: ContextVaultOptions becomes Omit again.", + "repair-merge": "Required: kithmootContextOptions spreads caller options first and installs KithMoot's verifyDelegation last ({ ...options, verifyDelegation }).", + "regression": "Required, both: (a) the test passes a hostile verifyDelegation through an unsafe cast (for example `as any` or `as ContextVaultOptions`), which is needed because the repaired type no longer admits it; the cast must be stated; (b) it shows KithMoot's verifier is used rather than the hostile one, either by asserting what kithmootContextOptions returns or by showing that a forged proof the hostile verifier would approve is still rejected." + }, + "requiredEvidence": [ + { + "path": "src/context.ts", + "token": "kithmootContextOptions" + }, + { + "path": "src/context.ts", + "token": "verifyAgentOwnership" + }, + { + "path": "src/context.test.ts", + "token": "rejects forged ownership" + } + ], + "basedOn": "d5-20260921/acceptance/diagnosis-kithmoot.json", + "reviewerRules": [ + "Judge each dimension against the whole answer: the summary and every finding. A required point stated under a different finding id counts; never fail a dimension for placement alone.", + "A required point passes only when the answer states it explicitly, in words a reader could quote. Different wording is fine; a point that is only implied by other statements does not pass.", + "Where a dimension lists items, each listed item must be named. A generic phrase such as \"edge cases\" or \"all semantics unchanged\" does not cover named items.", + "A dimension passes when every required point is stated, no \"fails if\" condition applies and nothing it states contradicts the frozen source excerpts. Points marked \"not required\" never cause a failure. Accepted alternatives pass equally." + ] +} diff --git a/docs/experiments/rubric-v2-20260923/acceptance/impact-context.json b/docs/experiments/rubric-v2-20260923/acceptance/impact-context.json new file mode 100644 index 0000000..95ef84f --- /dev/null +++ b/docs/experiments/rubric-v2-20260923/acceptance/impact-context.json @@ -0,0 +1,50 @@ +{ + "version": 2, + "checkerSha256": "539b5d7cd593f0dd6c5707b3426e243e90092614921c8024b8a69a5c49ffaa9c", + "kind": "structured", + "requiredFindingIds": [ + "request-surface", + "default", + "filter", + "cursor", + "response", + "coverage" + ], + "reviewerRubric": { + "request-surface": "Required, both: optional caseSensitive is added to (a) the search options type (NavigationSearchOptions in repository-navigation.ts) and (b) the repository_search MCP input schema.", + "default": "Required: omitted or false keeps today's behaviour: the term is normalised by normalizeTerm (lower-cased) and looked up in the case-insensitive postings, so existing callers see no change.", + "filter": "Required: the postings are lower-case, so case-sensitive mode must filter candidate source lines for the exact-case token. Accepted alternatives: reuse the lower-case postings and post-filter the candidate lines, or add a separate case-preserving index, provided exact-case filtering of results is stated.", + "cursor": "Required, both: (a) caseSensitive is stored in the Cursor; (b) continuing a cursor with a different mode is rejected.", + "response": "Required: the flag leaves existing response semantics unchanged, naming each of: the response term, freshness, policy, byte accounting, result count, visit count, cancellation and cursor consumption. Adding a separate field that echoes the mode is allowed. Fails if: the answer proposes changing what the existing term field reports (for example echoing the raw-case term).", + "coverage": "Required, all five named: default mixed-case matching, exact-case filtering, MCP schema validation of caseSensitive, pagination within one mode, and cross-mode cursor rejection." + }, + "requiredEvidence": [ + { + "path": "packages/context-tools/src/repository-navigation.ts", + "token": "export interface NavigationSearchOptions" + }, + { + "path": "packages/context-tools/src/repository-navigation.ts", + "token": "interface Cursor" + }, + { + "path": "packages/context-tools/src/repository-navigation-mcp.ts", + "token": "repository_search" + }, + { + "path": "packages/context-tools/src/repository-navigation.test.ts", + "token": "wrong-term, stale, and unknown cursors" + }, + { + "path": "docs/LOCAL-NAVIGATION.md", + "token": "case-insensitive" + } + ], + "basedOn": "d5-20260921/acceptance/impact-context.json", + "reviewerRules": [ + "Judge each dimension against the whole answer: the summary and every finding. A required point stated under a different finding id counts; never fail a dimension for placement alone.", + "A required point passes only when the answer states it explicitly, in words a reader could quote. Different wording is fine; a point that is only implied by other statements does not pass.", + "Where a dimension lists items, each listed item must be named. A generic phrase such as \"edge cases\" or \"all semantics unchanged\" does not cover named items.", + "A dimension passes when every required point is stated, no \"fails if\" condition applies and nothing it states contradicts the frozen source excerpts. Points marked \"not required\" never cause a failure. Accepted alternatives pass equally." + ] +} diff --git a/docs/experiments/rubric-v2-20260923/acceptance/impact-kithmoot.json b/docs/experiments/rubric-v2-20260923/acceptance/impact-kithmoot.json new file mode 100644 index 0000000..19f67c3 --- /dev/null +++ b/docs/experiments/rubric-v2-20260923/acceptance/impact-kithmoot.json @@ -0,0 +1,50 @@ +{ + "version": 2, + "checkerSha256": "539b5d7cd593f0dd6c5707b3426e243e90092614921c8024b8a69a5c49ffaa9c", + "kind": "structured", + "requiredFindingIds": [ + "contract", + "callers", + "privacy", + "tests", + "guard", + "compatibility" + ], + "reviewerRubric": { + "contract": "Required: shortId returns the first eight characters only for exactly 64 lower-case hexadecimal characters, and [invalid-id] for every other input.", + "callers": "Required: server/forwarder.mjs passes config.roomId and config.pubkey through shortId before startup logging.", + "privacy": "Required, both: (a) the invalid marker is a constant; stating that the literal [invalid-id] is returned for all invalid input satisfies this; (b) it echoes no part of the input, including no prefix.", + "tests": "Required, both: (a) the existing permissive expectations (\"never returns more than it was given\", which returns 'abcd' and '' unchanged) must be replaced; (b) new cases cover each of: empty, short, long, upper-case, non-hex and Unicode input. All six classes must be named.", + "guard": "Required, both: (a) the log-redaction scan (test/log-redaction-scan.test.ts) is retained and still rejects direct full-identifier logging; (b) the forwarder startup logging assertion is retained. Fails if: either is dismissed as unrelated or not mentioned.", + "compatibility": "Required, both: (a) valid canonical 64-character lower-case hex identifiers keep the same eight-character display; (b) malformed configured identifiers now display as [invalid-id], visibly invalid, instead of a prefix." + }, + "requiredEvidence": [ + { + "path": "src/log-redact.ts", + "token": "export function shortId" + }, + { + "path": "server/forwarder.mjs", + "token": "shortId(config.roomId)" + }, + { + "path": "server/forwarder.mjs", + "token": "shortId(config.pubkey)" + }, + { + "path": "src/log-redact.test.ts", + "token": "never returns more than it was given" + }, + { + "path": "test/log-redaction-scan.test.ts", + "token": "shortId" + } + ], + "basedOn": "d5-20260921/acceptance/impact-kithmoot.json", + "reviewerRules": [ + "Judge each dimension against the whole answer: the summary and every finding. A required point stated under a different finding id counts; never fail a dimension for placement alone.", + "A required point passes only when the answer states it explicitly, in words a reader could quote. Different wording is fine; a point that is only implied by other statements does not pass.", + "Where a dimension lists items, each listed item must be named. A generic phrase such as \"edge cases\" or \"all semantics unchanged\" does not cover named items.", + "A dimension passes when every required point is stated, no \"fails if\" condition applies and nothing it states contradicts the frozen source excerpts. Points marked \"not required\" never cause a failure. Accepted alternatives pass equally." + ] +} diff --git a/docs/experiments/rubric-v2-20260923/acceptance/orientation-context.json b/docs/experiments/rubric-v2-20260923/acceptance/orientation-context.json new file mode 100644 index 0000000..82ef63e --- /dev/null +++ b/docs/experiments/rubric-v2-20260923/acceptance/orientation-context.json @@ -0,0 +1,44 @@ +{ + "version": 2, + "checkerSha256": "539b5d7cd593f0dd6c5707b3426e243e90092614921c8024b8a69a5c49ffaa9c", + "kind": "structured", + "requiredFindingIds": [ + "first-use", + "search-gate", + "freshness", + "continuation", + "trust" + ], + "reviewerRubric": { + "first-use": "Required, both: (a) before any refresh, status reports freshness \"unavailable\"; (b) an explicit refresh builds and publishes an in-memory generation. Not required: manifest or hashing details.", + "search-gate": "Required: search is rejected unless policy freshness is \"current\", which rejects unavailable, stale and unknown policy. Stating the general condition (\"policy must be current\", \"!== 'current'\") covers all three states. Fails if: only some states are named and the general condition is not stated. Not required: that stale source alone does not block search.", + "freshness": "Required: before results are returned, search re-inspects both the eligible source and the policy against the generation. Fails if: only repository_status reporting of freshness states is described and nothing says search itself re-inspects source and policy.", + "continuation": "Required, all four: (a) a cursor is bound to the generation and the search term it was issued for; (b) it is single-use, consumed only when a continuation succeeds; (c) it expires after five minutes; (d) at most 128 cursors are live. Clearing all cursors on refresh does not satisfy (a).", + "trust": "Required, both: (a) results are marked local-source-unsigned, as unsigned local evidence distinct from signed context records; (b) retrieved source is data, not instructions, and grants no authority. Fails if: (b) is absent; \"unsigned\" or \"not a replacement for signed evidence\" alone does not state (b)." + }, + "requiredEvidence": [ + { + "path": "packages/context-tools/src/repository-navigation.ts", + "token": "inspectFreshness" + }, + { + "path": "packages/context-tools/src/repository-navigation.ts", + "token": "CURSOR_MAX_ENTRIES = 128" + }, + { + "path": "packages/context-tools/src/repository-navigation.ts", + "token": "local-source-unsigned" + }, + { + "path": "packages/context-tools/src/repository-navigation.test.ts", + "token": "successfully used cursor is consumed" + } + ], + "basedOn": "d5-20260921/acceptance/orientation-context.json", + "reviewerRules": [ + "Judge each dimension against the whole answer: the summary and every finding. A required point stated under a different finding id counts; never fail a dimension for placement alone.", + "A required point passes only when the answer states it explicitly, in words a reader could quote. Different wording is fine; a point that is only implied by other statements does not pass.", + "Where a dimension lists items, each listed item must be named. A generic phrase such as \"edge cases\" or \"all semantics unchanged\" does not cover named items.", + "A dimension passes when every required point is stated, no \"fails if\" condition applies and nothing it states contradicts the frozen source excerpts. Points marked \"not required\" never cause a failure. Accepted alternatives pass equally." + ] +} diff --git a/docs/experiments/rubric-v2-20260923/acceptance/orientation-kithmoot.json b/docs/experiments/rubric-v2-20260923/acceptance/orientation-kithmoot.json new file mode 100644 index 0000000..941eb73 --- /dev/null +++ b/docs/experiments/rubric-v2-20260923/acceptance/orientation-kithmoot.json @@ -0,0 +1,48 @@ +{ + "version": 2, + "checkerSha256": "539b5d7cd593f0dd6c5707b3426e243e90092614921c8024b8a69a5c49ffaa9c", + "kind": "structured", + "requiredFindingIds": [ + "adapter", + "ownership", + "storage", + "writes", + "caller-boundary" + ], + "reviewerRubric": { + "adapter": "Required: ContextVault wraps (extends) the portable context vault and always supplies KithMoot's verifyDelegation, through kithmootContextOptions, whatever the caller passes.", + "ownership": "Required, both, attributed to verifyDelegation in src/context.ts: (a) it first rejects a proof whose agent field does not equal options.agent; (b) it then calls verifyAgentOwnership. Fails if: only verifyAgentOwnership's internal checks are described, or the agent check is attributed only to verifyAgentOwnership.", + "storage": "Required: ContextFileStore in src/node/context-store.ts extends or wraps the portable file store and constructs it with kithmootContextOptions(options).", + "writes": "Required: all five tools are named as entering store.run with the write flag: context_create, context_append, context_import, context_upload and context_set_grants. Fails if: any of the five is missing. Not required: read-only tools.", + "caller-boundary": "Required: ContextVaultOptions is PortableOptions with verifyDelegation omitted (Omit), so callers cannot pass or select the room's trust policy." + }, + "requiredEvidence": [ + { + "path": "src/context.ts", + "token": "Omit" + }, + { + "path": "src/context.ts", + "token": "return { ...options, verifyDelegation }" + }, + { + "path": "src/node/context-store.ts", + "token": "kithmootContextOptions(options)" + }, + { + "path": "packages/context-tools/src/context-mcp.ts", + "token": "store.run(v => v.append" + }, + { + "path": "src/context.test.ts", + "token": "rejects forged ownership" + } + ], + "basedOn": "d5-20260921/acceptance/orientation-kithmoot.json", + "reviewerRules": [ + "Judge each dimension against the whole answer: the summary and every finding. A required point stated under a different finding id counts; never fail a dimension for placement alone.", + "A required point passes only when the answer states it explicitly, in words a reader could quote. Different wording is fine; a point that is only implied by other statements does not pass.", + "Where a dimension lists items, each listed item must be named. A generic phrase such as \"edge cases\" or \"all semantics unchanged\" does not cover named items.", + "A dimension passes when every required point is stated, no \"fails if\" condition applies and nothing it states contradicts the frozen source excerpts. Points marked \"not required\" never cause a failure. Accepted alternatives pass equally." + ] +} diff --git a/docs/experiments/rubric-v2-20260923/build.mjs b/docs/experiments/rubric-v2-20260923/build.mjs new file mode 100644 index 0000000..f600537 --- /dev/null +++ b/docs/experiments/rubric-v2-20260923/build.mjs @@ -0,0 +1,21 @@ +#!/usr/bin/env node +// Write acceptance/*.json (version 2) from rubrics.mjs and the locked version 1 files, and check that +// everything except the rubric text is unchanged. Usage: node build.mjs +import { readFileSync, writeFileSync } from 'node:fs' +import { dirname, join, resolve } from 'node:path' +import { fileURLToPath } from 'node:url' +import { reviewerRules, rubrics } from './rubrics.mjs' + +const here = dirname(fileURLToPath(import.meta.url)) +const pack = resolve(here, '../d5-20260921') +for (const [id, rubric] of Object.entries(rubrics)) { + const v1 = JSON.parse(readFileSync(join(pack, 'acceptance', `${id}.json`), 'utf8')) + const task = JSON.parse(readFileSync(join(pack, 'tasks', `${id}.json`), 'utf8')) + const ids = Object.keys(rubric) + if (v1.kind !== 'structured') throw new Error(`${id}: not a structured task`) + if (JSON.stringify(ids) !== JSON.stringify(task.requiredFindingIds) || JSON.stringify(ids) !== JSON.stringify(Object.keys(v1.reviewerRubric))) throw new Error(`${id}: dimension ids differ from version 1`) + for (const [dim, text] of Object.entries(rubric)) if (!text.startsWith('Required')) throw new Error(`${id}/${dim}: rubric text must start with its required points`) + const v2 = { ...v1, version: 2, basedOn: `d5-20260921/acceptance/${id}.json`, reviewerRules, reviewerRubric: rubric } + writeFileSync(join(here, 'acceptance', `${id}.json`), `${JSON.stringify(v2, null, 2)}\n`) +} +process.stdout.write(`wrote ${Object.keys(rubrics).length} acceptance files\n`) diff --git a/docs/experiments/rubric-v2-20260923/diagnosis-context-probe.test.ts b/docs/experiments/rubric-v2-20260923/diagnosis-context-probe.test.ts new file mode 100644 index 0000000..1bb65db --- /dev/null +++ b/docs/experiments/rubric-v2-20260923/diagnosis-context-probe.test.ts @@ -0,0 +1,48 @@ +// Ground-truth probe for the diagnosis-context rubric. Copy into packages/context-tools/src of a prepared +// diagnosis-context workspace and run with vitest: the status test fails on the seeded tree and passes once +// setup/diagnosis-context.patch is reverted; the search test passes on both. +import { describe, expect, it, vi } from 'vitest'; +import { promises as fsp } from 'node:fs'; +import path from 'node:path'; +import os from 'node:os'; +import { RepositoryNavigation } from './repository-navigation.js'; + +async function fixture() { + const root = await fsp.mkdtemp(path.join(os.tmpdir(), 'probe-')); + await fsp.writeFile(path.join(root, 'secret.ts'), 'shared one\nshared two\n'); + return root; +} +function tightenOnOpen(root: string, source: string) { + const originalOpen = fsp.open.bind(fsp); + let tightened = false; + vi.spyOn(fsp, 'open').mockImplementation(async (file: any, flags: any) => { + const handle = await originalOpen(file, flags); + if (!tightened && String(file) === source) { tightened = true; await fsp.writeFile(path.join(root, '.gitignore'), 'secret.ts\n'); } + return handle; + }); +} +describe('probe', () => { + it('status reports policy change made during freshness source reads', async () => { + const root = await fixture(); + const source = await fsp.realpath(path.join(root, 'secret.ts')); + const nav = new RepositoryNavigation(root); + await nav.refresh(); + tightenOnOpen(root, source); + const status = await nav.status(); + vi.restoreAllMocks(); + expect(status.policy.freshness).not.toBe('current'); + }); + it('search blocks at the start check, before traversal', async () => { + const root = await fixture(); + const source = await fsp.realpath(path.join(root, 'secret.ts')); + const nav = new RepositoryNavigation(root); + await nav.refresh(); + const first = await nav.search({ term: 'shared', maxResults: 1 }); + tightenOnOpen(root, source); + const err = await nav.search({ term: 'shared', cursor: first.nextCursor! }).catch((e) => e); + const opens = (fsp.open as any).mock.calls.filter((c: any[]) => String(c[0]) === source).length; + vi.restoreAllMocks(); + expect(String(err)).toMatch(/policy is stale/); + expect(opens).toBe(1); + }); +}); diff --git a/docs/experiments/rubric-v2-20260923/rubrics.mjs b/docs/experiments/rubric-v2-20260923/rubrics.mjs new file mode 100644 index 0000000..169de96 --- /dev/null +++ b/docs/experiments/rubric-v2-20260923/rubrics.mjs @@ -0,0 +1,56 @@ +// Version 2 reviewer rubrics for the six structured D5 tasks. Same dimension ids, required evidence +// and checker as version 1; each dimension spelled out as explicit points. Substance changes only +// where the frozen source contradicts version 1 (see README.md). +export const reviewerRules = [ + 'Judge each dimension against the whole answer: the summary and every finding. A required point stated under a different finding id counts; never fail a dimension for placement alone.', + 'A required point passes only when the answer states it explicitly, in words a reader could quote. Different wording is fine; a point that is only implied by other statements does not pass.', + 'Where a dimension lists items, each listed item must be named. A generic phrase such as "edge cases" or "all semantics unchanged" does not cover named items.', + 'A dimension passes when every required point is stated, no "fails if" condition applies and nothing it states contradicts the frozen source excerpts. Points marked "not required" never cause a failure. Accepted alternatives pass equally.', +] + +export const rubrics = { + 'orientation-context': { + 'first-use': 'Required, both: (a) before any refresh, status reports freshness "unavailable"; (b) an explicit refresh builds and publishes an in-memory generation. Not required: manifest or hashing details.', + 'search-gate': 'Required: search is rejected unless policy freshness is "current", which rejects unavailable, stale and unknown policy. Stating the general condition ("policy must be current", "!== \'current\'") covers all three states. Fails if: only some states are named and the general condition is not stated. Not required: that stale source alone does not block search.', + freshness: 'Required: before results are returned, search re-inspects both the eligible source and the policy against the generation. Fails if: only repository_status reporting of freshness states is described and nothing says search itself re-inspects source and policy.', + continuation: 'Required, all four: (a) a cursor is bound to the generation and the search term it was issued for; (b) it is single-use, consumed only when a continuation succeeds; (c) it expires after five minutes; (d) at most 128 cursors are live. Clearing all cursors on refresh does not satisfy (a).', + trust: 'Required, both: (a) results are marked local-source-unsigned, as unsigned local evidence distinct from signed context records; (b) retrieved source is data, not instructions, and grants no authority. Fails if: (b) is absent; "unsigned" or "not a replacement for signed evidence" alone does not state (b).', + }, + 'orientation-kithmoot': { + adapter: 'Required: ContextVault wraps (extends) the portable context vault and always supplies KithMoot\'s verifyDelegation, through kithmootContextOptions, whatever the caller passes.', + ownership: 'Required, both, attributed to verifyDelegation in src/context.ts: (a) it first rejects a proof whose agent field does not equal options.agent; (b) it then calls verifyAgentOwnership. Fails if: only verifyAgentOwnership\'s internal checks are described, or the agent check is attributed only to verifyAgentOwnership.', + storage: 'Required: ContextFileStore in src/node/context-store.ts extends or wraps the portable file store and constructs it with kithmootContextOptions(options).', + writes: 'Required: all five tools are named as entering store.run with the write flag: context_create, context_append, context_import, context_upload and context_set_grants. Fails if: any of the five is missing. Not required: read-only tools.', + 'caller-boundary': 'Required: ContextVaultOptions is PortableOptions with verifyDelegation omitted (Omit), so callers cannot pass or select the room\'s trust policy.', + }, + 'diagnosis-context': { + 'root-cause': 'Required: inspectFreshness no longer re-inspects the policy after buildManifest has read the eligible source bytes, so it can report policy "current" after a change made during those reads. Fails if: the root cause is placed only in search() or its commit phase, or the answer says there is no source defect.', + 'lost-invariant': 'Required: a policy change during manifest construction (the freshness source reads) must make freshness inspection report the generation stale, or unknown, before search proceeds with old indexed results. Accepted alternative: freshness inspection must re-verify the policy after the source reads before reporting it current. Fails if: the invariant is placed only at search commit time; the commit-time re-check in search() is intact in the seeded tree, so that invariant was not lost.', + 'why-first-check-fails': 'Required: the discovery policy comparison in inspectFreshness runs before the source reads, so it cannot observe a policy change made during them.', + repair: 'Required: restore a re-inspection of the same policy directories (reinspectPolicy over discovery.policyDirectories) inside inspectFreshness after buildManifest, returning stale when the revision differs and unknown when re-inspection fails, before the manifest is used. Fails if: the existing commit-phase reinspectPolicy in search() is named as the repair, or the answer says no source change is needed; that check is present in the seeded tree and does not correct status().', + regression: 'Required: a test that fails on the seeded tree: tighten .gitignore while a source file handle is opened during freshness inspection and assert that status() (or inspectFreshness) reports the policy as stale or unknown, not current. Fails if: the only test proposed is the existing "blocks a policy change made during freshness source reads without consuming the cursor" search test, which passes on the seeded tree because the commit-time re-check still blocks the search. Keeping that test as well is fine.', + }, + 'diagnosis-kithmoot': { + 'root-cause': 'Required, both: (a) the seeded ContextVaultOptions is PortableOptions and so admits verifyDelegation; (b) kithmootContextOptions spreads caller options after installing KithMoot\'s verifier ({ verifyDelegation, ...options }), so a caller-supplied verifier overwrites it.', + 'trust-impact': 'Required: a caller-supplied verifier can approve forged or unrelated agent ownership, so grants that should be refused are accepted and the room adapter\'s grant boundary is weakened.', + 'repair-type': 'Required: ContextVaultOptions becomes Omit again.', + 'repair-merge': 'Required: kithmootContextOptions spreads caller options first and installs KithMoot\'s verifyDelegation last ({ ...options, verifyDelegation }).', + regression: 'Required, both: (a) the test passes a hostile verifyDelegation through an unsafe cast (for example `as any` or `as ContextVaultOptions`), which is needed because the repaired type no longer admits it; the cast must be stated; (b) it shows KithMoot\'s verifier is used rather than the hostile one, either by asserting what kithmootContextOptions returns or by showing that a forged proof the hostile verifier would approve is still rejected.', + }, + 'impact-context': { + 'request-surface': 'Required, both: optional caseSensitive is added to (a) the search options type (NavigationSearchOptions in repository-navigation.ts) and (b) the repository_search MCP input schema.', + default: 'Required: omitted or false keeps today\'s behaviour: the term is normalised by normalizeTerm (lower-cased) and looked up in the case-insensitive postings, so existing callers see no change.', + filter: 'Required: the postings are lower-case, so case-sensitive mode must filter candidate source lines for the exact-case token. Accepted alternatives: reuse the lower-case postings and post-filter the candidate lines, or add a separate case-preserving index, provided exact-case filtering of results is stated.', + cursor: 'Required, both: (a) caseSensitive is stored in the Cursor; (b) continuing a cursor with a different mode is rejected.', + response: 'Required: the flag leaves existing response semantics unchanged, naming each of: the response term, freshness, policy, byte accounting, result count, visit count, cancellation and cursor consumption. Adding a separate field that echoes the mode is allowed. Fails if: the answer proposes changing what the existing term field reports (for example echoing the raw-case term).', + coverage: 'Required, all five named: default mixed-case matching, exact-case filtering, MCP schema validation of caseSensitive, pagination within one mode, and cross-mode cursor rejection.', + }, + 'impact-kithmoot': { + contract: 'Required: shortId returns the first eight characters only for exactly 64 lower-case hexadecimal characters, and [invalid-id] for every other input.', + callers: 'Required: server/forwarder.mjs passes config.roomId and config.pubkey through shortId before startup logging.', + privacy: 'Required, both: (a) the invalid marker is a constant; stating that the literal [invalid-id] is returned for all invalid input satisfies this; (b) it echoes no part of the input, including no prefix.', + tests: 'Required, both: (a) the existing permissive expectations ("never returns more than it was given", which returns \'abcd\' and \'\' unchanged) must be replaced; (b) new cases cover each of: empty, short, long, upper-case, non-hex and Unicode input. All six classes must be named.', + guard: 'Required, both: (a) the log-redaction scan (test/log-redaction-scan.test.ts) is retained and still rejects direct full-identifier logging; (b) the forwarder startup logging assertion is retained. Fails if: either is dismissed as unrelated or not mentioned.', + compatibility: 'Required, both: (a) valid canonical 64-character lower-case hex identifiers keep the same eight-character display; (b) malformed configured identifiers now display as [invalid-id], visibly invalid, instead of a prefix.', + }, +}