diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 75bac6b05..ee2c6cd75 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,6 +39,12 @@ jobs: package-manager-cache: false node-version-file: ".nvmrc" + # Needs no dependencies, so it runs before install and fails in ~1s. + # Asserts every docs/START_HERE.md line citation and every .tours/ step + # still points at the text it claims — not merely at a line that exists. + - name: Walkthrough citations + run: node scripts/validate-tours.mjs + - name: Install dependencies run: npm install --no-audit --no-fund diff --git a/AGENTS.md b/AGENTS.md index e23983059..f4ee7d9c8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,5 +1,12 @@ # NodeBench Agent Workflow +**New to this repository? Read `docs/START_HERE.md` first.** It follows one +question from the keypress to the answer, step by step through the real files, +and tells you what you can run without a Convex deployment. Everything below +assumes you already know that path. Its line citations and the CodeTour steps +in `.tours/` are checked by `node scripts/validate-tours.mjs` — run that after +moving any code those documents point at. + ## Agent coordination (Codex ↔ Claude) — read FIRST `AGENT_COORDINATION.md` (repo root) is the live ledger of **who is editing what right now** diff --git a/CLAUDE.md b/CLAUDE.md index 14eaab4de..3fc853d6b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,5 +1,12 @@ # NodeBench AI — Claude Code Project Instructions +**New to this repository? Read `docs/START_HERE.md` first.** It follows one +question from the keypress to the answer, step by step through the real files, +and tells you what you can run without a Convex deployment. Everything below +assumes you already know that path. Its line citations and the CodeTour steps +in `.tours/` are checked by `node scripts/validate-tours.mjs` — run that after +moving any code those documents point at. + ## Agent coordination (read FIRST when other agents may be active) `AGENT_COORDINATION.md` (repo root) is the live ledger of **who is editing what right now** diff --git a/docs/SIMPLIFICATION_REPORT.md b/docs/SIMPLIFICATION_REPORT.md index d04544c47..8d3cf4105 100644 --- a/docs/SIMPLIFICATION_REPORT.md +++ b/docs/SIMPLIFICATION_REPORT.md @@ -39,7 +39,7 @@ appear in search results but are dead. Those are what moved. | Typecheck | **not re-measured in this pass** — `promotion/PROMOTION_LOG.md` records exit 2 / 5,383 errors at the Wave 1 baseline | exit 2 · **5,378** errors | no delta claimed | `npx tsc -p tsconfig.app.json --noEmit --pretty false 2>&1 \| grep -c "error TS"` | | Browser workflow passes | not run — needs a Convex deployment | not run — same reason | — | `npx playwright test evals/e2e/one-flow-regression.spec.ts` (requires `VITE_CONVEX_URL`) | | Production bundle size | exit 0 · PWA precache 338 entries / 22,522.18 KiB | exit 0 · PWA precache 338 entries / **22,522.06 KiB** | −0.12 KiB | `npm run build` | -| CodeTour steps resolving | n/a (no tours existed) | 27 / 27 | +27 | `node scripts/validate-tours.mjs` | +| CodeTour steps resolving | n/a (no tours existed) | 27 / 27 | +27 | `node scripts/validate-tours.mjs` (the validator has since grown to cover `docs/START_HERE.md` citations too, so the same command now reports a larger total — 27 of them are still the tour steps counted here) | | Additions/deletions | — | — | 34 files, +1,700 / −1,885 | `git diff --shortstat` | diff --git a/docs/START_HERE.md b/docs/START_HERE.md index c3f0e3778..fc6f906c9 100644 --- a/docs/START_HERE.md +++ b/docs/START_HERE.md @@ -59,7 +59,7 @@ tests the URL for **validity**, not merely for presence. **Core code** ```tsx -// apps/web/src/main.tsx:156 +// apps/web/src/main.tsx:156 — const convexUrl = configuredConvexUrl() const convexUrl = configuredConvexUrl(); const convex = convexUrl ? new ConvexReactClient(convexUrl) : null; ``` @@ -90,7 +90,7 @@ before you go looking for them. **Core code** ```tsx -// apps/web/src/App.tsx:209 +// apps/web/src/App.tsx:209 — const isRedesignRoute const isRedesignRoute = location.pathname === "/redesign" || location.pathname.startsWith("/redesign/"); if (isRedesignRoute) { return ( @@ -124,7 +124,7 @@ does not know what a run is. **Core code** ```tsx -// apps/web/src/features/redesign/components/UniversalComposer.tsx:248 +// apps/web/src/features/redesign/components/UniversalComposer.tsx:248 — const handleSubmit const handleSubmit = (mode: ComposerMode = "research") => { const trimmed = text.trim(); if (!trimmed || streaming) return; @@ -160,7 +160,7 @@ immediately with a plain-English reason. **Core code** ```tsx -// apps/web/src/features/redesign/surfaces/ChatSurface.tsx:637 +// apps/web/src/features/redesign/surfaces/ChatSurface.tsx:637 — const sendMessage const sendMessage = (text: string, submittedTier: RouterTier) => { const canRunLiveChat = chatRun.state.available && !_skipLiveSeed; // ... push the user turn + an assistant turn (thinking, or the reason it can't run) @@ -185,7 +185,7 @@ anonymous account). No network call is made. **File:** `backend/convex/domains/redesign/chatRuns.ts` **Symbol:** `startChat` (a public Convex `mutation`) -**Called by:** `useRedesignChatRun.submit` (`apps/web/src/features/redesign/hooks/useRedesignChatRun.ts:501`) +**Called by:** `useRedesignChatRun.submit` in `apps/web/src/features/redesign/hooks/useRedesignChatRun.ts` **Calls next:** `ctx.scheduler.runAfter(0, internal…chatRuns.runStreamingChat, …)` **Why this exists** @@ -204,10 +204,10 @@ schema layer. **Core code** ```ts -// backend/convex/domains/redesign/chatRuns.ts:1492 +// backend/convex/domains/redesign/chatRuns.ts:1492 — const prompt = args.prompt.slice(0, MAX_PROMPT_CHARS) const prompt = args.prompt.slice(0, MAX_PROMPT_CHARS); if (prompt.trim().length < 3) throw new Error("Prompt too short — write at least a 3-character question."); -const userId = await requirePaidChatUserId(ctx); // line 159: rejects anonymous accounts +const userId = await requirePaidChatUserId(ctx); // rejects anonymous accounts const clientRequestId = args.clientRequestId?.trim().slice(0, 160); if (clientRequestId) { /* by_user_client_request index → return existing.runId */ } ``` @@ -243,7 +243,7 @@ parse, bind evidence, seal. **Core code** ```ts -// backend/convex/domains/redesign/chatRuns.ts:1868 +// backend/convex/domains/redesign/chatRuns.ts:1868 — export const runStreamingChat = internalAction export const runStreamingChat = internalAction({ handler: async (ctx, args) => { const append = (eventType, payload) => @@ -284,7 +284,7 @@ renders multi-step agent panels). None of it is on the `/redesign/chat` path. Se **Core code** ```ts -// backend/convex/domains/redesign/chatRuns.ts:2077 +// backend/convex/domains/redesign/chatRuns.ts:2077 — streamGenerateContent?alt=sse const url = `https://generativelanguage.googleapis.com/v1beta/models/${args.model}:streamGenerateContent?alt=sse&key=${apiKey}`; const res = await fetch(url, { method: "POST", signal: controller.signal, body: JSON.stringify({ systemInstruction: { parts: [{ text: systemPrompt }] }, @@ -322,7 +322,7 @@ to `status: "complete"` with its content hash. **Core code** ```ts -// backend/convex/domains/redesign/chatRuns.ts:1775 +// backend/convex/domains/redesign/chatRuns.ts:1775 — export const appendEvent = internalMutation export const appendEvent = internalMutation({ handler: async (ctx, args) => { const existing = await ctx.db.query("redesignChatStreamEvents") @@ -337,7 +337,7 @@ export const appendEvent = internalMutation({ `grounding_chunk`, `board_state`, `packet_complete`, `error`), and a payload. **Output** — one row in `redesignChatStreamEvents`, ordered by `idx`. **Failure behavior** — Convex mutations are transactional; a failed append -leaves no partial row. `finalizeRun` (line 1821) is what makes an answer +leaves no partial row. `finalizeRun` is what makes an answer readable, so a crash before it leaves the run visibly unfinished rather than silently truncated. **Next** — the browser is already subscribed to those rows. @@ -348,7 +348,7 @@ silently truncated. **File:** `apps/web/src/features/redesign/hooks/useRedesignChatRun.ts` **Symbol:** `useRedesignChatRun` -**Called by:** `ChatSurface` (line 339) +**Called by:** `ChatSurface` — the surface's single `useRedesignChatRun()` call **Calls next:** `buildPartialChatAnswer` → `ChatAssistantMessage` **Why this exists** @@ -361,7 +361,7 @@ results whenever the underlying rows change. The event rows written in Step 8 **Core code** ```ts -// apps/web/src/features/redesign/hooks/useRedesignChatRun.ts:290 +// apps/web/src/features/redesign/hooks/useRedesignChatRun.ts:290 — const events = useQuery( const events = useQuery(api.domains.redesign.chatRuns.streamEventsForRun, activeRunId ? { runId: activeRunId } : "skip"); const runRow = useQuery(api.domains.redesign.chatRuns.getRun, activeRunId ? { runId: activeRunId } : "skip"); ``` @@ -370,9 +370,9 @@ const runRow = useQuery(api.domains.redesign.chatRuns.getRun, acti subscribe yet"). **Output** — a projected `RealChatRun`: partial answer, evidence rows, trace rows and metrics, rebuilt on every batch of new events by -`buildPartialChatAnswer` (line 186). -**Failure behavior** — both queries call `assertRunReadable` on the server -(line 175), so a run belonging to someone else throws instead of leaking. If the +`buildPartialChatAnswer`. +**Failure behavior** — both queries call `assertRunReadable` on the +server, so a run belonging to someone else throws instead of leaking. If the tab reloads, `getLatestOwnedRun` re-attaches to the newest owned run, and the answer resumes from the durable rows. **Next** — failure and recovery. @@ -398,7 +398,7 @@ as evidence. **Core code** ```ts -// backend/convex/domains/redesign/chatRuns.ts:2366 +// backend/convex/domains/redesign/chatRuns.ts:2366 — } catch (err: any) { } catch (err: any) { if ((err?.message || String(err)) === "RUN_CANCELLED" || await isCancelled()) return; // cancel ≠ error const errorMessage = (err?.message || String(err)).slice(0, 280); @@ -409,12 +409,12 @@ as evidence. **Input** — any thrown value from the orchestration. **Output** — an `error` event row plus `status: "error"` on the run row — unless -the run was cancelled, in which case nothing is overwritten (`failRun`, line -1849, explicitly refuses to touch a `cancelled` row). +the run was cancelled, in which case nothing is overwritten (`failRun` explicitly +refuses to touch a `cancelled` row). **Failure behavior** — there is no retry on this path. A failed run stays failed -and the user re-asks. After a *successful* run, `validateRunSources` (line 2464) -is scheduled: it re-fetches each cited URL through an SSRF check (`isUrlSafe`, -line 2384) and asserts the quoted text is literally a substring of the page, +and the user re-asks. After a *successful* run, `validateRunSources` +is scheduled: it re-fetches each cited URL through an SSRF check (`isUrlSafe`) +and asserts the quoted text is literally a substring of the page, patching verification flags onto the evidence rows. The UI updates through the same subscription, so verification appears after the answer rather than delaying it. @@ -430,14 +430,37 @@ it. | Step 4's tier mapping, idempotency key and answer projection | `apps/web/src/features/redesign/hooks/useRedesignChatRun.test.ts` | `npx vitest run apps/web/src/features/redesign/hooks/useRedesignChatRun.test.ts` | | The answer packet keeps required fields and leaks no forbidden ones, driven through the real runtime functions | `backend/convex/domains/redesign/chatRuns.contract.test.ts` | `npx vitest run backend/convex/domains/redesign/chatRuns.contract.test.ts` | | The response-shape policy (compact vs. full) matches the UI contract | `backend/convex/domains/redesign/chatRuns.responseShape.test.ts` | `npx vitest run backend/convex/domains/redesign/chatRuns.responseShape.test.ts` | -| Only owners can read a run's route and events | `evals/e2e/redesign-runtime-route-ownership.spec.ts` | `npx playwright test evals/e2e/redesign-runtime-route-ownership.spec.ts` | -| The whole surface renders end to end in a browser | `evals/e2e/one-flow-regression.spec.ts` | `npx playwright test evals/e2e/one-flow-regression.spec.ts` | +| Only owners can read a run's route and events — **requires a configured Convex deployment; fails from a clean clone** | `evals/e2e/redesign-runtime-route-ownership.spec.ts` | `npx playwright test evals/e2e/redesign-runtime-route-ownership.spec.ts` | +| The whole surface renders end to end in a browser — **requires a configured Convex deployment; fails from a clean clone** | `evals/e2e/one-flow-regression.spec.ts` | `npx playwright test evals/e2e/one-flow-regression.spec.ts` | + +The first four rows are vitest and run from a clean clone. **The last two are +Playwright and do not.** They need a browser *and* a `VITE_CONVEX_URL`, for the +reason Step 1 already gave: with no deployment URL `main.tsx` renders +`MissingConvexUrlScreen` instead of the app, so no product element ever mounts +and the specs time out waiting for elements that cannot exist. Measured on +2026-08-13 from a fresh clone, vite serving the frontend with no +`VITE_CONVEX_URL` set and `BASE_URL` pointed at it: `one-flow-regression` +**6 failed / 0 passed** (2 tests across chromium, firefox and webkit), every +failure `getByTestId('one-surface-workspace')` never found; +`redesign-runtime-route-ownership` on chromium **3 failed / 1 passed**, failing +on `right-inspector`, `reports-runtime-inspector` and `exact-web-chat-stream`. +The page under all of them was the "Convex backend not configured" card. +`docs/codebase/TESTING.md` says the same thing under "Browser checks need a +running app". + +There is no local fixture backend. Steps 5–10 are Convex server code and this +repo ships no substitute for Convex, so those steps are readable and testable +but not observable in a browser without a deployment. The full suite is four segments — `npm run test:run`. It is **red at HEAD** for reasons that predate this document; the exact counts and causes are in `docs/codebase/CONCERNS.md`, so you can tell a pre-existing failure from one you just caused. +`node scripts/validate-tours.mjs` validates this page's line citations and the CodeTour steps +in `.tours/`. It needs no browser, no backend and no install — run it after +moving any code either one points at. + --- ## Where you would add the next capability diff --git a/docs/codebase/TESTING.md b/docs/codebase/TESTING.md index b4680212e..ea15280b9 100644 --- a/docs/codebase/TESTING.md +++ b/docs/codebase/TESTING.md @@ -1,14 +1,20 @@ # TESTING — what to run, what it proves, and what is already red -## The three commands you actually need +## The four commands you actually need ```bash npx vitest run # one file, fast — this is your inner loop npm run test:run # the full suite, four segments, ~8 minutes -npx playwright test evals/e2e/.spec.ts # browser, needs a running app -node scripts/validate-tours.mjs # the CodeTour files still point at real lines +npx playwright test evals/e2e/.spec.ts # browser + a configured Convex deployment +node scripts/validate-tours.mjs # START_HERE.md and .tours/ cite the right lines ``` +The last one is the only one that needs neither an install nor a backend, which +is why CI runs it before `npm install`. It asserts each citation matches the +text it names, not merely that the line number is in range — a range check +passes a citation that has drifted onto a different symbol, which is the +failure it exists to prevent. + ## The suite is segmented, and that is deliberate `npm run test:run` is `node scripts/testing/runSegmentedVitest.mjs`, which runs diff --git a/scripts/validate-tours.mjs b/scripts/validate-tours.mjs index 806023f38..2a2748b77 100644 --- a/scripts/validate-tours.mjs +++ b/scripts/validate-tours.mjs @@ -1,25 +1,39 @@ /** - * A tour with a broken line reference is worse than no tour, because a reader - * follows it into the wrong function and believes what they see. This asserts - * every step in .tours/ still resolves: the file exists, the line is inside it, - * and the step's `pattern` still matches that exact line. + * A citation with a broken line reference is worse than no citation, because a + * reader follows it into the wrong function and believes what they see. * - * Run: node scripts/validate-tours.mjs (exit 0 = every step resolves) + * Checking that the cited line NUMBER is inside the file proves the anchor is + * stable. It does not prove the anchor is correct: a citation that has drifted + * onto a different symbol is still in range, and a range-only check passes it. + * So every citation here must also carry the text expected on that line, and + * this script asserts the line MATCHES it. + * + * .tours/*.tour every step must carry `pattern`, and the cited line + * must match that regex. A step with no `pattern` is + * rejected rather than range-checked. + * docs/START_HERE.md every `path:line` citation must be written + * `path:line — expected text`, and that text must appear + * literally on the cited line. Bare "line N" prose + * references are rejected: nothing can check them, so + * they rot silently. + * + * Run from the repo root: node scripts/validate-tours.mjs (exit 0 = clean) */ import { readdirSync, readFileSync } from "node:fs"; import { join } from "node:path"; -const dir = ".tours"; let checked = 0; const problems = []; +const readLines = (file) => readFileSync(file, "utf8").split(/\r?\n/); -for (const name of readdirSync(dir).filter((f) => f.endsWith(".tour"))) { - const tour = JSON.parse(readFileSync(join(dir, name), "utf8")); +// ---------------------------------------------------------------- .tours/ +for (const name of readdirSync(".tours").filter((f) => f.endsWith(".tour"))) { + const tour = JSON.parse(readFileSync(join(".tours", name), "utf8")); tour.steps.forEach((step, i) => { const where = `${name} step ${i + 1} (${step.file}:${step.line})`; let source; try { - source = readFileSync(step.file, "utf8").split(/\r?\n/); + source = readLines(step.file); } catch { problems.push(`${where} — file does not exist`); return; @@ -28,8 +42,12 @@ for (const name of readdirSync(dir).filter((f) => f.endsWith(".tour"))) { problems.push(`${where} — line out of range (file has ${source.length} lines)`); return; } - if (step.pattern && !new RegExp(step.pattern).test(source[step.line - 1])) { - problems.push(`${where} — pattern /${step.pattern}/ no longer matches: ${source[step.line - 1].trim()}`); + if (!step.pattern) { + problems.push(`${where} — no "pattern"; a line number alone proves the anchor is stable, not that it points at the right symbol`); + return; + } + if (!new RegExp(step.pattern).test(source[step.line - 1])) { + problems.push(`${where} — pattern /${step.pattern}/ does not match that line: ${source[step.line - 1].trim()}`); return; } if (!step.title || !step.description) { @@ -40,9 +58,53 @@ for (const name of readdirSync(dir).filter((f) => f.endsWith(".tour"))) { }); } +// ------------------------------------------------------- docs/START_HERE.md +const doc = "docs/START_HERE.md"; +// A repo file path followed by :line. Requires a source extension immediately +// before the colon, so http://localhost:5173 and `temperature: 0.3` are not +// mistaken for citations. +const CITATION = /([\w./-]+\.(?:tsx?|mjs|cjs|js|json|md)):(\d+)/; + +readLines(doc).forEach((text, i) => { + const where = `${doc}:${i + 1}`; + + if (/\bline \d+\b/i.test(text)) { + problems.push(`${where} — bare "line N" reference has nothing to check it against; cite it as \`path:line — text on that line\`, or drop the number and keep the symbol name`); + return; + } + + const cite = text.match(CITATION); + if (!cite) return; + + const [matched, file, lineText] = cite; + const line = Number(lineText); + const anchor = text.slice(cite.index + matched.length).replace(/^\s*[—-]+\s*/, "").trim(); + + if (!anchor) { + problems.push(`${where} — citation ${file}:${line} carries no anchor; write \`${file}:${line} — \` so a drifted line number cannot pass`); + return; + } + let source; + try { + source = readLines(file); + } catch { + problems.push(`${where} — ${file} does not exist`); + return; + } + if (line < 1 || line > source.length) { + problems.push(`${where} — ${file}:${line} out of range (file has ${source.length} lines)`); + return; + } + if (!source[line - 1].includes(anchor)) { + problems.push(`${where} — ${file}:${line} does not contain "${anchor}"; that line is: ${source[line - 1].trim()}`); + return; + } + checked += 1; +}); + if (problems.length) { console.error(problems.join("\n")); - console.error(`\n${problems.length} broken tour step(s).`); + console.error(`\n${problems.length} broken citation(s).`); process.exit(1); } -console.log(`All ${checked} tour steps resolve.`); +console.log(`All ${checked} citations resolve and match their anchors.`);