diff --git a/.claude/commands/run-ai-assistant-eval.md b/.claude/commands/run-ai-assistant-eval.md index 8ef76d804c8..4d7095f79cf 100644 --- a/.claude/commands/run-ai-assistant-eval.md +++ b/.claude/commands/run-ai-assistant-eval.md @@ -182,9 +182,14 @@ evidence, whether each holds. Then score: number of criteria), rounded to a whole number. A criterion that is half met (the field exists but is not shown, the edit landed but broke another view) counts as failed. -- **0**: the verdict is not `pass`, or no card of the asked-for kind exists +- **0**: the verdict is `model-failure`, or no card of the asked-for kind exists in the test workspace. A pass with the wrong card (a placeholder, an unrelated card, only the pre-copied card untouched) is also 0. +- **Not scored**: the verdict is `host-failure`, `bot-failure` or + `runner-failure`. The run says nothing about the model, so the result card + leaves it out of the scores (tier `unscored`) whatever score is recorded. + Record `--score 0` with an analysis that names the failure, and judge what + the model did up to that point in the analysis. Write the analysis as markdown: a checklist of the criteria with ✓ or ✗ and one line of evidence each; then, for anything that failed or looked rough, diff --git a/.claude/skills/ai-assistant-evals/SKILL.md b/.claude/skills/ai-assistant-evals/SKILL.md index 3e8030c89da..d8205089e2f 100644 --- a/.claude/skills/ai-assistant-evals/SKILL.md +++ b/.claude/skills/ai-assistant-evals/SKILL.md @@ -87,6 +87,7 @@ Knobs (env vars): | `EVAL_HOST_URL` / `EVAL_MATRIX_URL` | `https://localhost:4200` / `http://localhost:8008` | stack endpoints | | `EVAL_BOT_USER` | `@aibot:localhost` | the ai-bot's matrix id | | `EVAL_WORKERS` / `EVAL_STAGGER_SECONDS` | 5 / 40 | parallel workers, and the gap between their starts | +| `EVAL_SKILL_FEATURES` | none | comma-separated skill features to switch on in each room before the first prompt (`boxel-debug:feature:enable:`). A skill section between `` markers is left out of the prompt otherwise, so an evaluation of `catalog-reuse` needs `EVAL_SKILL_FEATURES=catalog-reuse` | Models run in parallel, `EVAL_WORKERS` at a time (default 5), starting `EVAL_STAGGER_SECONDS` apart (default 40) so the dev server's first page @@ -143,8 +144,8 @@ When one appears: is also printed at the end of the run. Columns: result grade, model id used, effort the room carried, verdict, turns, -tool calls by name with counts, run-realm-code writes, cost, wall time, whether a card -rendered and where (`stack` or `preview`), notes with the reasons, the missed +tool calls by name with counts, realm writes (run-realm-code writes plus other +writing tool calls), cost, wall time, whether a card rendered and where (`stack` or `preview`), notes with the reasons, the missed benchmarks, and the room id. The grade is the one-word answer per model: @@ -199,7 +200,7 @@ After every run, two checks on the money, whatever the grade: Verdicts: - `pass` — a card from the new workspace rendered with no error boundary, and at - least one run-realm-code write was applied. + least one applied run-realm-code or other writing tool call. - `model-failure` — the model did not get there: no write, or no card. Reasons are listed. - `host-failure` — a tool pill stayed in "applying" past the host's own two-minute tool timeout, or a tool call never got a result. The model may be diff --git a/packages/ai-assistant-evals/README.md b/packages/ai-assistant-evals/README.md index 12a5f4d28b5..30f06133497 100644 --- a/packages/ai-assistant-evals/README.md +++ b/packages/ai-assistant-evals/README.md @@ -140,8 +140,9 @@ A run also prints the table at the end. 4. Looks for a rendered card in the stack that is not the workspace index and has no error state; screenshots the page. 5. Reads the room's Matrix events: turns, tokens, cost, tool calls and their - outcomes, run-realm-code writes, files written. -6. Classifies: `pass` (card rendered), `model` failure (never wrote, wrong + outcomes, run-realm-code writes and other writing tool calls, files written. +6. Classifies: `pass` (card rendered, and at least one applied run-realm-code + or other writing tool call), `model` failure (never wrote, wrong tool, gave up), or `host` failure (a pill stuck applying, a tool with no result, the tab left the workspace). Keeps the room id so the room can be inspected afterwards. diff --git a/packages/ai-assistant-evals/assistant-eval.spec.ts b/packages/ai-assistant-evals/assistant-eval.spec.ts index 705e090d058..4af6ff73889 100644 --- a/packages/ai-assistant-evals/assistant-eval.spec.ts +++ b/packages/ai-assistant-evals/assistant-eval.spec.ts @@ -105,6 +105,16 @@ const MODELS = (process.env.EVAL_MODELS ?? 'Claude Sonnet 4.6') .map((m) => m.trim()) .filter(Boolean); +// Skill sections behind a feature flag (``) are left +// out of the prompt unless the room enabled them. Each name here is enabled +// in the room with `boxel-debug:feature:enable:` before the first +// prompt, in addition to the evaluation's own `skillFeatures`, so a run can +// measure the assistant with a feature on that the evaluation leaves off. +const SKILL_FEATURES = (process.env.EVAL_SKILL_FEATURES ?? '') + .split(',') + .map((f) => f.trim()) + .filter(Boolean); + function slugify(s: string) { return s .toLowerCase() @@ -463,12 +473,16 @@ async function readActivityOnce(page: Page): Promise { } // A loop: the same tool call, with the same arguments, three times. The // bot's own checkCorrectness is left out: it repeats with the same - // arguments on every repair round by design. + // arguments on every repair round by design. So is a call rejected as + // invalid: it never ran, its pill shows no arguments, so several of them + // in one turn look identical, and the failed-turn count above already + // covers them. let toolElements = Array.from( document.querySelectorAll('[data-tool-name]'), ); let calls = toolElements .filter((el) => el.dataset.toolName !== 'checkCorrectness') + .filter((el) => !el.querySelector('[data-test-apply-state="invalid"]')) .map( (el) => `${el.dataset.toolName}|${el.dataset.toolArgumentsLength}|${( @@ -529,11 +543,47 @@ async function stopGeneration(page: Page) { // expensive runs are not stopped; they are graded afterwards. // `botMessagesBefore` is how many finished bot turns the room already had // when the prompt went out; a follow-up prompt waits for a new one. +// Some tools open a room of their own and move the assistant panel to it: +// `listing-remix`, for one, starts a "Remixing …" session. The evaluation's +// conversation carries on in its own room, but the panel no longer shows it, +// so everything read from the page would describe the wrong room. When the +// panel is on another room, reopen the evaluation's room from past sessions. +// Returns true when it switched back. It runs on every poll, so reading the +// current room never waits, and a switch that fails (the past-sessions button +// disabled, the host reloading) returns false for the next poll to retry +// rather than failing the run. +async function returnToRoom(page: Page, roomId: string): Promise { + let current = await page + .evaluate( + () => + document + .querySelector('[data-test-room]') + ?.getAttribute('data-test-room') ?? null, + ) + .catch(() => null); + if (!current || current === roomId) { + return false; + } + try { + await page.locator('[data-test-past-sessions-button]').click(); + await page.locator(`[data-test-enter-room="${roomId}"]`).click(); + await page + .locator(`[data-test-room="${roomId}"]`) + .waitFor({ timeout: 60_000 }); + await page.locator('[data-test-room-settled]').waitFor({ timeout: 60_000 }); + return true; + } catch { + return false; + } +} + async function waitForIdle( page: Page, deadline: number, botMessagesBefore = 0, onActivity?: (activity: Activity) => void, + roomId?: string, + roomReturns: { count: number } = { count: 0 }, ): Promise<{ stoppedBy: RunResult['stoppedBy']; irregularities: string[] }> { let idleSince: number | undefined; let applyingSince: number | undefined; @@ -541,6 +591,9 @@ async function waitForIdle( let lastTextLength = -1; let startedAt = Date.now(); for (;;) { + if (roomId && (await returnToRoom(page, roomId))) { + roomReturns.count++; + } let now = Date.now(); let activity = await readActivity(page); onActivity?.(activity); @@ -745,8 +798,11 @@ function classify( ); return { verdict: 'host-failure', reasons }; } - if (analysis.realmCodeWrites === 0) { - reasons.push('no file was written (no applied run-realm-code write)'); + let writes = analysis.realmCodeWrites + analysis.toolWrites; + if (writes === 0) { + reasons.push( + 'no file was written (no applied run-realm-code or other writing tool call)', + ); } if (!cardId) { reasons.push(...cardReasons); @@ -761,7 +817,7 @@ function classify( if ( cardId && cardReasons.length === 0 && - analysis.realmCodeWrites > 0 && + writes > 0 && promptsSent === promptsTotal ) { return { verdict: 'pass', reasons }; @@ -813,6 +869,8 @@ async function runModel( initialCards: [], initialFiles: [], skillsUsed: [], + skillFeatures: [], + roomReturns: 0, username, stoppedBy: 'error', irregularities: [], @@ -871,14 +929,46 @@ async function runModel( await ensureActMode(page, requestedModel); setStep('confirm the tab is inside the new workspace'); await ensureInsideWorkspace(page, result.realmUrl, result.initialCards); + // The evaluation's own features, plus any EVAL_SKILL_FEATURES adds. + let skillFeatures = [ + ...new Set([...(evaluation?.skillFeatures ?? []), ...SKILL_FEATURES]), + ]; + for (let [index, feature] of skillFeatures.entries()) { + setStep(`enable skill feature ${feature}`); + await sendPrompt( + page, + result.roomId, + `boxel-debug:feature:enable:${feature}`, + index + 1, + ); + // The bot answers the command itself, without the model; its reply + // carries no usage, so it is not counted as a turn. A name the bot + // does not know gets "There is no feature named …" instead, which ends + // the run straight away rather than at the timeout. + let enabled = page.getByText(`${feature} is now enabled`).first(); + let unknown = page.getByText('There is no feature named').first(); + await expect(enabled.or(unknown)).toBeVisible({ timeout: NO_REPLY_MS }); + if (await unknown.isVisible()) { + throw new Error( + `The ai-bot has no skill feature named "${feature}"; check the evaluation's skillFeatures and EVAL_SKILL_FEATURES.`, + ); + } + } + result.skillFeatures = skillFeatures; // One safety clock for the whole run, follow-ups included. let deadline = Date.now() + MAX_MINUTES * 60_000; let botMessagesBefore = 0; + let roomReturns = { count: 0 }; for (let [index, prompt] of prompts.entries()) { setStep( index === 0 ? 'send prompt' : `send follow-up prompt ${index + 1}`, ); - await sendPrompt(page, result.roomId, prompt, index + 1); + await sendPrompt( + page, + result.roomId, + prompt, + skillFeatures.length + index + 1, + ); result.promptsSent = index + 1; setStep( index === 0 @@ -904,6 +994,8 @@ async function runModel( }); } }, + result.roomId, + roomReturns, ); result.stoppedBy = waited.stoppedBy; result.irregularities = waited.irregularities; @@ -912,6 +1004,10 @@ async function runModel( } botMessagesBefore = (await readActivity(page)).botMessages; } + if (result.roomId && (await returnToRoom(page, result.roomId))) { + roomReturns.count++; + } + result.roomReturns = roomReturns.count; setStep('check render'); let rendered = await findRenderedCard( @@ -941,6 +1037,19 @@ async function runModel( ); result.verdict = verdict.verdict; result.reasons = verdict.reasons; + // An enabled feature whose skill the model never opened means the run + // did not exercise it, whatever the verdict says. This assumes each + // feature's skill lives in a skills//SKILL.md directory. + for (let feature of result.skillFeatures) { + let read = result.analysis.filesRead.some((url) => + url.endsWith(`/skills/${feature}/SKILL.md`), + ); + if (!read) { + result.reasons.push( + `skill feature "${feature}" was enabled but skills/${feature}/SKILL.md was never read`, + ); + } + } } catch (error) { result.error = error instanceof Error ? error.message : String(error); result.reasons.push( @@ -1110,7 +1219,7 @@ test.afterAll(async () => { ) .join(', ') : ''; - let writes = a ? `${a.realmCodeWrites}` : ''; + let writes = a ? `${a.realmCodeWrites + a.toolWrites}` : ''; return `| ${graded.grade} | ${r.modelId ?? r.requestedModel} | ${r.reasoningEffort ?? '–'} | ${r.verdict} | ${ a?.turns ?? '–' } | ${tools} | ${writes} | ${a ? `$${a.costUsd.toFixed(3)}` : '–'} | ${ diff --git a/packages/ai-assistant-evals/eval-card.ts b/packages/ai-assistant-evals/eval-card.ts index 1174f6bc924..35971789c8a 100644 --- a/packages/ai-assistant-evals/eval-card.ts +++ b/packages/ai-assistant-evals/eval-card.ts @@ -20,6 +20,8 @@ export interface Evaluation { // Absolute URLs, as the realm serializes them. initialCards: string[]; initialFiles: string[]; + // Skill features to enable in the room before the first prompt. + skillFeatures: string[]; adoptsFrom: { module: string; name: string }; } @@ -87,6 +89,11 @@ export async function loadEvaluation( successCriteria: attributes.successCriteria ?? '', initialCards: linkedIds(doc, 'initialCards', id), initialFiles: linkedIds(doc, 'initialFiles', id), + skillFeatures: Array.isArray(attributes.skillFeatures) + ? attributes.skillFeatures.filter( + (f: unknown): f is string => typeof f === 'string' && f.trim() !== '', + ) + : [], adoptsFrom: doc.data.meta.adoptsFrom, }; } diff --git a/packages/ai-assistant-evals/eval-realm/Evaluation/reuse-listing-ownership.json b/packages/ai-assistant-evals/eval-realm/Evaluation/reuse-listing-ownership.json new file mode 100644 index 00000000000..fd7f91abc94 --- /dev/null +++ b/packages/ai-assistant-evals/eval-realm/Evaluation/reuse-listing-ownership.json @@ -0,0 +1,32 @@ +{ + "data": { + "type": "card", + "attributes": { + "assistantPrompt": "I want to keep track of our wine collection \u2014 each bottle with its producer, vintage and what we paid for it, and a cellar page where we can see them all. Create one example instance and show it on screen.", + "followUpPrompts": [ + "We'd like our own version of this now \u2014 something we can restyle and add fields to, that won't shift under us when someone else changes theirs." + ], + "successCriteria": "1. The assistant searched the catalog before writing any `.gts`.\n2. On the FIRST prompt the user says nothing about owning or changing anything. The assistant must NOT install or remix a Listing there \u2014 a Listing hands over a copy, and nothing was asked to be owned.\n3. On the first prompt it reaches the catalog's wine definitions by module URL \u2014 `WineCellar` and `WineBottle` under `@cardstack/catalog/4b6602-wine-cellar-card-definition/` \u2014 linking, extending or instantiating them.\n4. It does not author its own cellar or bottle card while those catalog definitions exist.\n5. After the first prompt at least one instance exists, is shown on screen, and renders with no error boundary.\n6. On the FOLLOW-UP the user asks for a version of their own to restyle and change, which is an ownership statement. The assistant now installs or remixes the Wine Cellar Listing rather than continuing to reference it.\n7. It takes that copy through the install or remix tool (`listing-install` / `listing-remix`), never by copying the catalog source by hand.\n8. It does not treat the follow-up as a reason to rebuild anything from scratch \u2014 the copy comes from the catalog.\n9. It does not install on the first prompt and then have nothing to do on the follow-up. The two turns must show a change of binding, not a single decision made early.\n10. The follow-up reply says in one line what the user now holds: their own copy, which will not receive later catalog changes.", + "cardInfo": { + "name": "Reuse: reference first, own on request", + "summary": "The first prompt asks for a wine collection with bottles and a cellar page, which the catalog's Wine Cellar Listing already answers; the second asks for a version of their own to restyle. This eval tests whether the selected models reference the catalog definitions first and install or remix the Listing only once ownership is asked for.", + "notes": "This card exists because reference-and-install look equally successful from the outside: both leave a working cellar on screen. The difference is what the user is left holding \u2014 definitions that stay wired to the catalog, or a copy that will not receive upstream fixes. Criterion 2 is the one under test; criterion 6 guards the over-correction, where the assistant refuses to copy even when asked.\n\nTiming targets do not apply: this is a two-turn build and will report ROUGH on turns and seconds as a matter of course.\n\nThe Wine Cellar Listing (`4b6602-wine-cellar-card-definition`) was chosen because it installs and remixes cleanly: every module it needs has a Spec, so the follow-up is completable. Its shape matches the case under test \u2014 a collection card (`WineCellar.bottles`, linksToMany) over item cards (`WineBottle`) \u2014 and the domain appears in no skill example, keeping eval vocabulary disjoint from the skill's own. Before changing the domain, check that the new Listing remixes into a fresh realm with no unresolved imports.", + "cardThumbnailURL": null + }, + "skillFeatures": ["catalog-reuse"] + }, + "relationships": { + "cardInfo.theme": { + "links": { + "self": null + } + } + }, + "meta": { + "adoptsFrom": { + "module": "../evaluation", + "name": "EvaluationCard" + } + } + } +} diff --git a/packages/ai-assistant-evals/eval-realm/Evaluation/reuse-service-desk-build.json b/packages/ai-assistant-evals/eval-realm/Evaluation/reuse-service-desk-build.json new file mode 100644 index 00000000000..9d246ae0173 --- /dev/null +++ b/packages/ai-assistant-evals/eval-realm/Evaluation/reuse-service-desk-build.json @@ -0,0 +1,32 @@ +{ + "data": { + "type": "card", + "attributes": { + "assistantPrompt": "We're standing up customer support for our product. I want to log incoming tickets, know who reported each one and which queue owns it, see whether we're breaching our response targets, and let an agent escalate a ticket that's going badly. Lean on what the catalog already has wherever it fits. Create one example instance and show it on screen.", + "followUpPrompts": [ + "Our priority rules aren't the standard ones \u2014 we use P0 through P4, and a P0 has to auto-escalate if nobody picks it up within 15 minutes." + ], + "successCriteria": "1. The assistant searched the catalog before writing any `.gts`, and searched per part of the brief rather than once for the deliverable as a whole.\n2. If it models the ticket itself, it reuses the catalog's `Ticket` definition rather than authoring an equivalent ticket card.\n3. If it models who reported the ticket, it links or extends the catalog's `SupportContact` rather than declaring name/email string fields.\n4. If it models the owning queue, it links the catalog's `Queue` rather than a string or a bespoke queue card.\n5. If it models the handling agent, it links the catalog's `SupportAgent` rather than a string.\n6. Ticket status uses the catalog's `TicketStatusField` \u2014 never a bare string or a hand-rolled enum.\n7. Response targets reuse the catalog's SLA definitions (`Sla`, `SlaPolicy`, `SlaTimerField`) rather than hand-rolled deadline arithmetic.\n8. The escalate action reuses the catalog's escalation command rather than a hand-written function \u2014 or, if none fits, the assistant says so explicitly and records the gap.\n9. Reuse ratio: at least 6 of the 7 opportunities in criteria 2-8 are answered from the catalog.\n10. Nothing on this list was hand-built: ticket status as a bare string or enum; a bespoke reporter card made of name/email strings; a hand-written escalate function; a hand-rolled SLA timer.\n11. On the follow-up, the assistant specializes the catalog definition rather than copying it wholesale \u2014 unless it explains why the divergence needs a copy.\n12. On the follow-up, the reply names in one line what the user now has: a reference that stays in sync with the catalog, or a copy they own that will not receive upstream fixes.\n13. At least one instance exists and renders with no error boundary. A new definition is not required if the catalog's own cards answer the brief.", + "cardInfo": { + "name": "Reuse: service desk build", + "summary": "One realistic brief with seven catalog reuse opportunities across card, field and command \u2014 graded on the ratio taken, not a single decision.", + "notes": "Timing targets do not apply to this card. It is a multi-part build with a follow-up, so the <=8 turn and <=120s targets will report ROUGH as a matter of course; that is not a finding. Judge it on the reuse ratio and the must-not-hand-roll list.\n\nThe service-desk domain carries no Listing, so this card exercises the Spec/parts path only. Install and remix of a whole bundle are deliberately unmeasured here.\n\nDomain chosen because no other evaluation touches it, keeping eval vocabulary disjoint from the skill's own examples.", + "cardThumbnailURL": null + }, + "skillFeatures": ["catalog-reuse"] + }, + "relationships": { + "cardInfo.theme": { + "links": { + "self": null + } + } + }, + "meta": { + "adoptsFrom": { + "module": "../evaluation", + "name": "EvaluationCard" + } + } + } +} diff --git a/packages/ai-assistant-evals/eval-realm/evaluation.gts b/packages/ai-assistant-evals/eval-realm/evaluation.gts index 3effac792bb..b87bb785f25 100644 --- a/packages/ai-assistant-evals/eval-realm/evaluation.gts +++ b/packages/ai-assistant-evals/eval-realm/evaluation.gts @@ -61,9 +61,21 @@ function seconds(value: number | undefined | null) { return value == null ? '–' : `${Math.round(value)} s`; } -const TIERS = ['pending', 'failed', 'rough', 'good', 'great'] as const; +const TIERS = [ + 'pending', + 'unscored', + 'failed', + 'rough', + 'good', + 'great', +] as const; export type EffectivenessTier = (typeof TIERS)[number]; +// Verdicts where the host, ai-bot or runner broke first: the run says nothing +// about the model. Such a result is left unscored rather than counted as a +// zero, and the report's mean leaves it out. +const UNSCORED_VERDICTS = ['host-failure', 'bot-failure', 'runner-failure']; + // The one place the score-to-tier thresholds live: the result card computes // its own tier with it, and the report tints its headline number with it. function tierForScore(score: number | undefined | null): EffectivenessTier { @@ -101,6 +113,11 @@ export class EvaluationCard extends CardDef { // cards in the stack so they are part of the message context. @field initialCards = linksToMany(CardDef); @field initialFiles = linksToMany(FileDef); + // Skill features the evaluation needs switched on (`boxel-debug:feature: + // enable:` in the room before the first prompt). A skill section + // between `` markers is left out of the prompt + // otherwise, so an evaluation of that section must name it here. + @field skillFeatures = containsMany(StringField); @field sessionReports = linksToMany(() => EvaluationReportCard, { query: { @@ -292,10 +309,14 @@ export class EvaluationResultCard extends CardDef { // One number, 0 to 100. Half of it is the judged quality; the rest rewards // getting there in few turns, cheaply, with the prompt cache working, and - // quickly. A run that did not pass, or that a judge scored zero, is zero. - // Empty until the quality score is in. + // quickly. A run the model failed, or that a judge scored zero, is zero; a + // run with an unscored verdict has no score. Otherwise empty until the + // quality score is in. @field effectivenessScore = contains(NumberField, { computeVia: function (this: EvaluationResultCard) { + if (this.verdict && UNSCORED_VERDICTS.includes(this.verdict)) { + return undefined; + } if (this.verdict && this.verdict !== 'pass') { return 0; } @@ -320,6 +341,9 @@ export class EvaluationResultCard extends CardDef { enumField(StringField, { options: [...TIERS] }), { computeVia: function (this: EvaluationResultCard): EffectivenessTier { + if (this.verdict && UNSCORED_VERDICTS.includes(this.verdict)) { + return 'unscored'; + } return tierForScore(this.effectivenessScore); }, }, @@ -489,6 +513,12 @@ export class EvaluationResultCard extends CardDef { .tier-failed { background: #ffd6d6; } + .tier-unscored { + background: transparent; + border: 1px dashed var(--boxel-300); + color: var(--boxel-450); + font-style: italic; + } .numbers { margin: 0; display: grid; @@ -641,6 +671,12 @@ export class EvaluationResultCard extends CardDef { background: #fde4e2; color: #8d231f; } + .tier-unscored { + background: transparent; + border: 1px dashed var(--line); + color: var(--ink-soft); + font-style: italic; + } .who { flex: 1 1 11rem; diff --git a/packages/ai-assistant-evals/room-analysis.ts b/packages/ai-assistant-evals/room-analysis.ts index 865e2a77de1..d40cd90a475 100644 --- a/packages/ai-assistant-evals/room-analysis.ts +++ b/packages/ai-assistant-evals/room-analysis.ts @@ -25,6 +25,13 @@ export interface RoomAnalysis { unansweredToolCalls: number; // realm.fs writes in run-realm-code calls the host applied. realmCodeWrites: number; + // Applied calls to the other tools that write into a realm (copy, patch, + // save, listing install/remix/use, ...). A card those write is as much the + // model's output as one written with run-realm-code. + toolWrites: number; + // URLs the model read with readRealmFile. A skill file under test that is + // missing here was never in the model's context. + filesRead: string[]; filesWritten: string[]; showCardIds: string[]; lastBotBody: string; @@ -32,7 +39,14 @@ export interface RoomAnalysis { const BOT_MESSAGE_MSGTYPE = 'app.boxel.message'; const TOOL_REQUESTS_KEY = 'app.boxel.toolRequests'; -// Every write call counts; the path is recorded only when it is a literal. +// Tools, other than run-realm-code, whose applied call writes into a realm. +// Tool names carry a hash suffix (`copy-card_eefc`), so match the prefix. The +// list is kept by hand against packages/host/app/tools and the catalog's +// commands (packages/catalog/contents/commands). +const WRITE_TOOL = + /^(copy-card|copy-card-to-stack|copy-and-edit|copy-source|copy-file-to-realm|download-file-to-realm|patch-card-instance|patch-fields|patch-code|patch-theme|save-card|store-add|write-text-file|write-binary-file|apply-markdown-edit|add-field-to-card-definition|execute-atomic-operations|transform-cards|create-specs|generate-readme-spec|generate-example-cards|generate-thumbnail|migrate-skill|listing-create|listing-generate-example|listing-install|listing-remix|listing-use)(_|$)/; +// realm.fs writes inside run-realm-code. Every write call counts; the path is +// recorded only when it is a literal. const REALM_CODE_WRITE = /realm\.fs\.(?:writeText|replace)\(\s*(?:(['"`])([^'"`]+)\1)?/g; @@ -80,6 +94,8 @@ export function analyzeRoom( failedToolCalls: [], unansweredToolCalls: 0, realmCodeWrites: 0, + toolWrites: 0, + filesRead: [], filesWritten: [], showCardIds: [], lastBotBody: '', @@ -162,6 +178,18 @@ export function analyzeRoom( } else if (outcome.key !== 'applied') { result.failedToolCalls.push({ name, reason: outcome.reason }); } + if (name === 'readRealmFile') { + let args = parseArguments(request.arguments); + let urls = args.attributes?.urls ?? args.urls; + if (Array.isArray(urls)) { + result.filesRead.push( + ...urls.filter((u: unknown): u is string => typeof u === 'string'), + ); + } + } + if (WRITE_TOOL.test(name) && outcome?.key === 'applied') { + result.toolWrites++; + } if (name.startsWith('run-realm-code') && outcome?.key === 'applied') { // The host nests top-level fields under `attributes` before it runs a // call, so a model's flat `code` is a real write too. @@ -210,5 +238,6 @@ export function analyzeRoom( } result.filesWritten = [...new Set(result.filesWritten)]; + result.filesRead = [...new Set(result.filesRead)]; return result; } diff --git a/packages/ai-assistant-evals/run-eval.ts b/packages/ai-assistant-evals/run-eval.ts index 4930aeb6ac8..b9672077da4 100644 --- a/packages/ai-assistant-evals/run-eval.ts +++ b/packages/ai-assistant-evals/run-eval.ts @@ -262,8 +262,8 @@ async function createResultCard( `grade ${graded.grade}`, ...result.reasons, ...graded.misses, - ...(a && a.realmCodeWrites - ? [`${a.realmCodeWrites} realm write(s)`] + ...(a && a.realmCodeWrites + a.toolWrites + ? [`${a.realmCodeWrites + a.toolWrites} realm write(s)`] : []), ...(result.cardRendered ? [`rendered ${result.cardRendered} in the ${result.renderedIn}`] diff --git a/packages/ai-assistant-evals/run-result.ts b/packages/ai-assistant-evals/run-result.ts index ba0d4c863a3..a71e26c13f3 100644 --- a/packages/ai-assistant-evals/run-result.ts +++ b/packages/ai-assistant-evals/run-result.ts @@ -36,6 +36,9 @@ export interface RunResult { initialFiles: string[]; // Source URLs of the skill files the room had enabled. skillsUsed: string[]; + // Skill features enabled in the room before the first prompt + // (EVAL_SKILL_FEATURES). + skillFeatures: string[]; username: string; stoppedBy: | 'idle' @@ -46,6 +49,9 @@ export interface RunResult { | 'irregularity' | 'error'; irregularities: string[]; + // How often a tool (listing-remix, for one) moved the assistant panel to a + // room of its own and the runner reopened the evaluation's room. + roomReturns: number; // File name, next to this result, of what the workspace held when the run // ended: every source file and every indexed card document. workspaceSnapshot?: string;