From cd0497d789cecad419dc521d13c473732859b71b Mon Sep 17 00:00:00 2001 From: Fadhlan Ridhwanallah Date: Mon, 28 Sep 2026 18:10:13 +0700 Subject: [PATCH 1/7] Add catalog-reuse evaluation cards MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Five EvaluationCards covering the reuse decision surface: use-as-is, extend, remix on stated ownership, unstated ownership intent, and a multi-part brief where every part has a catalog answer. Each prompt is anchored on a definition that exists in the catalog realm and asks for an instance to be created and shown — the runner's verdict requires a rendered card independent of successCriteria, so a definition-only reply scores a failure whatever it reused. Co-Authored-By: Claude Opus 5 (1M context) --- .../Evaluation/reuse-ambiguous-intent.json | 29 +++++++++++++++++++ .../Evaluation/reuse-composite-brief.json | 29 +++++++++++++++++++ .../eval-realm/Evaluation/reuse-extend.json | 29 +++++++++++++++++++ .../Evaluation/reuse-reference-as-is.json | 29 +++++++++++++++++++ .../Evaluation/reuse-remix-on-ownership.json | 29 +++++++++++++++++++ 5 files changed, 145 insertions(+) create mode 100644 packages/ai-assistant-evals/eval-realm/Evaluation/reuse-ambiguous-intent.json create mode 100644 packages/ai-assistant-evals/eval-realm/Evaluation/reuse-composite-brief.json create mode 100644 packages/ai-assistant-evals/eval-realm/Evaluation/reuse-extend.json create mode 100644 packages/ai-assistant-evals/eval-realm/Evaluation/reuse-reference-as-is.json create mode 100644 packages/ai-assistant-evals/eval-realm/Evaluation/reuse-remix-on-ownership.json diff --git a/packages/ai-assistant-evals/eval-realm/Evaluation/reuse-ambiguous-intent.json b/packages/ai-assistant-evals/eval-realm/Evaluation/reuse-ambiguous-intent.json new file mode 100644 index 00000000000..2e1ce0ac3a9 --- /dev/null +++ b/packages/ai-assistant-evals/eval-realm/Evaluation/reuse-ambiguous-intent.json @@ -0,0 +1,29 @@ +{ + "data": { + "type": "card", + "attributes": { + "assistantPrompt": "Set up a Job Application card. It needs to show the candidate's contact details along with the role they applied for and the application status. Create one example instance and show it on screen.", + "followUpPrompts": [], + "successCriteria": "1. The assistant searched the catalog and found the existing Contact card definition before writing any `.gts` file.\n2. Ownership intent is unstated in the brief, so the assistant took the reversible path: it referenced (linked to or extended) the catalog Contact rather than silently forking an owned copy.\n3. The assistant surfaced the tradeoff in roughly one line \u2014 that the candidate details stay in sync with the catalog card, and the user can say so if they would rather own an editable copy.\n4. The assistant did NOT stop and block on a clarifying question for this case; the reversible default resolves it.\n5. The assistant did NOT ask more than one question total, and asked none that the brief already answers.\n6. A definition and at least one instance were written and the instance renders with no error boundary.", + "cardInfo": { + "name": "Reuse: ambiguous intent", + "summary": "Unstated ownership intent: take the reversible reference path and surface the tradeoff in one line, rather than forking silently or blocking on a question.", + "notes": null, + "cardThumbnailURL": null + } + }, + "relationships": { + "cardInfo.theme": { + "links": { + "self": null + } + } + }, + "meta": { + "adoptsFrom": { + "module": "../evaluation", + "name": "EvaluationCard" + } + } + } +} diff --git a/packages/ai-assistant-evals/eval-realm/Evaluation/reuse-composite-brief.json b/packages/ai-assistant-evals/eval-realm/Evaluation/reuse-composite-brief.json new file mode 100644 index 00000000000..6514358ecf9 --- /dev/null +++ b/packages/ai-assistant-evals/eval-realm/Evaluation/reuse-composite-brief.json @@ -0,0 +1,29 @@ +{ + "data": { + "type": "card", + "attributes": { + "assistantPrompt": "Build a Volunteer card for our outreach program. Each volunteer has a mailing address, an email address we can reach them at, and we want to be able to send them a message from the card. Create one example instance and show it on screen.", + "followUpPrompts": [], + "successCriteria": "1. The assistant enumerated all three parts of the brief (mailing address, email address, send-a-message action) before building, rather than starting on the first one.\n2. It searched the catalog for each part, not just once for the card as a whole.\n3. The mailing address uses the existing address field definition via `contains`, not a hand-rolled set of street/city/postcode string fields.\n4. The email address uses the existing email field definition via `contains`, not a plain string field.\n5. The send-a-message action reuses an existing catalog command rather than a hand-written stub, or \u2014 if no command matches \u2014 the assistant said so explicitly and recorded the gap.\n6. No part of the brief was silently hand-built while another was reused.\n7. A definition and at least one instance were written and the instance renders all three parts with no error boundary.", + "cardInfo": { + "name": "Reuse: composite brief", + "summary": "Multi-part brief: enumerate every part and reuse each available one \u2014 the documented partial-compliance failure mode.", + "notes": null, + "cardThumbnailURL": null + } + }, + "relationships": { + "cardInfo.theme": { + "links": { + "self": null + } + } + }, + "meta": { + "adoptsFrom": { + "module": "../evaluation", + "name": "EvaluationCard" + } + } + } +} diff --git a/packages/ai-assistant-evals/eval-realm/Evaluation/reuse-extend.json b/packages/ai-assistant-evals/eval-realm/Evaluation/reuse-extend.json new file mode 100644 index 00000000000..9202e7825b0 --- /dev/null +++ b/packages/ai-assistant-evals/eval-realm/Evaluation/reuse-extend.json @@ -0,0 +1,29 @@ +{ + "data": { + "type": "card", + "attributes": { + "assistantPrompt": "We need a VIP Contact for our key accounts. It should be everything the Contact card definition the catalog already ships is, plus a loyalty tier and the name of the dedicated account manager. Create one example instance and show it on screen.", + "followUpPrompts": [], + "successCriteria": "1. The assistant searched the catalog and found the existing Contact card definition before writing any `.gts` file.\n2. The new VIP Contact is a CardDef that `extends` the catalog's Contact definition, importing it by its catalog module URL.\n3. The assistant did NOT remix or install a Listing to get a copy of Contact, and did NOT hand-copy Contact's field declarations into the new file.\n4. The assistant did NOT rebuild the contact fields (name, email, phone, etc.) from scratch \u2014 they are inherited.\n5. The loyalty tier and account manager fields are declared on the subclass, reusing catalog/base field definitions where one fits.\n6. A definition and at least one instance were written and the instance renders inherited and new fields with no error boundary.", + "cardInfo": { + "name": "Reuse: extend", + "summary": "Additive change to a catalog card means subclassing it, keeping the live upstream dependency \u2014 not forking a copy.", + "notes": null, + "cardThumbnailURL": null + } + }, + "relationships": { + "cardInfo.theme": { + "links": { + "self": null + } + } + }, + "meta": { + "adoptsFrom": { + "module": "../evaluation", + "name": "EvaluationCard" + } + } + } +} diff --git a/packages/ai-assistant-evals/eval-realm/Evaluation/reuse-reference-as-is.json b/packages/ai-assistant-evals/eval-realm/Evaluation/reuse-reference-as-is.json new file mode 100644 index 00000000000..66a7a3c5b20 --- /dev/null +++ b/packages/ai-assistant-evals/eval-realm/Evaluation/reuse-reference-as-is.json @@ -0,0 +1,29 @@ +{ + "data": { + "type": "card", + "attributes": { + "assistantPrompt": "Build a Sales Call Log card that records one sales call: which contact was called, the date, how long it lasted, and notes from the call. Create one example instance and show it on screen.", + "followUpPrompts": [], + "successCriteria": "1. The assistant searched the catalog before writing any `.gts` file.\n2. The contact on the call is a relationship to the catalog's existing Contact card definition, reached by its catalog module URL \u2014 not a newly authored contact card and not a plain string name.\n3. The relationship uses `linksTo` (single contact), not `contains` and not `linksToMany`.\n4. The remaining fields (date, duration, notes) are reused base/catalog field definitions where one fits, rather than hand-rolled equivalents.\n5. The assistant did not remix or copy a Listing for this \u2014 the catalog Contact stays an upstream dependency.\n6. A definition and at least one instance were written and the instance renders with no error boundary.", + "cardInfo": { + "name": "Reuse: reference as-is", + "summary": "Use-as-is is the default: link the catalog's Contact rather than rebuilding or copying it.", + "notes": null, + "cardThumbnailURL": null + } + }, + "relationships": { + "cardInfo.theme": { + "links": { + "self": null + } + } + }, + "meta": { + "adoptsFrom": { + "module": "../evaluation", + "name": "EvaluationCard" + } + } + } +} diff --git a/packages/ai-assistant-evals/eval-realm/Evaluation/reuse-remix-on-ownership.json b/packages/ai-assistant-evals/eval-realm/Evaluation/reuse-remix-on-ownership.json new file mode 100644 index 00000000000..87e8ad300f5 --- /dev/null +++ b/packages/ai-assistant-evals/eval-realm/Evaluation/reuse-remix-on-ownership.json @@ -0,0 +1,29 @@ +{ + "data": { + "type": "card", + "attributes": { + "assistantPrompt": "The catalog has a mortgage calculator. We want our own version of it \u2014 we're replacing its amortization maths with our lender's formula and restyling it to our brand, and it should not change under us when the catalog one changes. Create one example instance and show it on screen.", + "followUpPrompts": [], + "successCriteria": "1. The assistant searched the catalog and found the existing mortgage calculator Listing before writing any `.gts` file.\n2. The assistant remixed (copied) the Listing rather than installing it as-is \u2014 the stated intent is divergence past the original's contract plus independence from upstream.\n3. The assistant did NOT hand-copy the Listing's files one by one; it used the remix mechanism.\n4. The assistant did NOT rebuild a mortgage calculator from scratch.\n5. The assistant stated, in one line, that the remix is now an owned fork that no longer receives catalog updates.\n6. The remixed card renders in the test workspace with no error boundary.", + "cardInfo": { + "name": "Reuse: remix on ownership", + "summary": "Guards against over-correcting to \"never copy\": stated ownership + divergence past the parent's contract is what remix is for.", + "notes": null, + "cardThumbnailURL": null + } + }, + "relationships": { + "cardInfo.theme": { + "links": { + "self": null + } + } + }, + "meta": { + "adoptsFrom": { + "module": "../evaluation", + "name": "EvaluationCard" + } + } + } +} From 13adf09ea70be6aef7372ed77bf262283ba9cf38 Mon Sep 17 00:00:00 2001 From: Fadhlan Ridhwanallah Date: Tue, 29 Sep 2026 10:24:15 +0700 Subject: [PATCH 2/7] Replace the five single-axis reuse evals with one complex build MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each of the five graded one reuse decision on a brief engineered so that decision was nearly forced. That measures isolated choices and misses the failure the skill itself names: reusing one part of a brief and hand-building the rest, which only appears at scale. One service-desk brief carries eight reuse opportunities across card, field, command and component, graded on the ratio taken plus a must-not-hand-roll list, with criteria phrased conditionally so a different-but-sound design still passes. A follow-up asks for a divergence, so the run also covers what happens when the catalog answer no longer fits. Service-desk carries no Listing, so install and remix of a whole bundle are unmeasured here — recorded in the card's notes along with why the timing targets do not apply to a build this size. Co-Authored-By: Claude Opus 5 (1M context) --- .../Evaluation/reuse-ambiguous-intent.json | 29 ----------------- .../Evaluation/reuse-composite-brief.json | 29 ----------------- .../eval-realm/Evaluation/reuse-extend.json | 29 ----------------- .../Evaluation/reuse-reference-as-is.json | 29 ----------------- .../Evaluation/reuse-remix-on-ownership.json | 29 ----------------- .../Evaluation/reuse-service-desk-build.json | 31 +++++++++++++++++++ 6 files changed, 31 insertions(+), 145 deletions(-) delete mode 100644 packages/ai-assistant-evals/eval-realm/Evaluation/reuse-ambiguous-intent.json delete mode 100644 packages/ai-assistant-evals/eval-realm/Evaluation/reuse-composite-brief.json delete mode 100644 packages/ai-assistant-evals/eval-realm/Evaluation/reuse-extend.json delete mode 100644 packages/ai-assistant-evals/eval-realm/Evaluation/reuse-reference-as-is.json delete mode 100644 packages/ai-assistant-evals/eval-realm/Evaluation/reuse-remix-on-ownership.json create mode 100644 packages/ai-assistant-evals/eval-realm/Evaluation/reuse-service-desk-build.json diff --git a/packages/ai-assistant-evals/eval-realm/Evaluation/reuse-ambiguous-intent.json b/packages/ai-assistant-evals/eval-realm/Evaluation/reuse-ambiguous-intent.json deleted file mode 100644 index 2e1ce0ac3a9..00000000000 --- a/packages/ai-assistant-evals/eval-realm/Evaluation/reuse-ambiguous-intent.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "data": { - "type": "card", - "attributes": { - "assistantPrompt": "Set up a Job Application card. It needs to show the candidate's contact details along with the role they applied for and the application status. Create one example instance and show it on screen.", - "followUpPrompts": [], - "successCriteria": "1. The assistant searched the catalog and found the existing Contact card definition before writing any `.gts` file.\n2. Ownership intent is unstated in the brief, so the assistant took the reversible path: it referenced (linked to or extended) the catalog Contact rather than silently forking an owned copy.\n3. The assistant surfaced the tradeoff in roughly one line \u2014 that the candidate details stay in sync with the catalog card, and the user can say so if they would rather own an editable copy.\n4. The assistant did NOT stop and block on a clarifying question for this case; the reversible default resolves it.\n5. The assistant did NOT ask more than one question total, and asked none that the brief already answers.\n6. A definition and at least one instance were written and the instance renders with no error boundary.", - "cardInfo": { - "name": "Reuse: ambiguous intent", - "summary": "Unstated ownership intent: take the reversible reference path and surface the tradeoff in one line, rather than forking silently or blocking on a question.", - "notes": null, - "cardThumbnailURL": null - } - }, - "relationships": { - "cardInfo.theme": { - "links": { - "self": null - } - } - }, - "meta": { - "adoptsFrom": { - "module": "../evaluation", - "name": "EvaluationCard" - } - } - } -} diff --git a/packages/ai-assistant-evals/eval-realm/Evaluation/reuse-composite-brief.json b/packages/ai-assistant-evals/eval-realm/Evaluation/reuse-composite-brief.json deleted file mode 100644 index 6514358ecf9..00000000000 --- a/packages/ai-assistant-evals/eval-realm/Evaluation/reuse-composite-brief.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "data": { - "type": "card", - "attributes": { - "assistantPrompt": "Build a Volunteer card for our outreach program. Each volunteer has a mailing address, an email address we can reach them at, and we want to be able to send them a message from the card. Create one example instance and show it on screen.", - "followUpPrompts": [], - "successCriteria": "1. The assistant enumerated all three parts of the brief (mailing address, email address, send-a-message action) before building, rather than starting on the first one.\n2. It searched the catalog for each part, not just once for the card as a whole.\n3. The mailing address uses the existing address field definition via `contains`, not a hand-rolled set of street/city/postcode string fields.\n4. The email address uses the existing email field definition via `contains`, not a plain string field.\n5. The send-a-message action reuses an existing catalog command rather than a hand-written stub, or \u2014 if no command matches \u2014 the assistant said so explicitly and recorded the gap.\n6. No part of the brief was silently hand-built while another was reused.\n7. A definition and at least one instance were written and the instance renders all three parts with no error boundary.", - "cardInfo": { - "name": "Reuse: composite brief", - "summary": "Multi-part brief: enumerate every part and reuse each available one \u2014 the documented partial-compliance failure mode.", - "notes": null, - "cardThumbnailURL": null - } - }, - "relationships": { - "cardInfo.theme": { - "links": { - "self": null - } - } - }, - "meta": { - "adoptsFrom": { - "module": "../evaluation", - "name": "EvaluationCard" - } - } - } -} diff --git a/packages/ai-assistant-evals/eval-realm/Evaluation/reuse-extend.json b/packages/ai-assistant-evals/eval-realm/Evaluation/reuse-extend.json deleted file mode 100644 index 9202e7825b0..00000000000 --- a/packages/ai-assistant-evals/eval-realm/Evaluation/reuse-extend.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "data": { - "type": "card", - "attributes": { - "assistantPrompt": "We need a VIP Contact for our key accounts. It should be everything the Contact card definition the catalog already ships is, plus a loyalty tier and the name of the dedicated account manager. Create one example instance and show it on screen.", - "followUpPrompts": [], - "successCriteria": "1. The assistant searched the catalog and found the existing Contact card definition before writing any `.gts` file.\n2. The new VIP Contact is a CardDef that `extends` the catalog's Contact definition, importing it by its catalog module URL.\n3. The assistant did NOT remix or install a Listing to get a copy of Contact, and did NOT hand-copy Contact's field declarations into the new file.\n4. The assistant did NOT rebuild the contact fields (name, email, phone, etc.) from scratch \u2014 they are inherited.\n5. The loyalty tier and account manager fields are declared on the subclass, reusing catalog/base field definitions where one fits.\n6. A definition and at least one instance were written and the instance renders inherited and new fields with no error boundary.", - "cardInfo": { - "name": "Reuse: extend", - "summary": "Additive change to a catalog card means subclassing it, keeping the live upstream dependency \u2014 not forking a copy.", - "notes": null, - "cardThumbnailURL": null - } - }, - "relationships": { - "cardInfo.theme": { - "links": { - "self": null - } - } - }, - "meta": { - "adoptsFrom": { - "module": "../evaluation", - "name": "EvaluationCard" - } - } - } -} diff --git a/packages/ai-assistant-evals/eval-realm/Evaluation/reuse-reference-as-is.json b/packages/ai-assistant-evals/eval-realm/Evaluation/reuse-reference-as-is.json deleted file mode 100644 index 66a7a3c5b20..00000000000 --- a/packages/ai-assistant-evals/eval-realm/Evaluation/reuse-reference-as-is.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "data": { - "type": "card", - "attributes": { - "assistantPrompt": "Build a Sales Call Log card that records one sales call: which contact was called, the date, how long it lasted, and notes from the call. Create one example instance and show it on screen.", - "followUpPrompts": [], - "successCriteria": "1. The assistant searched the catalog before writing any `.gts` file.\n2. The contact on the call is a relationship to the catalog's existing Contact card definition, reached by its catalog module URL \u2014 not a newly authored contact card and not a plain string name.\n3. The relationship uses `linksTo` (single contact), not `contains` and not `linksToMany`.\n4. The remaining fields (date, duration, notes) are reused base/catalog field definitions where one fits, rather than hand-rolled equivalents.\n5. The assistant did not remix or copy a Listing for this \u2014 the catalog Contact stays an upstream dependency.\n6. A definition and at least one instance were written and the instance renders with no error boundary.", - "cardInfo": { - "name": "Reuse: reference as-is", - "summary": "Use-as-is is the default: link the catalog's Contact rather than rebuilding or copying it.", - "notes": null, - "cardThumbnailURL": null - } - }, - "relationships": { - "cardInfo.theme": { - "links": { - "self": null - } - } - }, - "meta": { - "adoptsFrom": { - "module": "../evaluation", - "name": "EvaluationCard" - } - } - } -} diff --git a/packages/ai-assistant-evals/eval-realm/Evaluation/reuse-remix-on-ownership.json b/packages/ai-assistant-evals/eval-realm/Evaluation/reuse-remix-on-ownership.json deleted file mode 100644 index 87e8ad300f5..00000000000 --- a/packages/ai-assistant-evals/eval-realm/Evaluation/reuse-remix-on-ownership.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "data": { - "type": "card", - "attributes": { - "assistantPrompt": "The catalog has a mortgage calculator. We want our own version of it \u2014 we're replacing its amortization maths with our lender's formula and restyling it to our brand, and it should not change under us when the catalog one changes. Create one example instance and show it on screen.", - "followUpPrompts": [], - "successCriteria": "1. The assistant searched the catalog and found the existing mortgage calculator Listing before writing any `.gts` file.\n2. The assistant remixed (copied) the Listing rather than installing it as-is \u2014 the stated intent is divergence past the original's contract plus independence from upstream.\n3. The assistant did NOT hand-copy the Listing's files one by one; it used the remix mechanism.\n4. The assistant did NOT rebuild a mortgage calculator from scratch.\n5. The assistant stated, in one line, that the remix is now an owned fork that no longer receives catalog updates.\n6. The remixed card renders in the test workspace with no error boundary.", - "cardInfo": { - "name": "Reuse: remix on ownership", - "summary": "Guards against over-correcting to \"never copy\": stated ownership + divergence past the parent's contract is what remix is for.", - "notes": null, - "cardThumbnailURL": null - } - }, - "relationships": { - "cardInfo.theme": { - "links": { - "self": null - } - } - }, - "meta": { - "adoptsFrom": { - "module": "../evaluation", - "name": "EvaluationCard" - } - } - } -} diff --git a/packages/ai-assistant-evals/eval-realm/Evaluation/reuse-service-desk-build.json b/packages/ai-assistant-evals/eval-realm/Evaluation/reuse-service-desk-build.json new file mode 100644 index 00000000000..c0551c69019 --- /dev/null +++ b/packages/ai-assistant-evals/eval-realm/Evaluation/reuse-service-desk-build.json @@ -0,0 +1,31 @@ +{ + "data": { + "type": "card", + "attributes": { + "assistantPrompt": "We're standing up customer support for our product. I want to log incoming tickets, know who reported each one and which queue owns it, see whether we're breaching our response targets, and let an agent escalate a ticket that's going badly. Lean on what the catalog already has wherever it fits. Create one example instance and show it on screen.", + "followUpPrompts": [ + "Our priority rules aren't the standard ones \u2014 we use P0 through P4, and a P0 has to auto-escalate if nobody picks it up within 15 minutes." + ], + "successCriteria": "1. The assistant searched the catalog before writing any `.gts`, and searched per part of the brief rather than once for the deliverable as a whole.\n2. If it models the ticket itself, it reuses the catalog's `Ticket` definition rather than authoring an equivalent ticket card.\n3. If it models who reported the ticket, it links or extends the catalog's `SupportContact` rather than declaring name/email string fields.\n4. If it models the owning queue, it links the catalog's `Queue` rather than a string or a bespoke queue card.\n5. If it models the handling agent, it links the catalog's `SupportAgent` rather than a string.\n6. Ticket status uses the catalog's `TicketStatusField` \u2014 never a bare string or a hand-rolled enum.\n7. Response targets reuse the catalog's SLA definitions (`Sla`, `SlaPolicy`, `SlaTimerField`) rather than hand-rolled deadline arithmetic.\n8. The escalate action reuses the catalog's escalation command rather than a hand-written function \u2014 or, if none fits, the assistant says so explicitly and records the gap.\n9. Reuse ratio: at least 6 of the 7 opportunities in criteria 2-8 are answered from the catalog.\n10. Nothing on this list was hand-built: ticket status as a bare string or enum; a bespoke reporter card made of name/email strings; a hand-written escalate function; a hand-rolled SLA timer.\n11. On the follow-up, the assistant specializes the catalog definition rather than copying it wholesale \u2014 unless it explains why the divergence needs a copy.\n12. On the follow-up, the reply names in one line what the user now has: a reference that stays in sync with the catalog, or a copy they own that will not receive upstream fixes.\n13. At least one instance exists and renders with no error boundary. A new definition is not required if the catalog's own cards answer the brief.", + "cardInfo": { + "name": "Reuse: service desk build", + "summary": "One realistic brief with eight catalog reuse opportunities across card, field, command and component \u2014 graded on the ratio taken, not a single decision.", + "notes": "Timing targets do not apply to this card. It is a multi-part build with a follow-up, so the <=8 turn and <=120s targets will report ROUGH as a matter of course; that is not a finding. Judge it on the reuse ratio and the must-not-hand-roll list.\n\nThe service-desk domain carries no Listing, so this card exercises the Spec/parts path only. Install and remix of a whole bundle are deliberately unmeasured here.\n\nDomain chosen because no other evaluation touches it, keeping eval vocabulary disjoint from the skill's own examples.", + "cardThumbnailURL": null + } + }, + "relationships": { + "cardInfo.theme": { + "links": { + "self": null + } + } + }, + "meta": { + "adoptsFrom": { + "module": "../evaluation", + "name": "EvaluationCard" + } + } + } +} From 224c54f05ed4d01b2d6ed3501ef3aeb374e34569 Mon Sep 17 00:00:00 2001 From: Fadhlan Ridhwanallah Date: Tue, 29 Sep 2026 11:39:20 +0700 Subject: [PATCH 3/7] Add an evaluation card for reference-vs-own on a Listing match MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reference and install look equally successful from the outside: both leave a working blog on screen. The difference is what the user is left holding — definitions still wired to the catalog, or a copy that will not receive upstream fixes. Two turns. The first describes a need the blog Listing matches and says nothing about owning anything, so the definitions should be referenced. The second asks for a version of their own, so the copy is now what was asked for. The binding has to move between them, which a single-turn card cannot observe. Co-Authored-By: Claude Opus 5 (1M context) --- .../Evaluation/reuse-listing-ownership.json | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 packages/ai-assistant-evals/eval-realm/Evaluation/reuse-listing-ownership.json diff --git a/packages/ai-assistant-evals/eval-realm/Evaluation/reuse-listing-ownership.json b/packages/ai-assistant-evals/eval-realm/Evaluation/reuse-listing-ownership.json new file mode 100644 index 00000000000..ca7f31221e1 --- /dev/null +++ b/packages/ai-assistant-evals/eval-realm/Evaluation/reuse-listing-ownership.json @@ -0,0 +1,31 @@ +{ + "data": { + "type": "card", + "attributes": { + "assistantPrompt": "I want to start publishing articles on our site \u2014 posts with an author and a category, and somewhere to see them all. Create one example instance and show it on screen.", + "followUpPrompts": [ + "We'd like our own version of this now \u2014 something we can restyle and add fields to, that won't shift under us when someone else changes theirs." + ], + "successCriteria": "1. The assistant searched the catalog before writing any `.gts`.\n2. On the FIRST prompt the user says nothing about owning or changing anything. The assistant must NOT install or remix a Listing there \u2014 a Listing hands over a copy, and nothing was asked to be owned.\n3. On the first prompt it reaches the catalog's blog definitions by module URL \u2014 `BlogPost`, `Author`, `BlogCategory` under `@cardstack/catalog/7af9aa-blog-app/` \u2014 linking, extending or instantiating them.\n4. It does not author its own post, author or category card while those catalog definitions exist.\n5. At least one instance exists after the first prompt and renders with no error boundary.\n6. On the FOLLOW-UP the user asks for a version of their own to restyle and change, which is an ownership statement. The assistant now installs or remixes the blog Listing rather than continuing to reference it.\n7. It takes that copy through the install or remix mechanism, never by hand-copying files.\n8. It does not treat the follow-up as a reason to rebuild anything from scratch \u2014 the copy comes from the catalog.\n9. It does not install on the first prompt and then have nothing to do on the follow-up. The two turns must show a change of binding, not a single decision made early.", + "cardInfo": { + "name": "Reuse: reference first, own on request", + "summary": "A brief that matches a whole Listing but asks for no ownership, followed by an explicit request to own it \u2014 the binding should change between the two turns.", + "notes": "This card exists because reference-and-install look equally successful from the outside: both leave a working blog on screen. The difference is what the user is left holding \u2014 definitions that stay wired to the catalog, or a copy that will not receive upstream fixes. Criterion 2 is the one under test; criterion 6 guards the over-correction, where the assistant refuses to copy even when asked.\n\nTiming targets do not apply: this is a two-turn build and will report ROUGH on turns and seconds as a matter of course.\n\nThe blog domain was chosen because it carries a real Listing (19 specs) and appears in no skill example, keeping eval vocabulary disjoint from the skill's own.", + "cardThumbnailURL": null + } + }, + "relationships": { + "cardInfo.theme": { + "links": { + "self": null + } + } + }, + "meta": { + "adoptsFrom": { + "module": "../evaluation", + "name": "EvaluationCard" + } + } + } +} From 4cd3be560beb5c6670b20df69a832a2261712deb Mon Sep 17 00:00:00 2001 From: Fadhlan Ridhwanallah Date: Tue, 6 Oct 2026 20:48:20 +0700 Subject: [PATCH 4/7] Enable skill features per evaluation, keep platform failures out of the scores, and move the ownership eval to an installable Listing An EvaluationCard can name the skill features a run needs (skillFeatures); the runner enables each in the room before the first prompt, and EVAL_SKILL_FEATURES adds more for an A/B run. Host, bot and runner failures land in an `unscored` tier the report mean leaves out, and every applied write counts, not only run-realm-code. The runner reopens the evaluation's room when a tool such as listing-remix moves the panel, clicks a pill still spinning past the host's tool timeout once and records it, flags a run that never read an enabled feature's skill, and no longer counts never-run invalid calls as a loop. reuse-listing-ownership now uses the Wine Cellar Listing, which installs and remixes cleanly. Co-Authored-By: Claude Opus 5.5 --- .claude/commands/run-ai-assistant-eval.md | 7 +- .claude/skills/ai-assistant-evals/SKILL.md | 1 + .../ai-assistant-evals/assistant-eval.spec.ts | 154 +++++++++++++++++- packages/ai-assistant-evals/eval-card.ts | 7 + .../Evaluation/reuse-listing-ownership.json | 11 +- .../Evaluation/reuse-service-desk-build.json | 3 +- .../eval-realm/evaluation.gts | 25 ++- packages/ai-assistant-evals/room-analysis.ts | 26 +++ packages/ai-assistant-evals/run-result.ts | 9 + 9 files changed, 229 insertions(+), 14 deletions(-) diff --git a/.claude/commands/run-ai-assistant-eval.md b/.claude/commands/run-ai-assistant-eval.md index 8ef76d804c8..4d7095f79cf 100644 --- a/.claude/commands/run-ai-assistant-eval.md +++ b/.claude/commands/run-ai-assistant-eval.md @@ -182,9 +182,14 @@ evidence, whether each holds. Then score: number of criteria), rounded to a whole number. A criterion that is half met (the field exists but is not shown, the edit landed but broke another view) counts as failed. -- **0**: the verdict is not `pass`, or no card of the asked-for kind exists +- **0**: the verdict is `model-failure`, or no card of the asked-for kind exists in the test workspace. A pass with the wrong card (a placeholder, an unrelated card, only the pre-copied card untouched) is also 0. +- **Not scored**: the verdict is `host-failure`, `bot-failure` or + `runner-failure`. The run says nothing about the model, so the result card + leaves it out of the scores (tier `unscored`) whatever score is recorded. + Record `--score 0` with an analysis that names the failure, and judge what + the model did up to that point in the analysis. Write the analysis as markdown: a checklist of the criteria with ✓ or ✗ and one line of evidence each; then, for anything that failed or looked rough, diff --git a/.claude/skills/ai-assistant-evals/SKILL.md b/.claude/skills/ai-assistant-evals/SKILL.md index 3e8030c89da..9fc708d5c8e 100644 --- a/.claude/skills/ai-assistant-evals/SKILL.md +++ b/.claude/skills/ai-assistant-evals/SKILL.md @@ -87,6 +87,7 @@ Knobs (env vars): | `EVAL_HOST_URL` / `EVAL_MATRIX_URL` | `https://localhost:4200` / `http://localhost:8008` | stack endpoints | | `EVAL_BOT_USER` | `@aibot:localhost` | the ai-bot's matrix id | | `EVAL_WORKERS` / `EVAL_STAGGER_SECONDS` | 5 / 40 | parallel workers, and the gap between their starts | +| `EVAL_SKILL_FEATURES` | none | comma-separated skill features to switch on in each room before the first prompt (`boxel-debug:feature:enable:`). A skill section between `` markers is left out of the prompt otherwise, so an evaluation of `catalog-reuse` needs `EVAL_SKILL_FEATURES=catalog-reuse` | Models run in parallel, `EVAL_WORKERS` at a time (default 5), starting `EVAL_STAGGER_SECONDS` apart (default 40) so the dev server's first page diff --git a/packages/ai-assistant-evals/assistant-eval.spec.ts b/packages/ai-assistant-evals/assistant-eval.spec.ts index 705e090d058..ba06649573e 100644 --- a/packages/ai-assistant-evals/assistant-eval.spec.ts +++ b/packages/ai-assistant-evals/assistant-eval.spec.ts @@ -70,6 +70,14 @@ const MAX_MINUTES = Number(process.env.EVAL_MAX_MINUTES ?? 15); // A pill that stays in "applying" this long is a stuck host, not a slow tool: // the host's own tool timeout is two minutes. const STUCK_APPLYING_MS = 150_000; +// A host tool can be left unclaimed with its pill spinning in "applying" +// although nothing is running it: the host's tool drain read the message +// before every tool had been built, and never looks at it again. Clicking the +// pill runs the tool and posts its result. The page shows the same spinner +// for a tool that is really running, so the runner waits past the host's own +// 120 s execute timeout, after which a running tool has failed or finished, +// and only then clicks each pill still spinning, once, and records it. +const RECOVER_APPLYING_MS = 125_000; // A turn that shows "Generating results" with no new text for this long is a // stalled request between the ai-bot and the provider; the bot has no timeout // of its own for it. @@ -105,6 +113,16 @@ const MODELS = (process.env.EVAL_MODELS ?? 'Claude Sonnet 4.6') .map((m) => m.trim()) .filter(Boolean); +// Skill sections behind a feature flag (``) are left +// out of the prompt unless the room enabled them. Each name here is enabled +// in the room with `boxel-debug:feature:enable:` before the first +// prompt, in addition to the evaluation's own `skillFeatures`, so a run can +// measure the assistant with a feature on that the evaluation leaves off. +const SKILL_FEATURES = (process.env.EVAL_SKILL_FEATURES ?? '') + .split(',') + .map((f) => f.trim()) + .filter(Boolean); + function slugify(s: string) { return s .toLowerCase() @@ -463,12 +481,16 @@ async function readActivityOnce(page: Page): Promise { } // A loop: the same tool call, with the same arguments, three times. The // bot's own checkCorrectness is left out: it repeats with the same - // arguments on every repair round by design. + // arguments on every repair round by design. So is a call rejected as + // invalid: it never ran, its pill shows no arguments, so several of them + // in one turn look identical, and the failed-turn count above already + // covers them. let toolElements = Array.from( document.querySelectorAll('[data-tool-name]'), ); let calls = toolElements .filter((el) => el.dataset.toolName !== 'checkCorrectness') + .filter((el) => !el.querySelector('[data-test-apply-state="invalid"]')) .map( (el) => `${el.dataset.toolName}|${el.dataset.toolArgumentsLength}|${( @@ -529,11 +551,73 @@ async function stopGeneration(page: Page) { // expensive runs are not stopped; they are graded afterwards. // `botMessagesBefore` is how many finished bot turns the room already had // when the prompt went out; a follow-up prompt waits for a new one. +// Clicks every tool pill still spinning in "applying" that has not been +// clicked before, and returns how many it clicked. See RECOVER_APPLYING_MS. +async function recoverStuckPills( + page: Page, + recoveredTools: string[], +): Promise { + let ids = await page.locator('[data-test-tool-call-id]').evaluateAll((els) => + els + .filter( + (el) => + // The host's own correctness check is slow by design and is not + // left unclaimed the way a model's tool call can be. + el.getAttribute('data-tool-name') !== 'checkCorrectness' && + el.querySelector('[data-test-tool-call-apply="applying"]'), + ) + .map((el) => el.getAttribute('data-test-tool-call-id') ?? ''), + ); + let clicked = 0; + for (let id of ids) { + if (!id || recoveredTools.includes(id)) { + continue; + } + recoveredTools.push(id); + await page + .locator( + `[data-test-tool-call-id="${id}"] [data-test-tool-call-apply="applying"]`, + ) + .first() + .click({ timeout: 5_000 }) + .catch(() => undefined); + clicked++; + } + return clicked; +} + +// Some tools open a room of their own and move the assistant panel to it: +// `listing-remix`, for one, starts a "Remixing …" session. The evaluation's +// conversation carries on in its own room, but the panel no longer shows it, +// so everything read from the page would describe the wrong room. When the +// panel is on another room, reopen the evaluation's room from past sessions. +// Returns true when it had to switch back. +async function returnToRoom(page: Page, roomId: string): Promise { + let current = await page + .locator('[data-test-room]') + .first() + .getAttribute('data-test-room', { timeout: 5_000 }) + .catch(() => undefined); + if (!current || current === roomId) { + return false; + } + await page.locator('[data-test-past-sessions-button]').click(); + await page.locator(`[data-test-enter-room="${roomId}"]`).click(); + await page + .locator(`[data-test-room="${roomId}"]`) + .waitFor({ timeout: 60_000 }); + await page.locator('[data-test-room-settled]').waitFor({ timeout: 60_000 }); + return true; +} + async function waitForIdle( page: Page, deadline: number, botMessagesBefore = 0, onActivity?: (activity: Activity) => void, + recoveredTools: string[] = [], + roomId?: string, + roomReturns: { count: number } = { count: 0 }, ): Promise<{ stoppedBy: RunResult['stoppedBy']; irregularities: string[] }> { let idleSince: number | undefined; let applyingSince: number | undefined; @@ -541,6 +625,9 @@ async function waitForIdle( let lastTextLength = -1; let startedAt = Date.now(); for (;;) { + if (roomId && (await returnToRoom(page, roomId))) { + roomReturns.count++; + } let now = Date.now(); let activity = await readActivity(page); onActivity?.(activity); @@ -563,7 +650,12 @@ async function waitForIdle( lastTextLength = activity.botTextLength; if (activity.applying > 0) { applyingSince ??= now; - if (now - applyingSince > STUCK_APPLYING_MS) { + if ( + now - applyingSince > RECOVER_APPLYING_MS && + (await recoverStuckPills(page, recoveredTools)) > 0 + ) { + applyingSince = now; + } else if (now - applyingSince > STUCK_APPLYING_MS) { return { stoppedBy: 'stuck', irregularities: [] }; } } else { @@ -745,8 +837,11 @@ function classify( ); return { verdict: 'host-failure', reasons }; } - if (analysis.realmCodeWrites === 0) { - reasons.push('no file was written (no applied run-realm-code write)'); + let writes = analysis.realmCodeWrites + analysis.toolWrites; + if (writes === 0) { + reasons.push( + 'no file was written (no applied run-realm-code or other writing tool call)', + ); } if (!cardId) { reasons.push(...cardReasons); @@ -761,7 +856,7 @@ function classify( if ( cardId && cardReasons.length === 0 && - analysis.realmCodeWrites > 0 && + writes > 0 && promptsSent === promptsTotal ) { return { verdict: 'pass', reasons }; @@ -813,6 +908,9 @@ async function runModel( initialCards: [], initialFiles: [], skillsUsed: [], + skillFeatures: [], + recoveredTools: [], + roomReturns: 0, username, stoppedBy: 'error', irregularities: [], @@ -871,14 +969,39 @@ async function runModel( await ensureActMode(page, requestedModel); setStep('confirm the tab is inside the new workspace'); await ensureInsideWorkspace(page, result.realmUrl, result.initialCards); + // The evaluation's own features, plus any EVAL_SKILL_FEATURES adds. + let skillFeatures = [ + ...new Set([...(evaluation?.skillFeatures ?? []), ...SKILL_FEATURES]), + ]; + for (let [index, feature] of skillFeatures.entries()) { + setStep(`enable skill feature ${feature}`); + await sendPrompt( + page, + result.roomId, + `boxel-debug:feature:enable:${feature}`, + index + 1, + ); + // The bot answers the command itself, without the model; its reply + // carries no usage, so it is not counted as a turn. + await expect( + page.getByText(`${feature} is now enabled`).first(), + ).toBeVisible({ timeout: 60_000 }); + } + result.skillFeatures = skillFeatures; // One safety clock for the whole run, follow-ups included. let deadline = Date.now() + MAX_MINUTES * 60_000; let botMessagesBefore = 0; + let roomReturns = { count: 0 }; for (let [index, prompt] of prompts.entries()) { setStep( index === 0 ? 'send prompt' : `send follow-up prompt ${index + 1}`, ); - await sendPrompt(page, result.roomId, prompt, index + 1); + await sendPrompt( + page, + result.roomId, + prompt, + skillFeatures.length + index + 1, + ); result.promptsSent = index + 1; setStep( index === 0 @@ -904,6 +1027,9 @@ async function runModel( }); } }, + result.recoveredTools, + result.roomId, + roomReturns, ); result.stoppedBy = waited.stoppedBy; result.irregularities = waited.irregularities; @@ -912,6 +1038,10 @@ async function runModel( } botMessagesBefore = (await readActivity(page)).botMessages; } + if (result.roomId && (await returnToRoom(page, result.roomId))) { + roomReturns.count++; + } + result.roomReturns = roomReturns.count; setStep('check render'); let rendered = await findRenderedCard( @@ -941,6 +1071,18 @@ async function runModel( ); result.verdict = verdict.verdict; result.reasons = verdict.reasons; + // An enabled feature whose skill the model never opened means the run + // did not exercise it, whatever the verdict says. + for (let feature of result.skillFeatures) { + let read = result.analysis.filesRead.some((url) => + url.endsWith(`/skills/${feature}/SKILL.md`), + ); + if (!read) { + result.reasons.push( + `skill feature "${feature}" was enabled but skills/${feature}/SKILL.md was never read`, + ); + } + } } catch (error) { result.error = error instanceof Error ? error.message : String(error); result.reasons.push( diff --git a/packages/ai-assistant-evals/eval-card.ts b/packages/ai-assistant-evals/eval-card.ts index 1174f6bc924..35971789c8a 100644 --- a/packages/ai-assistant-evals/eval-card.ts +++ b/packages/ai-assistant-evals/eval-card.ts @@ -20,6 +20,8 @@ export interface Evaluation { // Absolute URLs, as the realm serializes them. initialCards: string[]; initialFiles: string[]; + // Skill features to enable in the room before the first prompt. + skillFeatures: string[]; adoptsFrom: { module: string; name: string }; } @@ -87,6 +89,11 @@ export async function loadEvaluation( successCriteria: attributes.successCriteria ?? '', initialCards: linkedIds(doc, 'initialCards', id), initialFiles: linkedIds(doc, 'initialFiles', id), + skillFeatures: Array.isArray(attributes.skillFeatures) + ? attributes.skillFeatures.filter( + (f: unknown): f is string => typeof f === 'string' && f.trim() !== '', + ) + : [], adoptsFrom: doc.data.meta.adoptsFrom, }; } diff --git a/packages/ai-assistant-evals/eval-realm/Evaluation/reuse-listing-ownership.json b/packages/ai-assistant-evals/eval-realm/Evaluation/reuse-listing-ownership.json index ca7f31221e1..fd7f91abc94 100644 --- a/packages/ai-assistant-evals/eval-realm/Evaluation/reuse-listing-ownership.json +++ b/packages/ai-assistant-evals/eval-realm/Evaluation/reuse-listing-ownership.json @@ -2,17 +2,18 @@ "data": { "type": "card", "attributes": { - "assistantPrompt": "I want to start publishing articles on our site \u2014 posts with an author and a category, and somewhere to see them all. Create one example instance and show it on screen.", + "assistantPrompt": "I want to keep track of our wine collection \u2014 each bottle with its producer, vintage and what we paid for it, and a cellar page where we can see them all. Create one example instance and show it on screen.", "followUpPrompts": [ "We'd like our own version of this now \u2014 something we can restyle and add fields to, that won't shift under us when someone else changes theirs." ], - "successCriteria": "1. The assistant searched the catalog before writing any `.gts`.\n2. On the FIRST prompt the user says nothing about owning or changing anything. The assistant must NOT install or remix a Listing there \u2014 a Listing hands over a copy, and nothing was asked to be owned.\n3. On the first prompt it reaches the catalog's blog definitions by module URL \u2014 `BlogPost`, `Author`, `BlogCategory` under `@cardstack/catalog/7af9aa-blog-app/` \u2014 linking, extending or instantiating them.\n4. It does not author its own post, author or category card while those catalog definitions exist.\n5. At least one instance exists after the first prompt and renders with no error boundary.\n6. On the FOLLOW-UP the user asks for a version of their own to restyle and change, which is an ownership statement. The assistant now installs or remixes the blog Listing rather than continuing to reference it.\n7. It takes that copy through the install or remix mechanism, never by hand-copying files.\n8. It does not treat the follow-up as a reason to rebuild anything from scratch \u2014 the copy comes from the catalog.\n9. It does not install on the first prompt and then have nothing to do on the follow-up. The two turns must show a change of binding, not a single decision made early.", + "successCriteria": "1. The assistant searched the catalog before writing any `.gts`.\n2. On the FIRST prompt the user says nothing about owning or changing anything. The assistant must NOT install or remix a Listing there \u2014 a Listing hands over a copy, and nothing was asked to be owned.\n3. On the first prompt it reaches the catalog's wine definitions by module URL \u2014 `WineCellar` and `WineBottle` under `@cardstack/catalog/4b6602-wine-cellar-card-definition/` \u2014 linking, extending or instantiating them.\n4. It does not author its own cellar or bottle card while those catalog definitions exist.\n5. After the first prompt at least one instance exists, is shown on screen, and renders with no error boundary.\n6. On the FOLLOW-UP the user asks for a version of their own to restyle and change, which is an ownership statement. The assistant now installs or remixes the Wine Cellar Listing rather than continuing to reference it.\n7. It takes that copy through the install or remix tool (`listing-install` / `listing-remix`), never by copying the catalog source by hand.\n8. It does not treat the follow-up as a reason to rebuild anything from scratch \u2014 the copy comes from the catalog.\n9. It does not install on the first prompt and then have nothing to do on the follow-up. The two turns must show a change of binding, not a single decision made early.\n10. The follow-up reply says in one line what the user now holds: their own copy, which will not receive later catalog changes.", "cardInfo": { "name": "Reuse: reference first, own on request", - "summary": "A brief that matches a whole Listing but asks for no ownership, followed by an explicit request to own it \u2014 the binding should change between the two turns.", - "notes": "This card exists because reference-and-install look equally successful from the outside: both leave a working blog on screen. The difference is what the user is left holding \u2014 definitions that stay wired to the catalog, or a copy that will not receive upstream fixes. Criterion 2 is the one under test; criterion 6 guards the over-correction, where the assistant refuses to copy even when asked.\n\nTiming targets do not apply: this is a two-turn build and will report ROUGH on turns and seconds as a matter of course.\n\nThe blog domain was chosen because it carries a real Listing (19 specs) and appears in no skill example, keeping eval vocabulary disjoint from the skill's own.", + "summary": "The first prompt asks for a wine collection with bottles and a cellar page, which the catalog's Wine Cellar Listing already answers; the second asks for a version of their own to restyle. This eval tests whether the selected models reference the catalog definitions first and install or remix the Listing only once ownership is asked for.", + "notes": "This card exists because reference-and-install look equally successful from the outside: both leave a working cellar on screen. The difference is what the user is left holding \u2014 definitions that stay wired to the catalog, or a copy that will not receive upstream fixes. Criterion 2 is the one under test; criterion 6 guards the over-correction, where the assistant refuses to copy even when asked.\n\nTiming targets do not apply: this is a two-turn build and will report ROUGH on turns and seconds as a matter of course.\n\nThe Wine Cellar Listing (`4b6602-wine-cellar-card-definition`) was chosen because it installs and remixes cleanly: every module it needs has a Spec, so the follow-up is completable. Its shape matches the case under test \u2014 a collection card (`WineCellar.bottles`, linksToMany) over item cards (`WineBottle`) \u2014 and the domain appears in no skill example, keeping eval vocabulary disjoint from the skill's own. Before changing the domain, check that the new Listing remixes into a fresh realm with no unresolved imports.", "cardThumbnailURL": null - } + }, + "skillFeatures": ["catalog-reuse"] }, "relationships": { "cardInfo.theme": { diff --git a/packages/ai-assistant-evals/eval-realm/Evaluation/reuse-service-desk-build.json b/packages/ai-assistant-evals/eval-realm/Evaluation/reuse-service-desk-build.json index c0551c69019..482e4d70628 100644 --- a/packages/ai-assistant-evals/eval-realm/Evaluation/reuse-service-desk-build.json +++ b/packages/ai-assistant-evals/eval-realm/Evaluation/reuse-service-desk-build.json @@ -12,7 +12,8 @@ "summary": "One realistic brief with eight catalog reuse opportunities across card, field, command and component \u2014 graded on the ratio taken, not a single decision.", "notes": "Timing targets do not apply to this card. It is a multi-part build with a follow-up, so the <=8 turn and <=120s targets will report ROUGH as a matter of course; that is not a finding. Judge it on the reuse ratio and the must-not-hand-roll list.\n\nThe service-desk domain carries no Listing, so this card exercises the Spec/parts path only. Install and remix of a whole bundle are deliberately unmeasured here.\n\nDomain chosen because no other evaluation touches it, keeping eval vocabulary disjoint from the skill's own examples.", "cardThumbnailURL": null - } + }, + "skillFeatures": ["catalog-reuse"] }, "relationships": { "cardInfo.theme": { diff --git a/packages/ai-assistant-evals/eval-realm/evaluation.gts b/packages/ai-assistant-evals/eval-realm/evaluation.gts index 3effac792bb..b83d598090f 100644 --- a/packages/ai-assistant-evals/eval-realm/evaluation.gts +++ b/packages/ai-assistant-evals/eval-realm/evaluation.gts @@ -61,7 +61,19 @@ function seconds(value: number | undefined | null) { return value == null ? '–' : `${Math.round(value)} s`; } -const TIERS = ['pending', 'failed', 'rough', 'good', 'great'] as const; +const TIERS = [ + 'pending', + 'unscored', + 'failed', + 'rough', + 'good', + 'great', +] as const; + +// Verdicts that say the run says nothing about the model: the host, the +// ai-bot or the runner broke first. Such a result is left unscored rather +// than counted as a zero, and the report's mean leaves it out. +const UNSCORED_VERDICTS = ['host-failure', 'bot-failure', 'runner-failure']; export type EffectivenessTier = (typeof TIERS)[number]; // The one place the score-to-tier thresholds live: the result card computes @@ -101,6 +113,11 @@ export class EvaluationCard extends CardDef { // cards in the stack so they are part of the message context. @field initialCards = linksToMany(CardDef); @field initialFiles = linksToMany(FileDef); + // Skill features the evaluation needs switched on (`boxel-debug:feature: + // enable:` in the room before the first prompt). A skill section + // between `` markers is left out of the prompt + // otherwise, so an evaluation of that section must name it here. + @field skillFeatures = containsMany(StringField); @field sessionReports = linksToMany(() => EvaluationReportCard, { query: { @@ -296,6 +313,9 @@ export class EvaluationResultCard extends CardDef { // Empty until the quality score is in. @field effectivenessScore = contains(NumberField, { computeVia: function (this: EvaluationResultCard) { + if (this.verdict && UNSCORED_VERDICTS.includes(this.verdict)) { + return undefined; + } if (this.verdict && this.verdict !== 'pass') { return 0; } @@ -320,6 +340,9 @@ export class EvaluationResultCard extends CardDef { enumField(StringField, { options: [...TIERS] }), { computeVia: function (this: EvaluationResultCard): EffectivenessTier { + if (this.verdict && UNSCORED_VERDICTS.includes(this.verdict)) { + return 'unscored'; + } return tierForScore(this.effectivenessScore); }, }, diff --git a/packages/ai-assistant-evals/room-analysis.ts b/packages/ai-assistant-evals/room-analysis.ts index 865e2a77de1..3bcbf03081f 100644 --- a/packages/ai-assistant-evals/room-analysis.ts +++ b/packages/ai-assistant-evals/room-analysis.ts @@ -25,6 +25,13 @@ export interface RoomAnalysis { unansweredToolCalls: number; // realm.fs writes in run-realm-code calls the host applied. realmCodeWrites: number; + // Applied calls to the other tools that write into a realm (copy, patch, + // save, listing install/remix/use, ...). A card those write is as much the + // model's output as one written with run-realm-code. + toolWrites: number; + // URLs the model read with readRealmFile. A skill file under test that is + // missing here was never in the model's context. + filesRead: string[]; filesWritten: string[]; showCardIds: string[]; lastBotBody: string; @@ -33,6 +40,10 @@ export interface RoomAnalysis { const BOT_MESSAGE_MSGTYPE = 'app.boxel.message'; const TOOL_REQUESTS_KEY = 'app.boxel.toolRequests'; // Every write call counts; the path is recorded only when it is a literal. +// Tools, other than run-realm-code, whose applied call writes into a realm. +// Tool names carry a hash suffix (`copy-card_eefc`), so match the prefix. +const WRITE_TOOL = + /^(copy-card|copy-card-to-stack|copy-and-edit|copy-source|copy-file-to-realm|patch-card-instance|patch-fields|patch-code|patch-theme|save-card|write-text-file|write-binary-file|apply-markdown-edit|listing-install|listing-remix|listing-use)(_|$)/; const REALM_CODE_WRITE = /realm\.fs\.(?:writeText|replace)\(\s*(?:(['"`])([^'"`]+)\1)?/g; @@ -80,6 +91,8 @@ export function analyzeRoom( failedToolCalls: [], unansweredToolCalls: 0, realmCodeWrites: 0, + toolWrites: 0, + filesRead: [], filesWritten: [], showCardIds: [], lastBotBody: '', @@ -162,6 +175,18 @@ export function analyzeRoom( } else if (outcome.key !== 'applied') { result.failedToolCalls.push({ name, reason: outcome.reason }); } + if (name === 'readRealmFile') { + let args = parseArguments(request.arguments); + let urls = args.attributes?.urls ?? args.urls; + if (Array.isArray(urls)) { + result.filesRead.push( + ...urls.filter((u: unknown): u is string => typeof u === 'string'), + ); + } + } + if (WRITE_TOOL.test(name) && outcome?.key === 'applied') { + result.toolWrites++; + } if (name.startsWith('run-realm-code') && outcome?.key === 'applied') { // The host nests top-level fields under `attributes` before it runs a // call, so a model's flat `code` is a real write too. @@ -210,5 +235,6 @@ export function analyzeRoom( } result.filesWritten = [...new Set(result.filesWritten)]; + result.filesRead = [...new Set(result.filesRead)]; return result; } diff --git a/packages/ai-assistant-evals/run-result.ts b/packages/ai-assistant-evals/run-result.ts index ba0d4c863a3..4a1533bae38 100644 --- a/packages/ai-assistant-evals/run-result.ts +++ b/packages/ai-assistant-evals/run-result.ts @@ -36,6 +36,9 @@ export interface RunResult { initialFiles: string[]; // Source URLs of the skill files the room had enabled. skillsUsed: string[]; + // Skill features enabled in the room before the first prompt + // (EVAL_SKILL_FEATURES). + skillFeatures: string[]; username: string; stoppedBy: | 'idle' @@ -46,6 +49,12 @@ export interface RunResult { | 'irregularity' | 'error'; irregularities: string[]; + // Tool calls whose pill sat in "applying" with nothing running them, and + // which the runner clicked to run (RECOVER_APPLYING_MS in the spec). + recoveredTools: string[]; + // How often a tool (listing-remix, for one) moved the assistant panel to a + // room of its own and the runner reopened the evaluation's room. + roomReturns: number; // File name, next to this result, of what the workspace held when the run // ended: every source file and every indexed card document. workspaceSnapshot?: string; From 414cfbf8c787009bdd7a47af8bc8d56bba5fa46a Mon Sep 17 00:00:00 2001 From: Fadhlan Ridhwanallah Date: Tue, 6 Oct 2026 21:01:40 +0700 Subject: [PATCH 5/7] Drop the runner's stuck-pill click The host now marks a message finished only after its tools are built, so a tool is no longer left unclaimed for the runner to click; a pill still stuck past the host's tool timeout ends the run as a host failure, as before. Co-Authored-By: Claude Opus 5.5 --- .../ai-assistant-evals/assistant-eval.spec.ts | 53 +------------------ packages/ai-assistant-evals/run-result.ts | 3 -- 2 files changed, 1 insertion(+), 55 deletions(-) diff --git a/packages/ai-assistant-evals/assistant-eval.spec.ts b/packages/ai-assistant-evals/assistant-eval.spec.ts index ba06649573e..f1460b0d906 100644 --- a/packages/ai-assistant-evals/assistant-eval.spec.ts +++ b/packages/ai-assistant-evals/assistant-eval.spec.ts @@ -70,14 +70,6 @@ const MAX_MINUTES = Number(process.env.EVAL_MAX_MINUTES ?? 15); // A pill that stays in "applying" this long is a stuck host, not a slow tool: // the host's own tool timeout is two minutes. const STUCK_APPLYING_MS = 150_000; -// A host tool can be left unclaimed with its pill spinning in "applying" -// although nothing is running it: the host's tool drain read the message -// before every tool had been built, and never looks at it again. Clicking the -// pill runs the tool and posts its result. The page shows the same spinner -// for a tool that is really running, so the runner waits past the host's own -// 120 s execute timeout, after which a running tool has failed or finished, -// and only then clicks each pill still spinning, once, and records it. -const RECOVER_APPLYING_MS = 125_000; // A turn that shows "Generating results" with no new text for this long is a // stalled request between the ai-bot and the provider; the bot has no timeout // of its own for it. @@ -551,41 +543,6 @@ async function stopGeneration(page: Page) { // expensive runs are not stopped; they are graded afterwards. // `botMessagesBefore` is how many finished bot turns the room already had // when the prompt went out; a follow-up prompt waits for a new one. -// Clicks every tool pill still spinning in "applying" that has not been -// clicked before, and returns how many it clicked. See RECOVER_APPLYING_MS. -async function recoverStuckPills( - page: Page, - recoveredTools: string[], -): Promise { - let ids = await page.locator('[data-test-tool-call-id]').evaluateAll((els) => - els - .filter( - (el) => - // The host's own correctness check is slow by design and is not - // left unclaimed the way a model's tool call can be. - el.getAttribute('data-tool-name') !== 'checkCorrectness' && - el.querySelector('[data-test-tool-call-apply="applying"]'), - ) - .map((el) => el.getAttribute('data-test-tool-call-id') ?? ''), - ); - let clicked = 0; - for (let id of ids) { - if (!id || recoveredTools.includes(id)) { - continue; - } - recoveredTools.push(id); - await page - .locator( - `[data-test-tool-call-id="${id}"] [data-test-tool-call-apply="applying"]`, - ) - .first() - .click({ timeout: 5_000 }) - .catch(() => undefined); - clicked++; - } - return clicked; -} - // Some tools open a room of their own and move the assistant panel to it: // `listing-remix`, for one, starts a "Remixing …" session. The evaluation's // conversation carries on in its own room, but the panel no longer shows it, @@ -615,7 +572,6 @@ async function waitForIdle( deadline: number, botMessagesBefore = 0, onActivity?: (activity: Activity) => void, - recoveredTools: string[] = [], roomId?: string, roomReturns: { count: number } = { count: 0 }, ): Promise<{ stoppedBy: RunResult['stoppedBy']; irregularities: string[] }> { @@ -650,12 +606,7 @@ async function waitForIdle( lastTextLength = activity.botTextLength; if (activity.applying > 0) { applyingSince ??= now; - if ( - now - applyingSince > RECOVER_APPLYING_MS && - (await recoverStuckPills(page, recoveredTools)) > 0 - ) { - applyingSince = now; - } else if (now - applyingSince > STUCK_APPLYING_MS) { + if (now - applyingSince > STUCK_APPLYING_MS) { return { stoppedBy: 'stuck', irregularities: [] }; } } else { @@ -909,7 +860,6 @@ async function runModel( initialFiles: [], skillsUsed: [], skillFeatures: [], - recoveredTools: [], roomReturns: 0, username, stoppedBy: 'error', @@ -1027,7 +977,6 @@ async function runModel( }); } }, - result.recoveredTools, result.roomId, roomReturns, ); diff --git a/packages/ai-assistant-evals/run-result.ts b/packages/ai-assistant-evals/run-result.ts index 4a1533bae38..a71e26c13f3 100644 --- a/packages/ai-assistant-evals/run-result.ts +++ b/packages/ai-assistant-evals/run-result.ts @@ -49,9 +49,6 @@ export interface RunResult { | 'irregularity' | 'error'; irregularities: string[]; - // Tool calls whose pill sat in "applying" with nothing running them, and - // which the runner clicked to run (RECOVER_APPLYING_MS in the spec). - recoveredTools: string[]; // How often a tool (listing-remix, for one) moved the assistant panel to a // room of its own and the runner reopened the evaluation's room. roomReturns: number; From cc1964371dbbdb24547f04bec4cd065444bfc103 Mon Sep 17 00:00:00 2001 From: Fadhlan Ridhwanallah Date: Tue, 6 Oct 2026 21:24:20 +0700 Subject: [PATCH 6/7] Count every realm-writing tool, keep room return and feature waits from failing a run, and mark unscored results WRITE_TOOL covers the remaining host tools and catalog commands that write into a realm, and the summary and result card report that total. Room return reads the current room without waiting and retries on the next poll when a switch fails. The feature-enable wait matches the no-reply limit and stops at once on an unknown feature. Unscored results get their own style, and the service-desk summary counts the seven opportunities its criteria grade. Co-Authored-By: Claude Opus 5.5 --- .claude/skills/ai-assistant-evals/SKILL.md | 6 +-- packages/ai-assistant-evals/README.md | 5 +- .../ai-assistant-evals/assistant-eval.spec.ts | 54 ++++++++++++------- .../Evaluation/reuse-service-desk-build.json | 2 +- .../eval-realm/evaluation.gts | 31 ++++++++--- packages/ai-assistant-evals/room-analysis.ts | 9 ++-- packages/ai-assistant-evals/run-eval.ts | 4 +- 7 files changed, 76 insertions(+), 35 deletions(-) diff --git a/.claude/skills/ai-assistant-evals/SKILL.md b/.claude/skills/ai-assistant-evals/SKILL.md index 9fc708d5c8e..d8205089e2f 100644 --- a/.claude/skills/ai-assistant-evals/SKILL.md +++ b/.claude/skills/ai-assistant-evals/SKILL.md @@ -144,8 +144,8 @@ When one appears: is also printed at the end of the run. Columns: result grade, model id used, effort the room carried, verdict, turns, -tool calls by name with counts, run-realm-code writes, cost, wall time, whether a card -rendered and where (`stack` or `preview`), notes with the reasons, the missed +tool calls by name with counts, realm writes (run-realm-code writes plus other +writing tool calls), cost, wall time, whether a card rendered and where (`stack` or `preview`), notes with the reasons, the missed benchmarks, and the room id. The grade is the one-word answer per model: @@ -200,7 +200,7 @@ After every run, two checks on the money, whatever the grade: Verdicts: - `pass` — a card from the new workspace rendered with no error boundary, and at - least one run-realm-code write was applied. + least one applied run-realm-code or other writing tool call. - `model-failure` — the model did not get there: no write, or no card. Reasons are listed. - `host-failure` — a tool pill stayed in "applying" past the host's own two-minute tool timeout, or a tool call never got a result. The model may be diff --git a/packages/ai-assistant-evals/README.md b/packages/ai-assistant-evals/README.md index 12a5f4d28b5..30f06133497 100644 --- a/packages/ai-assistant-evals/README.md +++ b/packages/ai-assistant-evals/README.md @@ -140,8 +140,9 @@ A run also prints the table at the end. 4. Looks for a rendered card in the stack that is not the workspace index and has no error state; screenshots the page. 5. Reads the room's Matrix events: turns, tokens, cost, tool calls and their - outcomes, run-realm-code writes, files written. -6. Classifies: `pass` (card rendered), `model` failure (never wrote, wrong + outcomes, run-realm-code writes and other writing tool calls, files written. +6. Classifies: `pass` (card rendered, and at least one applied run-realm-code + or other writing tool call), `model` failure (never wrote, wrong tool, gave up), or `host` failure (a pill stuck applying, a tool with no result, the tab left the workspace). Keeps the room id so the room can be inspected afterwards. diff --git a/packages/ai-assistant-evals/assistant-eval.spec.ts b/packages/ai-assistant-evals/assistant-eval.spec.ts index f1460b0d906..4af6ff73889 100644 --- a/packages/ai-assistant-evals/assistant-eval.spec.ts +++ b/packages/ai-assistant-evals/assistant-eval.spec.ts @@ -548,23 +548,33 @@ async function stopGeneration(page: Page) { // conversation carries on in its own room, but the panel no longer shows it, // so everything read from the page would describe the wrong room. When the // panel is on another room, reopen the evaluation's room from past sessions. -// Returns true when it had to switch back. +// Returns true when it switched back. It runs on every poll, so reading the +// current room never waits, and a switch that fails (the past-sessions button +// disabled, the host reloading) returns false for the next poll to retry +// rather than failing the run. async function returnToRoom(page: Page, roomId: string): Promise { let current = await page - .locator('[data-test-room]') - .first() - .getAttribute('data-test-room', { timeout: 5_000 }) - .catch(() => undefined); + .evaluate( + () => + document + .querySelector('[data-test-room]') + ?.getAttribute('data-test-room') ?? null, + ) + .catch(() => null); if (!current || current === roomId) { return false; } - await page.locator('[data-test-past-sessions-button]').click(); - await page.locator(`[data-test-enter-room="${roomId}"]`).click(); - await page - .locator(`[data-test-room="${roomId}"]`) - .waitFor({ timeout: 60_000 }); - await page.locator('[data-test-room-settled]').waitFor({ timeout: 60_000 }); - return true; + try { + await page.locator('[data-test-past-sessions-button]').click(); + await page.locator(`[data-test-enter-room="${roomId}"]`).click(); + await page + .locator(`[data-test-room="${roomId}"]`) + .waitFor({ timeout: 60_000 }); + await page.locator('[data-test-room-settled]').waitFor({ timeout: 60_000 }); + return true; + } catch { + return false; + } } async function waitForIdle( @@ -932,10 +942,17 @@ async function runModel( index + 1, ); // The bot answers the command itself, without the model; its reply - // carries no usage, so it is not counted as a turn. - await expect( - page.getByText(`${feature} is now enabled`).first(), - ).toBeVisible({ timeout: 60_000 }); + // carries no usage, so it is not counted as a turn. A name the bot + // does not know gets "There is no feature named …" instead, which ends + // the run straight away rather than at the timeout. + let enabled = page.getByText(`${feature} is now enabled`).first(); + let unknown = page.getByText('There is no feature named').first(); + await expect(enabled.or(unknown)).toBeVisible({ timeout: NO_REPLY_MS }); + if (await unknown.isVisible()) { + throw new Error( + `The ai-bot has no skill feature named "${feature}"; check the evaluation's skillFeatures and EVAL_SKILL_FEATURES.`, + ); + } } result.skillFeatures = skillFeatures; // One safety clock for the whole run, follow-ups included. @@ -1021,7 +1038,8 @@ async function runModel( result.verdict = verdict.verdict; result.reasons = verdict.reasons; // An enabled feature whose skill the model never opened means the run - // did not exercise it, whatever the verdict says. + // did not exercise it, whatever the verdict says. This assumes each + // feature's skill lives in a skills//SKILL.md directory. for (let feature of result.skillFeatures) { let read = result.analysis.filesRead.some((url) => url.endsWith(`/skills/${feature}/SKILL.md`), @@ -1201,7 +1219,7 @@ test.afterAll(async () => { ) .join(', ') : ''; - let writes = a ? `${a.realmCodeWrites}` : ''; + let writes = a ? `${a.realmCodeWrites + a.toolWrites}` : ''; return `| ${graded.grade} | ${r.modelId ?? r.requestedModel} | ${r.reasoningEffort ?? '–'} | ${r.verdict} | ${ a?.turns ?? '–' } | ${tools} | ${writes} | ${a ? `$${a.costUsd.toFixed(3)}` : '–'} | ${ diff --git a/packages/ai-assistant-evals/eval-realm/Evaluation/reuse-service-desk-build.json b/packages/ai-assistant-evals/eval-realm/Evaluation/reuse-service-desk-build.json index 482e4d70628..9d246ae0173 100644 --- a/packages/ai-assistant-evals/eval-realm/Evaluation/reuse-service-desk-build.json +++ b/packages/ai-assistant-evals/eval-realm/Evaluation/reuse-service-desk-build.json @@ -9,7 +9,7 @@ "successCriteria": "1. The assistant searched the catalog before writing any `.gts`, and searched per part of the brief rather than once for the deliverable as a whole.\n2. If it models the ticket itself, it reuses the catalog's `Ticket` definition rather than authoring an equivalent ticket card.\n3. If it models who reported the ticket, it links or extends the catalog's `SupportContact` rather than declaring name/email string fields.\n4. If it models the owning queue, it links the catalog's `Queue` rather than a string or a bespoke queue card.\n5. If it models the handling agent, it links the catalog's `SupportAgent` rather than a string.\n6. Ticket status uses the catalog's `TicketStatusField` \u2014 never a bare string or a hand-rolled enum.\n7. Response targets reuse the catalog's SLA definitions (`Sla`, `SlaPolicy`, `SlaTimerField`) rather than hand-rolled deadline arithmetic.\n8. The escalate action reuses the catalog's escalation command rather than a hand-written function \u2014 or, if none fits, the assistant says so explicitly and records the gap.\n9. Reuse ratio: at least 6 of the 7 opportunities in criteria 2-8 are answered from the catalog.\n10. Nothing on this list was hand-built: ticket status as a bare string or enum; a bespoke reporter card made of name/email strings; a hand-written escalate function; a hand-rolled SLA timer.\n11. On the follow-up, the assistant specializes the catalog definition rather than copying it wholesale \u2014 unless it explains why the divergence needs a copy.\n12. On the follow-up, the reply names in one line what the user now has: a reference that stays in sync with the catalog, or a copy they own that will not receive upstream fixes.\n13. At least one instance exists and renders with no error boundary. A new definition is not required if the catalog's own cards answer the brief.", "cardInfo": { "name": "Reuse: service desk build", - "summary": "One realistic brief with eight catalog reuse opportunities across card, field, command and component \u2014 graded on the ratio taken, not a single decision.", + "summary": "One realistic brief with seven catalog reuse opportunities across card, field and command \u2014 graded on the ratio taken, not a single decision.", "notes": "Timing targets do not apply to this card. It is a multi-part build with a follow-up, so the <=8 turn and <=120s targets will report ROUGH as a matter of course; that is not a finding. Judge it on the reuse ratio and the must-not-hand-roll list.\n\nThe service-desk domain carries no Listing, so this card exercises the Spec/parts path only. Install and remix of a whole bundle are deliberately unmeasured here.\n\nDomain chosen because no other evaluation touches it, keeping eval vocabulary disjoint from the skill's own examples.", "cardThumbnailURL": null }, diff --git a/packages/ai-assistant-evals/eval-realm/evaluation.gts b/packages/ai-assistant-evals/eval-realm/evaluation.gts index b83d598090f..ef98860d195 100644 --- a/packages/ai-assistant-evals/eval-realm/evaluation.gts +++ b/packages/ai-assistant-evals/eval-realm/evaluation.gts @@ -69,12 +69,12 @@ const TIERS = [ 'good', 'great', ] as const; +export type EffectivenessTier = (typeof TIERS)[number]; -// Verdicts that say the run says nothing about the model: the host, the -// ai-bot or the runner broke first. Such a result is left unscored rather -// than counted as a zero, and the report's mean leaves it out. +// Verdicts where the host, ai-bot or runner broke first: the run says nothing +// about the model. Such a result is left unscored rather than counted as a +// zero, and the report's mean leaves it out. const UNSCORED_VERDICTS = ['host-failure', 'bot-failure', 'runner-failure']; -export type EffectivenessTier = (typeof TIERS)[number]; // The one place the score-to-tier thresholds live: the result card computes // its own tier with it, and the report tints its headline number with it. @@ -309,8 +309,9 @@ export class EvaluationResultCard extends CardDef { // One number, 0 to 100. Half of it is the judged quality; the rest rewards // getting there in few turns, cheaply, with the prompt cache working, and - // quickly. A run that did not pass, or that a judge scored zero, is zero. - // Empty until the quality score is in. + // quickly. A run the model failed, or that a judge scored zero, is zero; a + // run with an unscored verdict has no score. Otherwise empty until the + // quality score is in. @field effectivenessScore = contains(NumberField, { computeVia: function (this: EvaluationResultCard) { if (this.verdict && UNSCORED_VERDICTS.includes(this.verdict)) { @@ -512,6 +513,12 @@ export class EvaluationResultCard extends CardDef { .tier-failed { background: #ffd6d6; } + .tier-unscored { + background: transparent; + border: 1px dashed var(--boxel-300); + color: var(--boxel-450); + font-style: italic; + } .numbers { margin: 0; display: grid; @@ -664,6 +671,12 @@ export class EvaluationResultCard extends CardDef { background: #fde4e2; color: #8d231f; } + .tier-unscored { + background: transparent; + border: 1px dashed var(--line); + color: var(--ink-soft); + font-style: italic; + } .who { flex: 1 1 11rem; @@ -1005,6 +1018,12 @@ export class EvaluationReportCard extends CardDef { background: var(--sunken); color: var(--ink-soft); } + .tier-unscored { + background: transparent; + border: 1px dashed var(--line); + color: var(--ink-soft); + font-style: italic; + } .results { display: flex; diff --git a/packages/ai-assistant-evals/room-analysis.ts b/packages/ai-assistant-evals/room-analysis.ts index 3bcbf03081f..d40cd90a475 100644 --- a/packages/ai-assistant-evals/room-analysis.ts +++ b/packages/ai-assistant-evals/room-analysis.ts @@ -39,11 +39,14 @@ export interface RoomAnalysis { const BOT_MESSAGE_MSGTYPE = 'app.boxel.message'; const TOOL_REQUESTS_KEY = 'app.boxel.toolRequests'; -// Every write call counts; the path is recorded only when it is a literal. // Tools, other than run-realm-code, whose applied call writes into a realm. -// Tool names carry a hash suffix (`copy-card_eefc`), so match the prefix. +// Tool names carry a hash suffix (`copy-card_eefc`), so match the prefix. The +// list is kept by hand against packages/host/app/tools and the catalog's +// commands (packages/catalog/contents/commands). const WRITE_TOOL = - /^(copy-card|copy-card-to-stack|copy-and-edit|copy-source|copy-file-to-realm|patch-card-instance|patch-fields|patch-code|patch-theme|save-card|write-text-file|write-binary-file|apply-markdown-edit|listing-install|listing-remix|listing-use)(_|$)/; + /^(copy-card|copy-card-to-stack|copy-and-edit|copy-source|copy-file-to-realm|download-file-to-realm|patch-card-instance|patch-fields|patch-code|patch-theme|save-card|store-add|write-text-file|write-binary-file|apply-markdown-edit|add-field-to-card-definition|execute-atomic-operations|transform-cards|create-specs|generate-readme-spec|generate-example-cards|generate-thumbnail|migrate-skill|listing-create|listing-generate-example|listing-install|listing-remix|listing-use)(_|$)/; +// realm.fs writes inside run-realm-code. Every write call counts; the path is +// recorded only when it is a literal. const REALM_CODE_WRITE = /realm\.fs\.(?:writeText|replace)\(\s*(?:(['"`])([^'"`]+)\1)?/g; diff --git a/packages/ai-assistant-evals/run-eval.ts b/packages/ai-assistant-evals/run-eval.ts index 4930aeb6ac8..b9672077da4 100644 --- a/packages/ai-assistant-evals/run-eval.ts +++ b/packages/ai-assistant-evals/run-eval.ts @@ -262,8 +262,8 @@ async function createResultCard( `grade ${graded.grade}`, ...result.reasons, ...graded.misses, - ...(a && a.realmCodeWrites - ? [`${a.realmCodeWrites} realm write(s)`] + ...(a && a.realmCodeWrites + a.toolWrites + ? [`${a.realmCodeWrites + a.toolWrites} realm write(s)`] : []), ...(result.cardRendered ? [`rendered ${result.cardRendered} in the ${result.renderedIn}`] From da8f1a9b2b3c8f41ba664c0a396555a7a5911b27 Mon Sep 17 00:00:00 2001 From: Fadhlan Ridhwanallah Date: Tue, 6 Oct 2026 21:25:08 +0700 Subject: [PATCH 7/7] Drop the unscored style from the report's lead stat, which never shows that tier Co-Authored-By: Claude Opus 5.5 --- packages/ai-assistant-evals/eval-realm/evaluation.gts | 6 ------ 1 file changed, 6 deletions(-) diff --git a/packages/ai-assistant-evals/eval-realm/evaluation.gts b/packages/ai-assistant-evals/eval-realm/evaluation.gts index ef98860d195..b87bb785f25 100644 --- a/packages/ai-assistant-evals/eval-realm/evaluation.gts +++ b/packages/ai-assistant-evals/eval-realm/evaluation.gts @@ -1018,12 +1018,6 @@ export class EvaluationReportCard extends CardDef { background: var(--sunken); color: var(--ink-soft); } - .tier-unscored { - background: transparent; - border: 1px dashed var(--line); - color: var(--ink-soft); - font-style: italic; - } .results { display: flex;