From fa90c7b249c20f3ec188e6ace8a51604ca7b2947 Mon Sep 17 00:00:00 2001 From: Chris Trzesniewski Date: Tue, 13 Oct 2026 09:42:59 +0200 Subject: [PATCH 1/5] feat(ai): add EvaluateError for typed evaluate failures - Add EvaluateError, a core error class an evaluate adapter can throw when the provider answered but the call cannot become a result - It carries a stable code plus the usage, model and raw wire answers the provider already returned - Set name to EvaluateError in the constructor, as SkillLimitError does - Add EVALUATE_REFUSAL_CODE as the code for a refused question - Export both and the EvaluateErrorInit type from @tanstack/ai and @tanstack/ai/adapters - Add a unit test for the fields, name and code --- .../ai/src/activities/evaluate/adapter.ts | 76 ++++++++++++++++++ packages/ai/src/activities/index.ts | 4 + packages/ai/src/index.ts | 8 +- packages/ai/tests/evaluate-error.test.ts | 80 +++++++++++++++++++ 4 files changed, 167 insertions(+), 1 deletion(-) create mode 100644 packages/ai/tests/evaluate-error.test.ts diff --git a/packages/ai/src/activities/evaluate/adapter.ts b/packages/ai/src/activities/evaluate/adapter.ts index 20972d2733..8ec0313164 100644 --- a/packages/ai/src/activities/evaluate/adapter.ts +++ b/packages/ai/src/activities/evaluate/adapter.ts @@ -137,6 +137,82 @@ export interface EvaluateAdapterResult { provider?: string } +/** `EvaluateError.code` when the provider declined to answer a question. */ +export const EVALUATE_REFUSAL_CODE = 'refusal' + +/** Fields for {@link EvaluateError}. */ +export interface EvaluateErrorInit { + /** Human-readable error text. */ + message: string + /** Stable code to branch on, for example `EVALUATE_REFUSAL_CODE`. */ + code: string + /** Tokens the provider reported for this call. */ + usage?: TokenUsage + /** Resolved model id from the provider. */ + model?: string + /** + * Wire answers the provider did return, before `decide()` maps them. A + * `choice()` answer has `choice`, and a `boolean()` answer is + * `{ type: 'noul', noul }`. A refused question has no key. + */ + answers?: Record +} + +/** + * Thrown by an evaluate adapter when the provider answered, but the call + * cannot become a result. It carries what the provider already returned, + * so callers can still read the billed usage and the answers that came back. + * + * The `name` is always `EvaluateError`. + * Another package can read this shape without an import of this class. + * Use {@link isEvaluateError} to narrow a caught value. `instanceof` fails + * when the app has more than one copy of `@tanstack/ai`. + */ +export class EvaluateError extends Error { + /** Stable code to branch on, for example `EVALUATE_REFUSAL_CODE`. */ + readonly code: string + /** Tokens the provider reported for this call. */ + readonly usage?: TokenUsage + /** Resolved model id from the provider. */ + readonly model?: string + /** + * Wire answers the provider did return, before `decide()` maps them. A + * `choice()` answer has `choice`, and a `boolean()` answer is + * `{ type: 'noul', noul }`. A refused question has no key. + */ + readonly answers?: Record + + constructor(init: EvaluateErrorInit) { + super(init.message) + this.name = 'EvaluateError' + this.code = init.code + this.usage = init.usage + this.model = init.model + this.answers = init.answers + } +} + +/** + * This function returns true when `value` has the evaluate error shape. + * + * The check reads `name` and `code`. + * A plain object with those fields passes. + * The check does not use `instanceof`, so it also matches an error from + * another copy of `@tanstack/ai`. + * + * @param value - A thrown value, or a plain object + * + * @example + * if (isEvaluateError(error) && error.code === EVALUATE_REFUSAL_CODE) { + * error.usage + * } + */ +export function isEvaluateError(value: unknown): value is EvaluateError { + if (typeof value !== 'object' || value === null) return false + if (!('name' in value) || value.name !== 'EvaluateError') return false + return 'code' in value && typeof value.code === 'string' +} + /** * Evaluate adapter interface with pre-resolved generics. * diff --git a/packages/ai/src/activities/index.ts b/packages/ai/src/activities/index.ts index bfb3a65d3a..d6e833f40a 100644 --- a/packages/ai/src/activities/index.ts +++ b/packages/ai/src/activities/index.ts @@ -138,6 +138,10 @@ export { export { BaseEvaluateAdapter, + EvaluateError, + EVALUATE_REFUSAL_CODE, + isEvaluateError, + type EvaluateErrorInit, type EvaluateAdapter, type EvaluateAdapterConfig, type AnyEvaluateAdapter, diff --git a/packages/ai/src/index.ts b/packages/ai/src/index.ts index 0da17c595d..2032fca29c 100644 --- a/packages/ai/src/index.ts +++ b/packages/ai/src/index.ts @@ -84,7 +84,13 @@ export type { export { BaseRerankAdapter } from './activities/rerank/adapter' // Evaluate adapter base + types -export { BaseEvaluateAdapter } from './activities/evaluate/adapter' +export { + BaseEvaluateAdapter, + EvaluateError, + EVALUATE_REFUSAL_CODE, + isEvaluateError, +} from './activities/evaluate/adapter' +export type { EvaluateErrorInit } from './activities/evaluate/adapter' export { defineAgent, diff --git a/packages/ai/tests/evaluate-error.test.ts b/packages/ai/tests/evaluate-error.test.ts new file mode 100644 index 0000000000..9cf33ed174 --- /dev/null +++ b/packages/ai/tests/evaluate-error.test.ts @@ -0,0 +1,80 @@ +import { describe, expect, it } from 'vitest' +import { EVALUATE_REFUSAL_CODE, EvaluateError, isEvaluateError } from '../src' +import * as adapters from '../src/activities/index' +import { toRunErrorPayload } from '../src/activities/error-payload' + +describe('EvaluateError', () => { + it('carries the code, usage, model and answers it was built with', () => { + const usage = { promptTokens: 40, completionTokens: 5, totalTokens: 45 } + const answers = { tone: { type: 'noul' as const, noul: 0.9 } } + const error = new EvaluateError({ + message: 'OpenAI declined to answer question "refund"', + code: EVALUATE_REFUSAL_CODE, + usage, + model: 'gpt-5-mini', + answers, + }) + + expect(error).toBeInstanceOf(Error) + expect(error.name).toBe('EvaluateError') + expect(error.message).toBe('OpenAI declined to answer question "refund"') + expect(error.code).toBe('refusal') + expect(error.usage).toBe(usage) + expect(error.model).toBe('gpt-5-mini') + expect(error.answers).toBe(answers) + expect(toRunErrorPayload(error).code).toBe('refusal') + }) +}) + +describe('isEvaluateError', () => { + it('accepts an EvaluateError after throw and catch', () => { + let caught: unknown + + try { + throw new EvaluateError({ + message: 'refused', + code: EVALUATE_REFUSAL_CODE, + }) + } catch (error) { + caught = error + } + + expect(isEvaluateError(caught)).toBe(true) + expect(caught).toBeInstanceOf(EvaluateError) + }) + + it('accepts an error from another copy of the class', () => { + class OtherCopyEvaluateError extends Error { + readonly code = EVALUATE_REFUSAL_CODE + constructor() { + super('refused') + this.name = 'EvaluateError' + } + } + const other = new OtherCopyEvaluateError() + + expect(other instanceof EvaluateError).toBe(false) + expect(isEvaluateError(other)).toBe(true) + }) + + it('accepts a plain object with the same shape', () => { + const plain = { name: 'EvaluateError', code: 'refusal' } + + expect(isEvaluateError(plain)).toBe(true) + }) + + it('rejects an ordinary Error, non-objects and a wrong code', () => { + expect(isEvaluateError(new Error('nope'))).toBe(false) + expect(isEvaluateError(null)).toBe(false) + expect(isEvaluateError(undefined)).toBe(false) + expect(isEvaluateError('EvaluateError')).toBe(false) + expect(isEvaluateError(42)).toBe(false) + expect(isEvaluateError({ name: 'EvaluateError' })).toBe(false) + expect(isEvaluateError({ name: 'EvaluateError', code: 7 })).toBe(false) + expect(isEvaluateError({ name: 'TypeError', code: 'refusal' })).toBe(false) + }) + + it('is exported from the adapters entry point', () => { + expect(adapters.isEvaluateError).toBe(isEvaluateError) + }) +}) From 44d8a1d67a5c187d9f793dc42483319eb6f1e3d5 Mon Sep 17 00:00:00 2001 From: Chris Trzesniewski Date: Tue, 13 Oct 2026 10:13:56 +0200 Subject: [PATCH 2/5] feat(ai): report EvaluateError usage in decide() before rethrowing - When the adapter throws an EvaluateError that carries usage, decide() reports it through the evaluate:usage event and onUsage, with the same payload as a successful call - Detect the error with instanceof EvaluateError - Then run onError or onAbort as before and rethrow the original error - onFinish does not run on failure, and the usage is reported once - Run onError or onAbort in a finally block, so a throwing onUsage hook cannot leave the call without a terminal hook - Document the behaviour in the decide() JSDoc - Add tests for the usage report and hook order, a throwing onUsage hook, and an EvaluateError without usage --- packages/ai/src/activities/evaluate/index.ts | 33 +++- packages/ai/tests/evaluate.test.ts | 153 ++++++++++++++++++- 2 files changed, 177 insertions(+), 9 deletions(-) diff --git a/packages/ai/src/activities/evaluate/index.ts b/packages/ai/src/activities/evaluate/index.ts index 1c44c42c7c..3c2b94c5b9 100644 --- a/packages/ai/src/activities/evaluate/index.ts +++ b/packages/ai/src/activities/evaluate/index.ts @@ -17,6 +17,7 @@ import { runGenerationStart, runGenerationUsage, } from '../middleware/run' +import { isEvaluateError } from './adapter' import type { InternalLogger } from '../../logger/internal-logger' import type { DebugOption } from '../../logger/types' import type { TokenUsage } from '../../types' @@ -454,6 +455,10 @@ export function boolean(options: { * The question key `meta` is reserved. Throws if `questions` is empty or uses * that key. * + * When the adapter throws an `EvaluateError` that carries `usage`, `decide()` + * reports that usage through `onUsage` and the `evaluate:usage` event, then + * runs `onError` and rethrows the same error. + * * @param options.adapter Evaluate adapter created with a model. * @param options.state Shared state every question judges. * @param options.questions Questions built with `choice`, `score`, `boolean`. @@ -602,15 +607,27 @@ export async function decide< }) } catch (error) { const duration = Date.now() - startTime - if (isAbortError(error, abortSignal)) { - await runGenerationAbort(middleware, mwCtx, { - reason: error instanceof Error ? error.message : undefined, - duration, - }) - } else { - await runGenerationError(middleware, mwCtx, { error, duration }) + try { + if (isEvaluateError(error) && error.usage) { + aiEventClient.emit('evaluate:usage', { + requestId, + model: error.model ?? model, + usage: error.usage, + timestamp: Date.now(), + }) + await runGenerationUsage(middleware, mwCtx, error.usage) + } + } finally { + if (isAbortError(error, abortSignal)) { + await runGenerationAbort(middleware, mwCtx, { + reason: error instanceof Error ? error.message : undefined, + duration, + }) + } else { + await runGenerationError(middleware, mwCtx, { error, duration }) + } + logger.errors('evaluate activity failed', { error, source: 'evaluate' }) } - logger.errors('evaluate activity failed', { error, source: 'evaluate' }) throw error } } diff --git a/packages/ai/tests/evaluate.test.ts b/packages/ai/tests/evaluate.test.ts index aa7145b1c0..587dabdff5 100644 --- a/packages/ai/tests/evaluate.test.ts +++ b/packages/ai/tests/evaluate.test.ts @@ -1,6 +1,13 @@ import { describe, expect, it } from 'vitest' import { aiEventClient } from '@tanstack/ai-event-client' -import { boolean, choice, decide, score } from '../src/index' +import { + EVALUATE_REFUSAL_CODE, + EvaluateError, + boolean, + choice, + decide, + score, +} from '../src/index' import type { EvaluateAdapter } from '../src/activities/evaluate/adapter' import type { EvaluateAdapterResult, @@ -509,6 +516,150 @@ describe('decide()', () => { expect(events.finish).toHaveLength(0) }) + it('reports the usage of an EvaluateError once, then fires onError and rethrows it unchanged', async () => { + const usage = { promptTokens: 40, completionTokens: 5, totalTokens: 45 } + const refusal = new EvaluateError({ + message: 'declined to answer question "refund"', + code: EVALUATE_REFUSAL_CODE, + usage, + model: 'provider-model', + }) + const order: Array = [] + const { middleware, events } = recordingMiddleware() + const usageEvents: Array<{ model: string; usage: TokenUsage }> = [] + const unsubscribe = aiEventClient.on( + 'evaluate:usage', + (event) => { + usageEvents.push({ + model: event.payload.model, + usage: event.payload.usage, + }) + }, + { withEventTarget: true }, + ) + const adapter = mockEvaluateAdapter(async () => { + throw refusal + }) + + try { + await expect( + decide({ + adapter, + state: 'ticket', + questions: mixedQuestions, + middleware: [ + middleware, + { + onUsage: () => { + order.push('usage') + }, + onError: () => { + order.push('error') + }, + }, + ], + debug: false, + }), + ).rejects.toBe(refusal) + } finally { + unsubscribe() + } + + expect(events.usage).toEqual([usage]) + expect(usageEvents).toEqual([{ model: 'provider-model', usage }]) + expect(order).toEqual(['usage', 'error']) + expect(events.error).toHaveLength(1) + expect(events.error[0]!.error).toBe(refusal) + expect(events.abort).toHaveLength(0) + expect(events.finish).toHaveLength(0) + }) + + it('reports the usage of an EvaluateError from another copy of the package', async () => { + const usage = { promptTokens: 40, completionTokens: 5, totalTokens: 45 } + class OtherCopyEvaluateError extends Error { + readonly code = EVALUATE_REFUSAL_CODE + readonly usage = usage + constructor() { + super('declined to answer question "refund"') + this.name = 'EvaluateError' + } + } + const refusal = new OtherCopyEvaluateError() + const { middleware, events } = recordingMiddleware() + const adapter = mockEvaluateAdapter(async () => { + throw refusal + }) + + expect(refusal instanceof EvaluateError).toBe(false) + await expect( + decide({ + adapter, + state: 'ticket', + questions: mixedQuestions, + middleware: [middleware], + debug: false, + }), + ).rejects.toBe(refusal) + + expect(events.usage).toEqual([usage]) + expect(events.error).toHaveLength(1) + expect(events.error[0]!.error).toBe(refusal) + }) + + it('still fires onError when onUsage throws while reporting an EvaluateError', async () => { + const refusal = new EvaluateError({ + message: 'declined to answer question "refund"', + code: EVALUATE_REFUSAL_CODE, + usage: { promptTokens: 40, completionTokens: 5, totalTokens: 45 }, + }) + const { middleware, events } = recordingMiddleware() + const adapter = mockEvaluateAdapter(async () => { + throw refusal + }) + + await expect( + decide({ + adapter, + state: 'ticket', + questions: mixedQuestions, + middleware: [ + { + onUsage: () => { + throw new Error('usage hook boom') + }, + }, + middleware, + ], + debug: false, + }), + ).rejects.toThrow('usage hook boom') + + expect(events.error).toHaveLength(1) + expect(events.error[0]!.error).toBe(refusal) + expect(events.abort).toHaveLength(0) + expect(events.finish).toHaveLength(0) + }) + + it('reports no usage for an EvaluateError without usage', async () => { + const { middleware, events } = recordingMiddleware() + const adapter = mockEvaluateAdapter(async () => { + throw new EvaluateError({ message: 'no usage', code: 'other' }) + }) + + await expect( + decide({ + adapter, + state: 'ticket', + questions: mixedQuestions, + middleware: [middleware], + debug: false, + }), + ).rejects.toThrow('no usage') + + expect(events.usage).toHaveLength(0) + expect(events.error).toHaveLength(1) + }) + it('fires onAbort (not onError) when the request is cancelled', async () => { const { middleware, events } = recordingMiddleware() const controller = new AbortController() From 70dab01bb6f25996f1b8d73698e6efc01c4ae5bc Mon Sep 17 00:00:00 2001 From: Chris Trzesniewski Date: Tue, 13 Oct 2026 09:47:05 +0200 Subject: [PATCH 3/5] feat(ai-openai): throw EvaluateError on evaluate refusals - A refused question now throws EvaluateError with code EVALUATE_REFUSAL_CODE instead of a plain Error - The error carries the usage, the model and the wire answers that did come back, so the refused question is the missing key - The message text is the same as before, and names the first refused question - Map all answers before throwing, so a refusal no longer drops the other answers and the usage - Extend the refusal unit test to check the class, code, message, usage, model and answers - Add a test for two refusals in one call - Add a minor changeset for @tanstack/ai and @tanstack/ai-openai covering the new error, its exports, the refusal behaviour and the usage reporting in decide() --- .changeset/evaluate-refusal-error.md | 30 +++++++++++++ packages/ai-openai/src/adapters/evaluate.ts | 43 +++++++++++++------ .../ai-openai/tests/evaluate-adapter.test.ts | 42 ++++++++++++++++-- 3 files changed, 99 insertions(+), 16 deletions(-) create mode 100644 .changeset/evaluate-refusal-error.md diff --git a/.changeset/evaluate-refusal-error.md b/.changeset/evaluate-refusal-error.md new file mode 100644 index 0000000000..b9d6187486 --- /dev/null +++ b/.changeset/evaluate-refusal-error.md @@ -0,0 +1,30 @@ +--- +'@tanstack/ai': minor +'@tanstack/ai-openai': minor +--- + +Add `EvaluateError`, a typed error an evaluate adapter throws when the provider answered but the call cannot become a result. `@tanstack/ai` and `@tanstack/ai/adapters` export the class, the `isEvaluateError()` type guard, the `EvaluateErrorInit` type and `EVALUATE_REFUSAL_CODE`. The error keeps what the provider sent back: + +- `code`: a stable code to branch on. A refused question uses `EVALUATE_REFUSAL_CODE` (`'refusal'`). +- `usage`: the tokens the call used. +- `model`: the model that answered. +- `answers`: the raw wire answers that came back, before `decide()` maps them. A `choice()` answer has `choice` (for example `{ type: 'choice', choice: 'billing', ... }`), and a `boolean()` answer is `{ type: 'noul', noul: 0.9 }`, not `{ value }`. The refused question has no key. + +When OpenAI refuses to answer a question, `decide()` now throws an `EvaluateError` with `code: 'refusal'` instead of a plain `Error`. If more than one question is refused, the message names the first one. + +Use `isEvaluateError(error)` to narrow a caught error. It checks `name` and `code`, not `instanceof`, so it still works when the app installs more than one copy of `@tanstack/ai`. + +When an adapter throws an `EvaluateError` that carries `usage`, `decide()` reports that usage through the `evaluate:usage` event and the middleware `onUsage` hook, then runs `onError` and rethrows the same error. `onFinish` does not run. + +```ts +try { + await decide({ adapter, state, questions }) +} catch (error) { + if (isEvaluateError(error) && error.code === EVALUATE_REFUSAL_CODE) { + console.log(error.usage, error.answers) + } + throw error +} +``` + +The message text does not change. The error is still an `Error`, but its `name` is now `'EvaluateError'`, so logs and the OpenTelemetry `error.type` show that name instead of `'Error'`. diff --git a/packages/ai-openai/src/adapters/evaluate.ts b/packages/ai-openai/src/adapters/evaluate.ts index 2e70e9ea46..1a593556a6 100644 --- a/packages/ai-openai/src/adapters/evaluate.ts +++ b/packages/ai-openai/src/adapters/evaluate.ts @@ -1,6 +1,10 @@ import OpenAI from 'openai' import { buildBaseUsage } from '@tanstack/ai' -import { BaseEvaluateAdapter } from '@tanstack/ai/adapters' +import { + BaseEvaluateAdapter, + EVALUATE_REFUSAL_CODE, + EvaluateError, +} from '@tanstack/ai/adapters' import { toRunErrorPayload } from '@tanstack/ai/adapter-internals' import { getOpenAIApiKeyFromEnv } from '../utils/client' import type { @@ -71,7 +75,7 @@ function toDecisionQuestion( } } -function toWireAnswer(answer: DecisionAnswer, name: string): WireAnswer { +function toWireAnswer(answer: DecisionAnswer): WireAnswer | undefined { switch (answer.type) { case 'predicate': return { type: 'noul', noul: answer.probability } @@ -97,7 +101,7 @@ function toWireAnswer(answer: DecisionAnswer, name: string): WireAnswer { confidence: answer.confidence, } case 'refusal': - throw new Error(`OpenAI declined to answer question "${name}"`) + return undefined } } @@ -144,22 +148,35 @@ export class OpenAIEvaluateAdapter< // Answers come back in question order. `name` is null only for // unnamed questions, and we always send names. Object.fromEntries keeps // a question named "__proto__" as an own key. + const refused: Array = [] const answers: Record = Object.fromEntries( - decision.answers.map((answer, index) => { + decision.answers.flatMap((answer, index) => { const name = answer.name ?? questionKeys[index] ?? String(index) - return [name, toWireAnswer(answer, name)] + const wire = toWireAnswer(answer) + if (!wire) { + refused.push(name) + return [] + } + return [[name, wire]] }), ) + const usage = buildBaseUsage({ + promptTokens: decision.usage.input_tokens, + completionTokens: decision.usage.output_tokens, + totalTokens: decision.usage.total_tokens, + }) - return { - model: decision.model, - answers, - usage: buildBaseUsage({ - promptTokens: decision.usage.input_tokens, - completionTokens: decision.usage.output_tokens, - totalTokens: decision.usage.total_tokens, - }), + if (refused.length > 0) { + throw new EvaluateError({ + message: `OpenAI declined to answer question "${refused[0]}"`, + code: EVALUATE_REFUSAL_CODE, + usage, + model: decision.model, + answers, + }) } + + return { model: decision.model, answers, usage } } catch (error) { // toRunErrorPayload keeps request headers (and the API key) out of logs. logger.errors(`${this.name}.evaluate fatal`, { diff --git a/packages/ai-openai/tests/evaluate-adapter.test.ts b/packages/ai-openai/tests/evaluate-adapter.test.ts index f541b5ed0e..c8795d790b 100644 --- a/packages/ai-openai/tests/evaluate-adapter.test.ts +++ b/packages/ai-openai/tests/evaluate-adapter.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it, vi } from 'vitest' import { resolveDebugOption } from '@tanstack/ai/adapter-internals' +import { EVALUATE_REFUSAL_CODE, EvaluateError } from '@tanstack/ai/adapters' import { OpenAIEvaluateAdapter } from '../src/adapters/evaluate' import type { EvaluateOptions, WireQuestion } from '@tanstack/ai/adapters' import type { Decision } from 'openai/resources/decisions' @@ -193,9 +194,44 @@ describe('OpenAI evaluate adapter', () => { expect(Object.keys(result.answers)).toEqual(['__proto__']) }) - it('throws when a question is refused', async () => { - const { adapter } = setup(decision([{ type: 'refusal', name: 'refund' }])) - await expect(run(adapter)).rejects.toThrow('"refund"') + it('throws an EvaluateError with usage and the other answers when a question is refused', async () => { + const { adapter } = setup( + decision([ + ...allAnswers.answers.slice(0, 2), + { type: 'refusal', name: 'refund' }, + ]), + ) + const error = await run(adapter).catch((e: unknown) => e) + expect(error).toBeInstanceOf(EvaluateError) + if (!(error instanceof EvaluateError)) return + expect(error.code).toBe(EVALUATE_REFUSAL_CODE) + expect(error.message).toBe('OpenAI declined to answer question "refund"') + expect(error.model).toBe('gpt-6-luna') + expect(error.usage).toEqual({ + promptTokens: 40, + completionTokens: 0, + totalTokens: 40, + }) + expect(Object.keys(error.answers ?? {})).toEqual(['queue', 'urgency']) + expect(error.answers?.queue).toMatchObject({ + type: 'choice', + choice: 'billing', + }) + }) + + it('names the first refused question and leaves out every refused key', async () => { + const { adapter } = setup( + decision([ + { type: 'refusal', name: 'queue' }, + allAnswers.answers[1]!, + { type: 'refusal', name: 'refund' }, + ]), + ) + const error = await run(adapter).catch((e: unknown) => e) + expect(error).toBeInstanceOf(EvaluateError) + if (!(error instanceof EvaluateError)) return + expect(error.message).toBe('OpenAI declined to answer question "queue"') + expect(Object.keys(error.answers ?? {})).toEqual(['urgency']) }) it('rethrows SDK errors', async () => { From 01a344d8b297046fe7562ba97ac9ac4ed2f2c0a7 Mon Sep 17 00:00:00 2001 From: Chris Trzesniewski Date: Tue, 13 Oct 2026 09:47:41 +0200 Subject: [PATCH 4/5] docs(evaluate): document typed refusal errors - Add a Refused questions section to the evaluate guide with an instanceof EvaluateError sample that rethrows other errors - List the usage, model and answers fields the error keeps, and show that answers are raw wire answers - Say that decide() reports the usage through onUsage, then runs onError and rethrows - Point the OpenAI adapter note at the new section - Bump updatedAt for both pages in docs/config.json --- docs/adapters/openai.md | 2 +- docs/config.json | 2 +- docs/evaluate/evaluate.md | 65 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 67 insertions(+), 2 deletions(-) diff --git a/docs/adapters/openai.md b/docs/adapters/openai.md index 2812e181cd..b087504871 100644 --- a/docs/adapters/openai.md +++ b/docs/adapters/openai.md @@ -758,7 +758,7 @@ Good to know: - `gpt-6-luna` is the only Decisions model. - An object `state` goes to OpenAI as JSON text. -- If OpenAI refuses to answer a question, `decide()` throws an error with the name of that question. +- If OpenAI refuses to answer a question, `decide()` throws an `EvaluateError` with `code: EVALUATE_REFUSAL_CODE`. The message names the question. The error also has the usage and the raw answers that came back, for example `{ type: 'noul', noul: 0.9 }` for a `boolean()` question. Narrow it with `isEvaluateError(error)`. See [Refused questions](../evaluate/evaluate#refused-questions). - `boolean()` criteria work. The adapter adds the true and false meanings to the instructions. See the [Evaluate guide](../evaluate/evaluate) for question helpers, the result shape, abort, and middleware. diff --git a/docs/config.json b/docs/config.json index aee4f36442..47f739174c 100644 --- a/docs/config.json +++ b/docs/config.json @@ -1209,7 +1209,7 @@ "label": "OpenAI", "to": "adapters/openai", "addedAt": "2026-04-15", - "updatedAt": "2026-10-12" + "updatedAt": "2026-10-13" }, { "label": "Anthropic", diff --git a/docs/evaluate/evaluate.md b/docs/evaluate/evaluate.md index 3a27b976d2..551a15dedb 100644 --- a/docs/evaluate/evaluate.md +++ b/docs/evaluate/evaluate.md @@ -448,6 +448,71 @@ For example, an answer is missing, has the wrong type, or has no probability for its value. `onUsage` and the `evaluate:usage` event still get the token usage of that call, before `onError` runs. `onFinish` does not run. +### Refused questions + +OpenAI can refuse to answer a question. +Then `decide()` throws an `EvaluateError` with `code: EVALUATE_REFUSAL_CODE`. +The error keeps what the provider sent back: + +- `usage`: the tokens the call used. +- `model`: the model that answered. +- `answers`: the raw answers that came back, before `decide()` maps them. The refused question has no key. + +The raw answers do not have `value`. +A `choice()` answer has `choice`, for example `{ type: 'choice', choice: 'billing', ... }`. +A `boolean()` answer is `{ type: 'noul', noul: 0.9 }`, where `noul` is the chance of true. + +Use `isEvaluateError(error)` to narrow a caught error. +It checks `name` and `code`, so it works even when the app has two copies of `@tanstack/ai`. +`instanceof EvaluateError` does not. + +```typescript +import { + decide, + boolean, + choice, + isEvaluateError, + EVALUATE_REFUSAL_CODE, +} from '@tanstack/ai' +import { openaiDecider } from '@tanstack/ai-openai' + +try { + const result = await decide({ + adapter: openaiDecider('gpt-6-luna'), + state: 'Please refund the extra payment.', + questions: { + queue: choice({ + instructions: 'Which team should handle this ticket?', + options: { + billing: 'Payments, invoices, refunds', + tech: 'Bugs, outages, integrations', + }, + }), + refund: boolean({ + instructions: 'Is the customer asking for a refund?', + }), + }, + }) + console.log(result.refund.value) +} catch (error) { + if (isEvaluateError(error) && error.code === EVALUATE_REFUSAL_CODE) { + console.error(error.message) // OpenAI declined to answer question "refund" + console.log(error.usage) + console.log(error.answers?.queue) // { type: 'choice', choice: 'billing', ... } + } + throw error +} +``` + +The provider still bills a refused call. +So before `decide()` rethrows an `EvaluateError` that has `usage`, it reports that usage: + +1. Middleware `onUsage` runs with the usage. +2. Middleware `onError` runs with the error. +3. `decide()` rethrows the same error. + +`onFinish` does not run. See [Observability](#observability) for middleware. + ## Runnable Example `examples/react/evaluate` is a small TanStack Start app that runs this page. From cbf9e8d6eeeb7242b1a42296bab0d8e5f2cc5b2b Mon Sep 17 00:00:00 2001 From: Chris Trzesniewski Date: Tue, 13 Oct 2026 15:21:45 +0200 Subject: [PATCH 5/5] test(e2e): cover decide() throwing EvaluateError on an OpenAI refusal - Mount /evaluate-refusal in aimock to answer /v1/decisions with a refusal answer and billed usage - Add /api/evaluate-refusal, which runs decide() with the OpenAI decider against that mount and returns the rejection as name, code, message and usage - Record the usage seen by onUsage middleware - Add evaluate-refusal.spec.ts asserting an EvaluateError with code refusal and the usage on the error and in onUsage --- testing/e2e/global-setup.ts | 40 +++++++++++++ testing/e2e/src/routeTree.gen.ts | 21 +++++++ .../e2e/src/routes/api.evaluate-refusal.ts | 58 +++++++++++++++++++ testing/e2e/tests/evaluate-refusal.spec.ts | 36 ++++++++++++ 4 files changed, 155 insertions(+) create mode 100644 testing/e2e/src/routes/api.evaluate-refusal.ts create mode 100644 testing/e2e/tests/evaluate-refusal.spec.ts diff --git a/testing/e2e/global-setup.ts b/testing/e2e/global-setup.ts index 4f06314e99..3762c7e370 100644 --- a/testing/e2e/global-setup.ts +++ b/testing/e2e/global-setup.ts @@ -186,6 +186,10 @@ export default async function globalSetup() { // asserts `decide()` rejects and still reports that usage. mock.mount('/evaluate-usage-on-failure', evaluateUsageOnFailureMount()) + // aimock has no OpenAI Decisions API route, so this mount answers + // `/v1/decisions` with a refusal plus usage for the evaluate-refusal spec. + mock.mount('/evaluate-refusal', evaluateRefusalMount()) + // BytePlus. Ark (chat, Seedream image, Seedance video) serves its whole data // plane under /api/v3, and Seed Speech (TTS/ASR) mounts its own endpoints at // the same prefix on a different host — which collapses onto one prefix here @@ -2304,6 +2308,42 @@ function evaluateUsageOnFailureMount(): Mountable { } } +/** + * Mounts the OpenAI Decisions `/v1/decisions` response with a `refusal` answer + * and billed usage. The OpenAI evaluate adapter turns it into an + * `EvaluateError` with code `refusal`. + */ +function evaluateRefusalMount(): Mountable { + return { + async handleRequest( + req: http.IncomingMessage, + res: http.ServerResponse, + pathname: string, + ): Promise { + if (req.method !== 'POST' || pathname !== '/v1/decisions') { + return false + } + await drainBody(req) + res.statusCode = 200 + res.setHeader('Content-Type', 'application/json') + res.end( + JSON.stringify({ + model: 'gpt-6-luna', + answers: [{ type: 'refusal', name: 'refund' }], + usage: { + input_tokens: 40, + input_tokens_details: { cached_tokens: 0, cache_write_tokens: 0 }, + output_tokens: 5, + output_tokens_details: { reasoning_tokens: 0 }, + total_tokens: 45, + }, + }), + ) + return true + }, + } +} + function buildToolPlusServerToolEvents(): Array> { const messageId = 'msg_bug_604' const model = 'claude-sonnet-4-5' diff --git a/testing/e2e/src/routeTree.gen.ts b/testing/e2e/src/routeTree.gen.ts index 780d06b4dc..ddb8bef445 100644 --- a/testing/e2e/src/routeTree.gen.ts +++ b/testing/e2e/src/routeTree.gen.ts @@ -84,6 +84,7 @@ import { Route as ApiDevtoolsMemoryRouteImport } from './routes/api.devtools-mem import { Route as ApiDurableDeliveryRouteImport } from './routes/api.durable-delivery' import { Route as ApiDurableTakeoverRouteImport } from './routes/api.durable-takeover' import { Route as ApiEmbeddingRouteImport } from './routes/api.embedding' +import { Route as ApiEvaluateRefusalRouteImport } from './routes/api.evaluate-refusal' import { Route as ApiEvaluateUsageOnFailureRouteImport } from './routes/api.evaluate-usage-on-failure' import { Route as ApiFakeTextRouteImport } from './routes/api.fake-text' import { Route as ApiFileSourceWireRouteImport } from './routes/api.file-source-wire' @@ -575,6 +576,11 @@ const ApiEmbeddingRoute = ApiEmbeddingRouteImport.update({ path: '/api/embedding', getParentRoute: () => rootRouteImport, } as any) +const ApiEvaluateRefusalRoute = ApiEvaluateRefusalRouteImport.update({ + id: '/api/evaluate-refusal', + path: '/api/evaluate-refusal', + getParentRoute: () => rootRouteImport, +} as any) const ApiEvaluateUsageOnFailureRoute = ApiEvaluateUsageOnFailureRouteImport.update({ id: '/api/evaluate-usage-on-failure', @@ -1161,6 +1167,7 @@ export interface FileRoutesByFullPath { '/api/durable-delivery': typeof ApiDurableDeliveryRoute '/api/durable-takeover': typeof ApiDurableTakeoverRoute '/api/embedding': typeof ApiEmbeddingRoute + '/api/evaluate-refusal': typeof ApiEvaluateRefusalRoute '/api/evaluate-usage-on-failure': typeof ApiEvaluateUsageOnFailureRoute '/api/fake-text': typeof ApiFakeTextRoute '/api/file-source-wire': typeof ApiFileSourceWireRoute @@ -1335,6 +1342,7 @@ export interface FileRoutesByTo { '/api/durable-delivery': typeof ApiDurableDeliveryRoute '/api/durable-takeover': typeof ApiDurableTakeoverRoute '/api/embedding': typeof ApiEmbeddingRoute + '/api/evaluate-refusal': typeof ApiEvaluateRefusalRoute '/api/evaluate-usage-on-failure': typeof ApiEvaluateUsageOnFailureRoute '/api/fake-text': typeof ApiFakeTextRoute '/api/file-source-wire': typeof ApiFileSourceWireRoute @@ -1510,6 +1518,7 @@ export interface FileRoutesById { '/api/durable-delivery': typeof ApiDurableDeliveryRoute '/api/durable-takeover': typeof ApiDurableTakeoverRoute '/api/embedding': typeof ApiEmbeddingRoute + '/api/evaluate-refusal': typeof ApiEvaluateRefusalRoute '/api/evaluate-usage-on-failure': typeof ApiEvaluateUsageOnFailureRoute '/api/fake-text': typeof ApiFakeTextRoute '/api/file-source-wire': typeof ApiFileSourceWireRoute @@ -1686,6 +1695,7 @@ export interface FileRouteTypes { | '/api/durable-delivery' | '/api/durable-takeover' | '/api/embedding' + | '/api/evaluate-refusal' | '/api/evaluate-usage-on-failure' | '/api/fake-text' | '/api/file-source-wire' @@ -1860,6 +1870,7 @@ export interface FileRouteTypes { | '/api/durable-delivery' | '/api/durable-takeover' | '/api/embedding' + | '/api/evaluate-refusal' | '/api/evaluate-usage-on-failure' | '/api/fake-text' | '/api/file-source-wire' @@ -2034,6 +2045,7 @@ export interface FileRouteTypes { | '/api/durable-delivery' | '/api/durable-takeover' | '/api/embedding' + | '/api/evaluate-refusal' | '/api/evaluate-usage-on-failure' | '/api/fake-text' | '/api/file-source-wire' @@ -2209,6 +2221,7 @@ export interface RootRouteChildren { ApiDurableDeliveryRoute: typeof ApiDurableDeliveryRoute ApiDurableTakeoverRoute: typeof ApiDurableTakeoverRoute ApiEmbeddingRoute: typeof ApiEmbeddingRoute + ApiEvaluateRefusalRoute: typeof ApiEvaluateRefusalRoute ApiEvaluateUsageOnFailureRoute: typeof ApiEvaluateUsageOnFailureRoute ApiFakeTextRoute: typeof ApiFakeTextRoute ApiFileSourceWireRoute: typeof ApiFileSourceWireRoute @@ -2831,6 +2844,13 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof ApiEmbeddingRouteImport parentRoute: typeof rootRouteImport } + '/api/evaluate-refusal': { + id: '/api/evaluate-refusal' + path: '/api/evaluate-refusal' + fullPath: '/api/evaluate-refusal' + preLoaderRoute: typeof ApiEvaluateRefusalRouteImport + parentRoute: typeof rootRouteImport + } '/api/evaluate-usage-on-failure': { id: '/api/evaluate-usage-on-failure' path: '/api/evaluate-usage-on-failure' @@ -3651,6 +3671,7 @@ const rootRouteChildren: RootRouteChildren = { ApiDurableDeliveryRoute: ApiDurableDeliveryRoute, ApiDurableTakeoverRoute: ApiDurableTakeoverRoute, ApiEmbeddingRoute: ApiEmbeddingRoute, + ApiEvaluateRefusalRoute: ApiEvaluateRefusalRoute, ApiEvaluateUsageOnFailureRoute: ApiEvaluateUsageOnFailureRoute, ApiFakeTextRoute: ApiFakeTextRoute, ApiFileSourceWireRoute: ApiFileSourceWireRoute, diff --git a/testing/e2e/src/routes/api.evaluate-refusal.ts b/testing/e2e/src/routes/api.evaluate-refusal.ts new file mode 100644 index 0000000000..e99a156173 --- /dev/null +++ b/testing/e2e/src/routes/api.evaluate-refusal.ts @@ -0,0 +1,58 @@ +import { createFileRoute } from '@tanstack/react-router' +import { choice, decide } from '@tanstack/ai' +import { createOpenaiDecider } from '@tanstack/ai-openai' +import type { TokenUsage } from '@tanstack/ai' + +const LLMOCK_DEFAULT_BASE = process.env.LLMOCK_URL || 'http://127.0.0.1:4010' +const DUMMY_KEY = 'sk-e2e-test-dummy-key' + +/** + * Drives `decide()` with the OpenAI decider against the `/evaluate-refusal` + * aimock mount, whose Decisions response refuses the only question and bills + * usage. Returns the rejection as plain fields plus the usage seen by `onUsage`. + */ +export const Route = createFileRoute('/api/evaluate-refusal')({ + server: { + handlers: { + POST: async () => { + const adapter = createOpenaiDecider('gpt-6-luna', DUMMY_KEY, { + baseURL: `${LLMOCK_DEFAULT_BASE}/evaluate-refusal/v1`, + }) + const usageReports: Array = [] + + try { + const result = await decide({ + adapter, + state: 'Customer was double charged and wants the money back.', + questions: { + refund: choice({ + instructions: 'Should this ticket get a refund?', + options: { yes: null, no: null }, + }), + }, + middleware: [ + { + name: 'usage-observer', + onUsage: (_ctx, usage) => { + usageReports.push(usage) + }, + }, + ], + }) + return Response.json({ result, usageReports }) + } catch (error) { + const { name, code, message, usage } = error as { + name?: unknown + code?: unknown + message?: unknown + usage?: unknown + } + return Response.json({ + error: { name, code, message, usage }, + usageReports, + }) + } + }, + }, + }, +}) diff --git a/testing/e2e/tests/evaluate-refusal.spec.ts b/testing/e2e/tests/evaluate-refusal.spec.ts new file mode 100644 index 0000000000..4aebc2d773 --- /dev/null +++ b/testing/e2e/tests/evaluate-refusal.spec.ts @@ -0,0 +1,36 @@ +import { test, expect } from './fixtures' + +/** + * The OpenAI Decisions API can refuse a question and still bill the request. + * `decide()` must reject with an `EvaluateError` whose code is `refusal` and + * whose usage reaches `onUsage` first. + */ +test.describe('openai evaluate refusal', () => { + test('rejects with a refusal EvaluateError and reports its usage', async ({ + request, + }) => { + const res = await request.post('/api/evaluate-refusal') + expect(res.ok()).toBe(true) + + const body = (await res.json()) as { + result?: unknown + error?: { + name?: string + code?: string + message?: string + usage?: unknown + } + usageReports: Array + } + const usage = { promptTokens: 40, completionTokens: 5, totalTokens: 45 } + + expect(body.result).toBeUndefined() + expect(body.error).toEqual({ + name: 'EvaluateError', + code: 'refusal', + message: 'OpenAI declined to answer question "refund"', + usage, + }) + expect(body.usageReports).toEqual([usage]) + }) +})