feat(adapters): set the HTTP status on a failed evaluate request

.changeset/evaluate-http-status.md

@@ -0,0 +1,14 @@
+---
+'@tanstack/ai-typesafe': minor
+'@tanstack/ai-ollaya': minor
+'@tanstack/ai-openrouter': minor
+'@tanstack/ai-vercel-gateway': minor
+---
+
+The TypeSafe, Ollaya, OpenRouter and Vercel Gateway evaluate adapters now set `status` on the error they throw for an error response. `status` is the HTTP status code, so you can tell a rate limit (429) from a bad key (401) without parsing the message. The message does not change.
+
+```ts
+if (error instanceof Error && 'status' in error && error.status === 429) {
+  // wait, then call decide() again
+}
+```

docs/adapters/ollaya.md

@@ -151,6 +151,29 @@ const adapter = ollayaDecider("laya:latest", {
 
 `baseUrl` and `headers` are aliases of `baseURL` and `defaultHeaders`. If you set both forms, `baseURL` and `defaultHeaders` win. A blank `baseURL` keeps the default host.
 
+### Handle a failed request
+
+A wrong `apiKey` needs a fix, not a retry. When the server sends an error response, the thrown error has the HTTP status code in `status`:
+
+```typescript
+import { decide, boolean } from "@tanstack/ai";
+import { ollayaDecider } from "@tanstack/ai-ollaya";
+
+try {
+  await decide({
+    adapter: ollayaDecider("laya:latest"),
+    state: { body: "Please refund the extra payment." },
+    questions: {
+      refund: boolean({ instructions: "Is the customer asking for a refund?" }),
+    },
+  });
+} catch (error) {
+  if (error instanceof Error && "status" in error && error.status === 401) {
+    // The server rejected the apiKey. Check it before you call decide() again.
+  }
+}
+```
+
 ### API reference
 
 The adapter sends `POST /v1/systemone` to the base URL with `fetch`. There is no Ollaya SDK.

docs/adapters/openrouter.md

@@ -294,6 +294,27 @@ console.log(result.meta.usage);
 
 Known ids: `~typesafe/jev-latest`, `typesafe/jev-1.13`, `typesafe/jev-1.13.0`. See the [Evaluate guide](../evaluate/evaluate).
 
+When OpenRouter sends an error response, the thrown error has the HTTP status code in `status`. A rate limit (429) needs a retry, but a bad key (401) does not:
+
+```typescript
+import { decide, boolean } from "@tanstack/ai";
+import { openRouterDecider } from "@tanstack/ai-openrouter";
+
+try {
+  await decide({
+    adapter: openRouterDecider("~typesafe/jev-latest"),
+    state: { body: "Please refund the extra payment." },
+    questions: {
+      refund: boolean({ instructions: "Is the customer asking for a refund?" }),
+    },
+  });
+} catch (error) {
+  if (error instanceof Error && "status" in error && error.status === 429) {
+    // Rate limited. Wait, then call decide() again.
+  }
+}
+```
+
 ## Summarization
 
 ```typescript

docs/adapters/typesafe.md

@@ -134,6 +134,29 @@ const adapter = createTypesafeDecider("jev-latest", process.env.TYPESAFE_API_KEY
 
 `baseUrl` and `headers` are aliases of the same two options. If you set both forms, `baseURL` and `defaultHeaders` win.
 
+### Handle a failed request
+
+A rate limit needs a retry, but a bad API key does not. When TypeSafe sends an error response, the thrown error has the HTTP status code in `status`:
+
+```typescript
+import { decide, boolean } from "@tanstack/ai";
+import { typesafeDecider } from "@tanstack/ai-typesafe";
+
+try {
+  await decide({
+    adapter: typesafeDecider("jev-latest"),
+    state: { body: "Please refund the extra payment." },
+    questions: {
+      refund: boolean({ instructions: "Is the customer asking for a refund?" }),
+    },
+  });
+} catch (error) {
+  if (error instanceof Error && "status" in error && error.status === 429) {
+    // Rate limited. Wait, then call decide() again.
+  }
+}
+```
+
 ### Bring your own key
 
 `typesafeByok` from `@tanstack/ai-typesafe/byok` declares TypeSafe for the BYOK store, with `TYPESAFE_API_KEY` as its env var. See [Bring Your Own Key](../advanced/byok).

docs/adapters/vercel-gateway.md

@@ -270,6 +270,27 @@ console.log(result.refund.value); // true
 
 See the [Evaluate guide](../evaluate/evaluate) for the question helpers, the result shape, abort, and middleware.
 
+When the Gateway sends an error response, the thrown error has the HTTP status code in `status`. A rate limit (429) needs a retry, but a bad key (401) does not:
+
+```typescript
+import { decide, boolean } from "@tanstack/ai";
+import { vercelGatewayDecider } from "@tanstack/ai-vercel-gateway";
+
+try {
+  await decide({
+    adapter: vercelGatewayDecider("typesafe-ai/jev"),
+    state: { body: "Please refund the extra payment." },
+    questions: {
+      refund: boolean({ instructions: "Is the customer asking for a refund?" }),
+    },
+  });
+} catch (error) {
+  if (error instanceof Error && "status" in error && error.status === 429) {
+    // Rate limited. Wait, then call decide() again.
+  }
+}
+```
+
 ## Not in this package
 
 This package does not do video, speech, transcription, or rerank. Use a dedicated adapter for these.

packages/ai-ollaya/src/adapters/evaluate.ts

@@ -61,6 +61,9 @@ export class OllayaEvaluateAdapter<
   /**
    * POST `{ model, state, questions }` to Ollaya and return wire answers plus
    * mapped token usage. Does not invent unified `.value` fields.
+   *
+   * @throws An `Error` with a numeric `status` (the HTTP status code) when
+   * Ollaya answers with a non-OK response.
    */
   async evaluate(options: EvaluateOptions) {
     const { model, state, questions, abortSignal, logger } = options
@@ -95,10 +98,13 @@ export class OllayaEvaluateAdapter<
 
     if (!response.ok) {
       const detail = await response.text().catch(() => '')
-      const error = new Error(
-        `Ollaya evaluate request failed: ${response.status} ${response.statusText}${
-          detail ? ` — ${detail}` : ''
-        }`,
+      const error = Object.assign(
+        new Error(
+          `Ollaya evaluate request failed: ${response.status} ${response.statusText}${
+            detail ? ` — ${detail}` : ''
+          }`,
+        ),
+        { status: response.status },
       )
       logger.errors(`${this.name}.evaluate fatal`, {
         error,

packages/ai-ollaya/tests/evaluate-adapter.test.ts

@@ -171,7 +171,7 @@ describe('OllayaEvaluateAdapter', () => {
     ).toBe('Bearer proxy-token')
   })
 
-  it('throws with the response body detail on a non-OK status', async () => {
+  it('throws with status and the response body detail on a non-OK status', async () => {
     fetchMock.mockResolvedValue(
       new Response('model not found', {
         status: 404,
@@ -181,7 +181,12 @@ describe('OllayaEvaluateAdapter', () => {
 
     await expect(
       ollayaDecider('laya:latest').evaluate(evaluateOptions()),
-    ).rejects.toThrow('404')
+    ).rejects.toMatchObject({
+      status: 404,
+      message: expect.stringMatching(
+        /^Ollaya evaluate request failed: 404 Not Found .+ model not found$/,
+      ),
+    })
   })
 
   it('throws when the response shape is unexpected', async () => {

packages/ai-openrouter/src/adapters/evaluate.ts

@@ -151,10 +151,13 @@ export class OpenRouterEvaluateAdapter<
 
       if (!response.ok) {
         const detail = await response.text().catch(() => '')
-        throw new Error(
-          `OpenRouter evaluate request failed: ${response.status} ${response.statusText}${
-            detail ? ` — ${detail}` : ''
-          }`,
+        throw Object.assign(
+          new Error(
+            `OpenRouter evaluate request failed: ${response.status} ${response.statusText}${
+              detail ? ` — ${detail}` : ''
+            }`,
+          ),
+          { status: response.status },
         )
       }
 

packages/ai-openrouter/tests/evaluate-adapter.test.ts

@@ -237,7 +237,7 @@ describe('OpenRouterEvaluateAdapter', () => {
     })
   })
 
-  it('throws on a non-OK response', async () => {
+  it('throws with status on a non-OK response', async () => {
     fetchMock.mockResolvedValue(
       new Response('bad request', { status: 400, statusText: 'Bad Request' }),
     )
@@ -253,6 +253,11 @@ describe('OpenRouterEvaluateAdapter', () => {
           }),
         },
       }),
-    ).rejects.toThrow()
+    ).rejects.toMatchObject({
+      status: 400,
+      message: expect.stringMatching(
+        /^OpenRouter evaluate request failed: 400 Bad Request .+ bad request$/,
+      ),
+    })
   })
 })

packages/ai-typesafe/src/adapters/evaluate.ts

@@ -63,6 +63,9 @@ export class TypesafeEvaluateAdapter<
   /**
    * POST `{ model, state, questions }` to TypeSafe and return wire answers
    * plus mapped token usage. Does not invent unified `.value` fields.
+   *
+   * @throws An `Error` with a numeric `status` (the HTTP status code) when
+   * TypeSafe answers with a non-OK response.
    */
   async evaluate(options: EvaluateOptions) {
     const { model, state, questions, abortSignal, logger } = options
@@ -96,10 +99,13 @@ export class TypesafeEvaluateAdapter<
 
     if (!response.ok) {
       const detail = await response.text().catch(() => '')
-      const error = new Error(
-        `TypeSafe evaluate request failed: ${response.status} ${response.statusText}${
-          detail ? ` — ${detail}` : ''
-        }`,
+      const error = Object.assign(
+        new Error(
+          `TypeSafe evaluate request failed: ${response.status} ${response.statusText}${
+            detail ? ` — ${detail}` : ''
+          }`,
+        ),
+        { status: response.status },
       )
       logger.errors(`${this.name}.evaluate fatal`, {
         error,

packages/ai-typesafe/tests/evaluate-adapter.test.ts

@@ -137,22 +137,27 @@ describe('TypesafeEvaluateAdapter', () => {
     })
   })
 
-  it('throws on 401', async () => {
+  it('throws with status on 401', async () => {
     await expectStatus(401, 'Unauthorized', 'invalid api key')
   })
 
-  it('throws on 422', async () => {
+  it('throws with status on 422', async () => {
     await expectStatus(422, 'Unprocessable Entity', 'malformed question')
   })
 
-  it('throws on 429', async () => {
+  it('throws with status on 429', async () => {
     await expectStatus(429, 'Too Many Requests', 'rate limited')
   })
 })
 
 async function expectStatus(status: number, statusText: string, body: string) {
   fetchMock.mockResolvedValue(new Response(body, { status, statusText }))
-  await expect(adapter().evaluate(evaluateOptions())).rejects.toThrow(
-    String(status),
-  )
+  await expect(adapter().evaluate(evaluateOptions())).rejects.toMatchObject({
+    status,
+    message: expect.stringMatching(
+      new RegExp(
+        `^TypeSafe evaluate request failed: ${status} ${statusText} .+ ${body}$`,
+      ),
+    ),
+  })
 }

packages/ai-vercel-gateway/src/adapters/evaluate.ts

@@ -302,10 +302,13 @@ export class VercelGatewayEvaluateAdapter<
 
       if (!response.ok) {
         const detail = await response.text().catch(() => '')
-        throw new Error(
-          `Vercel Gateway evaluate request failed: ${response.status} ${response.statusText}${
-            detail ? ` — ${detail}` : ''
-          }`,
+        throw Object.assign(
+          new Error(
+            `Vercel Gateway evaluate request failed: ${response.status} ${response.statusText}${
+              detail ? ` — ${detail}` : ''
+            }`,
+          ),
+          { status: response.status },
         )
       }
 

packages/ai-vercel-gateway/tests/evaluate-adapter.test.ts

@@ -147,7 +147,7 @@ it('POSTs to /v4/ai/evaluation-model and maps boolean/choice/score answers', asy
   })
 })
 
-it('throws when the evaluate request is not OK', async () => {
+it('throws with status when the evaluate request is not OK', async () => {
   fetchMock.mockResolvedValue(
     new Response('gateway down', {
       status: 503,
@@ -157,7 +157,9 @@ it('throws when the evaluate request is not OK', async () => {
 
   const adapter = createVercelGatewayDecider('typesafe-ai/jev', 'gw_test_key')
 
-  await expect(adapter.evaluate(evaluateOptions())).rejects.toThrow(
-    'Vercel Gateway evaluate request failed: 503 Service Unavailable — gateway down',
-  )
+  await expect(adapter.evaluate(evaluateOptions())).rejects.toMatchObject({
+    status: 503,
+    message:
+      'Vercel Gateway evaluate request failed: 503 Service Unavailable — gateway down',
+  })
 })

testing/e2e/global-setup.ts

@@ -181,6 +181,13 @@ export default async function globalSetup() {
   // asserts `decide()` rejects and still reports that usage.
   mock.mount('/evaluate-usage-on-failure', evaluateUsageOnFailureMount())
 
+  // Failed evaluate requests. TypeSafe takes a path prefix, but the Vercel
+  // Gateway adapter keeps only the origin of its baseURL, so that mount sits on
+  // the real evaluate path. Both answer only requests carrying the
+  // `x-evaluate-http-status` header and fall through otherwise.
+  mock.mount('/evaluate-http-status', evaluateHttpStatusMount())
+  mock.mount('/v4/ai/evaluation-model', evaluateHttpStatusMount())
+
   // BytePlus. Ark (chat, Seedream image, Seedance video) serves its whole data
   // plane under /api/v3, and Seed Speech (TTS/ASR) mounts its own endpoints at
   // the same prefix on a different host — which collapses onto one prefix here
@@ -2226,6 +2233,29 @@ function evaluateUsageOnFailureMount(): Mountable {
   }
 }
 
+/**
+ * Answers an evaluate request with the HTTP status named in its
+ * `x-evaluate-http-status` header and a small JSON error body.
+ */
+function evaluateHttpStatusMount(): Mountable {
+  return {
+    async handleRequest(
+      req: http.IncomingMessage,
+      res: http.ServerResponse,
+    ): Promise<boolean> {
+      const status = Number(req.headers['x-evaluate-http-status'])
+      if (req.method !== 'POST' || !(status >= 400 && status <= 599)) {
+        return false
+      }
+      await drainBody(req)
+      res.statusCode = status
+      res.setHeader('Content-Type', 'application/json')
+      res.end(JSON.stringify({ error: 'rejected' }))
+      return true
+    },
+  }
+}
+
 function buildToolPlusServerToolEvents(): Array<Record<string, unknown>> {
   const messageId = 'msg_bug_604'
   const model = 'claude-sonnet-4-5'

testing/e2e/package.json

@@ -52,6 +52,7 @@
     "@tanstack/ai-sandbox-local-process": "workspace:*",
     "@tanstack/ai-skills": "workspace:*",
     "@tanstack/ai-solid": "workspace:*",
+    "@tanstack/ai-typesafe": "workspace:*",
     "@tanstack/ai-vercel-gateway": "workspace:*",
     "@tanstack/ai-vertex": "workspace:*",
     "@tanstack/ai-worldlabs": "workspace:*",