From f8828d3d29f356dd002fc5d9d54a930cbb8af6a7 Mon Sep 17 00:00:00 2001 From: Chris Trzesniewski Date: Tue, 13 Oct 2026 13:41:13 +0200 Subject: [PATCH 1/5] feat(ai-typesafe): set the HTTP status on a failed evaluate request - Throw the non-OK response error with a numeric `status` field - Keep the error message as it was - Match the `status` field that SDK errors and toRunErrorPayload already use - Assert `status` and the full message in the 401, 422 and 429 tests - Add a minor changeset for @tanstack/ai-typesafe --- .changeset/evaluate-http-status.md | 11 +++++++++++ packages/ai-typesafe/src/adapters/evaluate.ts | 14 ++++++++++---- .../ai-typesafe/tests/evaluate-adapter.test.ts | 17 +++++++++++------ 3 files changed, 32 insertions(+), 10 deletions(-) create mode 100644 .changeset/evaluate-http-status.md diff --git a/.changeset/evaluate-http-status.md b/.changeset/evaluate-http-status.md new file mode 100644 index 0000000000..94702ae82a --- /dev/null +++ b/.changeset/evaluate-http-status.md @@ -0,0 +1,11 @@ +--- +'@tanstack/ai-typesafe': minor +--- + +The TypeSafe evaluate adapter now sets `status` on the error it throws for an error response. `status` is the HTTP status code, so you can tell a rate limit (429) from a bad key (401) without parsing the message. The message does not change. + +```ts +if (error instanceof Error && 'status' in error && error.status === 429) { + // wait, then call decide() again +} +``` diff --git a/packages/ai-typesafe/src/adapters/evaluate.ts b/packages/ai-typesafe/src/adapters/evaluate.ts index 1e2ce9284d..6fa805486d 100644 --- a/packages/ai-typesafe/src/adapters/evaluate.ts +++ b/packages/ai-typesafe/src/adapters/evaluate.ts @@ -63,6 +63,9 @@ export class TypesafeEvaluateAdapter< /** * POST `{ model, state, questions }` to TypeSafe and return wire answers * plus mapped token usage. Does not invent unified `.value` fields. + * + * @throws An `Error` with a numeric `status` (the HTTP status code) when + * TypeSafe answers with a non-OK response. */ async evaluate(options: EvaluateOptions) { const { model, state, questions, abortSignal, logger } = options @@ -96,10 +99,13 @@ export class TypesafeEvaluateAdapter< if (!response.ok) { const detail = await response.text().catch(() => '') - const error = new Error( - `TypeSafe evaluate request failed: ${response.status} ${response.statusText}${ - detail ? ` — ${detail}` : '' - }`, + const error = Object.assign( + new Error( + `TypeSafe evaluate request failed: ${response.status} ${response.statusText}${ + detail ? ` — ${detail}` : '' + }`, + ), + { status: response.status }, ) logger.errors(`${this.name}.evaluate fatal`, { error, diff --git a/packages/ai-typesafe/tests/evaluate-adapter.test.ts b/packages/ai-typesafe/tests/evaluate-adapter.test.ts index f5477d9e58..33c0b410dd 100644 --- a/packages/ai-typesafe/tests/evaluate-adapter.test.ts +++ b/packages/ai-typesafe/tests/evaluate-adapter.test.ts @@ -137,22 +137,27 @@ describe('TypesafeEvaluateAdapter', () => { }) }) - it('throws on 401', async () => { + it('throws with status on 401', async () => { await expectStatus(401, 'Unauthorized', 'invalid api key') }) - it('throws on 422', async () => { + it('throws with status on 422', async () => { await expectStatus(422, 'Unprocessable Entity', 'malformed question') }) - it('throws on 429', async () => { + it('throws with status on 429', async () => { await expectStatus(429, 'Too Many Requests', 'rate limited') }) }) async function expectStatus(status: number, statusText: string, body: string) { fetchMock.mockResolvedValue(new Response(body, { status, statusText })) - await expect(adapter().evaluate(evaluateOptions())).rejects.toThrow( - String(status), - ) + await expect(adapter().evaluate(evaluateOptions())).rejects.toMatchObject({ + status, + message: expect.stringMatching( + new RegExp( + `^TypeSafe evaluate request failed: ${status} ${statusText} .+ ${body}$`, + ), + ), + }) } From 7d4df31884c1d32ce3ac9c4a023b8f51a10927c5 Mon Sep 17 00:00:00 2001 From: Chris Trzesniewski Date: Tue, 13 Oct 2026 12:18:35 +0200 Subject: [PATCH 2/5] docs(ai-typesafe): show how to read the status of a failed request - Add a "Handle a failed request" section to the TypeSafe adapter page - Refresh the page's updatedAt in docs/config.json --- docs/adapters/typesafe.md | 23 +++++++++++++++++++++++ docs/config.json | 3 ++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/docs/adapters/typesafe.md b/docs/adapters/typesafe.md index 9fa17e6f4f..7449ca5656 100644 --- a/docs/adapters/typesafe.md +++ b/docs/adapters/typesafe.md @@ -131,6 +131,29 @@ const adapter = createTypesafeDecider( `baseUrl` and `headers` are aliases of the same two options. If you set both forms, `baseURL` and `defaultHeaders` win. +## Handle a failed request + +A rate limit needs a retry, but a bad API key does not. When TypeSafe sends an error response, the thrown error has the HTTP status code in `status`: + +```typescript +import { decide, boolean } from "@tanstack/ai"; +import { typesafeDecider } from "@tanstack/ai-typesafe"; + +try { + await decide({ + adapter: typesafeDecider("jev-latest"), + state: { body: "Please refund the extra payment." }, + questions: { + refund: boolean({ instructions: "Is the customer asking for a refund?" }), + }, + }); +} catch (error) { + if (error instanceof Error && "status" in error && error.status === 429) { + // Rate limited. Wait, then call decide() again. + } +} +``` + ## API Reference ### `typesafeDecider(model, config?)` diff --git a/docs/config.json b/docs/config.json index aee4f36442..c8e0564e23 100644 --- a/docs/config.json +++ b/docs/config.json @@ -1262,7 +1262,8 @@ { "label": "TypeSafe", "to": "adapters/typesafe", - "addedAt": "2026-09-18" + "addedAt": "2026-09-18", + "updatedAt": "2026-10-13" }, { "label": "Ollaya", From 0b2fe32f15b96903fa93c373eaf7866d4ca9b977 Mon Sep 17 00:00:00 2001 From: Chris Trzesniewski Date: Tue, 13 Oct 2026 13:41:17 +0200 Subject: [PATCH 3/5] feat(adapters): set the HTTP status on failed Ollaya, OpenRouter and Vercel Gateway evaluate requests - Throw the non-OK response error with a numeric `status`, the same as the TypeSafe adapter - Keep each error message unchanged - Document the thrown `status` with a JSDoc `@throws` line on the Ollaya adapter - Assert `status` and the full message in each adapter's non-OK test - Extend the changeset to all four evaluate adapters --- .changeset/evaluate-http-status.md | 5 ++++- packages/ai-ollaya/src/adapters/evaluate.ts | 14 ++++++++++---- packages/ai-ollaya/tests/evaluate-adapter.test.ts | 9 +++++++-- packages/ai-openrouter/src/adapters/evaluate.ts | 11 +++++++---- .../ai-openrouter/tests/evaluate-adapter.test.ts | 9 +++++++-- .../ai-vercel-gateway/src/adapters/evaluate.ts | 11 +++++++---- .../tests/evaluate-adapter.test.ts | 10 ++++++---- 7 files changed, 48 insertions(+), 21 deletions(-) diff --git a/.changeset/evaluate-http-status.md b/.changeset/evaluate-http-status.md index 94702ae82a..918e7b1593 100644 --- a/.changeset/evaluate-http-status.md +++ b/.changeset/evaluate-http-status.md @@ -1,8 +1,11 @@ --- '@tanstack/ai-typesafe': minor +'@tanstack/ai-ollaya': minor +'@tanstack/ai-openrouter': minor +'@tanstack/ai-vercel-gateway': minor --- -The TypeSafe evaluate adapter now sets `status` on the error it throws for an error response. `status` is the HTTP status code, so you can tell a rate limit (429) from a bad key (401) without parsing the message. The message does not change. +The TypeSafe, Ollaya, OpenRouter and Vercel Gateway evaluate adapters now set `status` on the error they throw for an error response. `status` is the HTTP status code, so you can tell a rate limit (429) from a bad key (401) without parsing the message. The message does not change. ```ts if (error instanceof Error && 'status' in error && error.status === 429) { diff --git a/packages/ai-ollaya/src/adapters/evaluate.ts b/packages/ai-ollaya/src/adapters/evaluate.ts index 01e8715a39..f769cbab17 100644 --- a/packages/ai-ollaya/src/adapters/evaluate.ts +++ b/packages/ai-ollaya/src/adapters/evaluate.ts @@ -61,6 +61,9 @@ export class OllayaEvaluateAdapter< /** * POST `{ model, state, questions }` to Ollaya and return wire answers plus * mapped token usage. Does not invent unified `.value` fields. + * + * @throws An `Error` with a numeric `status` (the HTTP status code) when + * Ollaya answers with a non-OK response. */ async evaluate(options: EvaluateOptions) { const { model, state, questions, abortSignal, logger } = options @@ -95,10 +98,13 @@ export class OllayaEvaluateAdapter< if (!response.ok) { const detail = await response.text().catch(() => '') - const error = new Error( - `Ollaya evaluate request failed: ${response.status} ${response.statusText}${ - detail ? ` — ${detail}` : '' - }`, + const error = Object.assign( + new Error( + `Ollaya evaluate request failed: ${response.status} ${response.statusText}${ + detail ? ` — ${detail}` : '' + }`, + ), + { status: response.status }, ) logger.errors(`${this.name}.evaluate fatal`, { error, diff --git a/packages/ai-ollaya/tests/evaluate-adapter.test.ts b/packages/ai-ollaya/tests/evaluate-adapter.test.ts index 31c72befa9..75ca848b34 100644 --- a/packages/ai-ollaya/tests/evaluate-adapter.test.ts +++ b/packages/ai-ollaya/tests/evaluate-adapter.test.ts @@ -171,7 +171,7 @@ describe('OllayaEvaluateAdapter', () => { ).toBe('Bearer proxy-token') }) - it('throws with the response body detail on a non-OK status', async () => { + it('throws with status and the response body detail on a non-OK status', async () => { fetchMock.mockResolvedValue( new Response('model not found', { status: 404, @@ -181,7 +181,12 @@ describe('OllayaEvaluateAdapter', () => { await expect( ollayaDecider('laya:latest').evaluate(evaluateOptions()), - ).rejects.toThrow('404') + ).rejects.toMatchObject({ + status: 404, + message: expect.stringMatching( + /^Ollaya evaluate request failed: 404 Not Found .+ model not found$/, + ), + }) }) it('throws when the response shape is unexpected', async () => { diff --git a/packages/ai-openrouter/src/adapters/evaluate.ts b/packages/ai-openrouter/src/adapters/evaluate.ts index 6183fe056f..4b61b321cf 100644 --- a/packages/ai-openrouter/src/adapters/evaluate.ts +++ b/packages/ai-openrouter/src/adapters/evaluate.ts @@ -151,10 +151,13 @@ export class OpenRouterEvaluateAdapter< if (!response.ok) { const detail = await response.text().catch(() => '') - throw new Error( - `OpenRouter evaluate request failed: ${response.status} ${response.statusText}${ - detail ? ` — ${detail}` : '' - }`, + throw Object.assign( + new Error( + `OpenRouter evaluate request failed: ${response.status} ${response.statusText}${ + detail ? ` — ${detail}` : '' + }`, + ), + { status: response.status }, ) } diff --git a/packages/ai-openrouter/tests/evaluate-adapter.test.ts b/packages/ai-openrouter/tests/evaluate-adapter.test.ts index 4a7b2238d0..a3fbafdc5a 100644 --- a/packages/ai-openrouter/tests/evaluate-adapter.test.ts +++ b/packages/ai-openrouter/tests/evaluate-adapter.test.ts @@ -237,7 +237,7 @@ describe('OpenRouterEvaluateAdapter', () => { }) }) - it('throws on a non-OK response', async () => { + it('throws with status on a non-OK response', async () => { fetchMock.mockResolvedValue( new Response('bad request', { status: 400, statusText: 'Bad Request' }), ) @@ -253,6 +253,11 @@ describe('OpenRouterEvaluateAdapter', () => { }), }, }), - ).rejects.toThrow() + ).rejects.toMatchObject({ + status: 400, + message: expect.stringMatching( + /^OpenRouter evaluate request failed: 400 Bad Request .+ bad request$/, + ), + }) }) }) diff --git a/packages/ai-vercel-gateway/src/adapters/evaluate.ts b/packages/ai-vercel-gateway/src/adapters/evaluate.ts index 537ff486f7..dafdca6bfd 100644 --- a/packages/ai-vercel-gateway/src/adapters/evaluate.ts +++ b/packages/ai-vercel-gateway/src/adapters/evaluate.ts @@ -302,10 +302,13 @@ export class VercelGatewayEvaluateAdapter< if (!response.ok) { const detail = await response.text().catch(() => '') - throw new Error( - `Vercel Gateway evaluate request failed: ${response.status} ${response.statusText}${ - detail ? ` — ${detail}` : '' - }`, + throw Object.assign( + new Error( + `Vercel Gateway evaluate request failed: ${response.status} ${response.statusText}${ + detail ? ` — ${detail}` : '' + }`, + ), + { status: response.status }, ) } diff --git a/packages/ai-vercel-gateway/tests/evaluate-adapter.test.ts b/packages/ai-vercel-gateway/tests/evaluate-adapter.test.ts index 1105b5fb4a..bac7a8b525 100644 --- a/packages/ai-vercel-gateway/tests/evaluate-adapter.test.ts +++ b/packages/ai-vercel-gateway/tests/evaluate-adapter.test.ts @@ -147,7 +147,7 @@ it('POSTs to /v4/ai/evaluation-model and maps boolean/choice/score answers', asy }) }) -it('throws when the evaluate request is not OK', async () => { +it('throws with status when the evaluate request is not OK', async () => { fetchMock.mockResolvedValue( new Response('gateway down', { status: 503, @@ -157,7 +157,9 @@ it('throws when the evaluate request is not OK', async () => { const adapter = createVercelGatewayDecider('typesafe-ai/jev', 'gw_test_key') - await expect(adapter.evaluate(evaluateOptions())).rejects.toThrow( - 'Vercel Gateway evaluate request failed: 503 Service Unavailable — gateway down', - ) + await expect(adapter.evaluate(evaluateOptions())).rejects.toMatchObject({ + status: 503, + message: + 'Vercel Gateway evaluate request failed: 503 Service Unavailable — gateway down', + }) }) From d946ae2078afb94a75ecc779841a4be518431fdd Mon Sep 17 00:00:00 2001 From: Chris Trzesniewski Date: Tue, 13 Oct 2026 13:41:18 +0200 Subject: [PATCH 4/5] docs(adapters): show how to read the status of a failed evaluate request - Add a "Handle a failed request" section to the Ollaya adapter page - Add a failed-request note to the Evaluate sections of the OpenRouter and Vercel Gateway pages - Refresh updatedAt for the Ollaya and Vercel Gateway pages in docs/config.json --- docs/adapters/ollaya.md | 23 +++++++++++++++++++++++ docs/adapters/openrouter.md | 21 +++++++++++++++++++++ docs/adapters/vercel-gateway.md | 21 +++++++++++++++++++++ docs/config.json | 5 +++-- 4 files changed, 68 insertions(+), 2 deletions(-) diff --git a/docs/adapters/ollaya.md b/docs/adapters/ollaya.md index f315e126b9..83a514f71a 100644 --- a/docs/adapters/ollaya.md +++ b/docs/adapters/ollaya.md @@ -134,6 +134,29 @@ const adapter = ollayaDecider("laya:latest", { If you set both forms, `baseURL` and `defaultHeaders` win. A blank `baseURL` keeps the default host. +## Handle a failed request + +A wrong `apiKey` needs a fix, not a retry. When the server sends an error response, the thrown error has the HTTP status code in `status`: + +```typescript +import { decide, boolean } from "@tanstack/ai"; +import { ollayaDecider } from "@tanstack/ai-ollaya"; + +try { + await decide({ + adapter: ollayaDecider("laya:latest"), + state: { body: "Please refund the extra payment." }, + questions: { + refund: boolean({ instructions: "Is the customer asking for a refund?" }), + }, + }); +} catch (error) { + if (error instanceof Error && "status" in error && error.status === 401) { + // The server rejected the apiKey. Check it before you call decide() again. + } +} +``` + ## API reference ### `ollayaDecider(model, config?)` diff --git a/docs/adapters/openrouter.md b/docs/adapters/openrouter.md index f180126cd3..e19e15baf3 100644 --- a/docs/adapters/openrouter.md +++ b/docs/adapters/openrouter.md @@ -644,6 +644,27 @@ Known slugs: - `typesafe/jev-1.13` - `typesafe/jev-1.13.0` +When OpenRouter sends an error response, the thrown error has the HTTP status code in `status`. A rate limit (429) needs a retry, but a bad key (401) does not: + +```typescript +import { decide, boolean } from "@tanstack/ai"; +import { openRouterDecider } from "@tanstack/ai-openrouter"; + +try { + await decide({ + adapter: openRouterDecider("~typesafe/jev-latest"), + state: { body: "Please refund the extra payment." }, + questions: { + refund: boolean({ instructions: "Is the customer asking for a refund?" }), + }, + }); +} catch (error) { + if (error instanceof Error && "status" in error && error.status === 429) { + // Rate limited. Wait, then call decide() again. + } +} +``` + See the [Evaluate guide](../evaluate/evaluate) for question helpers, the result shape, abort, and middleware. diff --git a/docs/adapters/vercel-gateway.md b/docs/adapters/vercel-gateway.md index dc2963a006..20e12858d8 100644 --- a/docs/adapters/vercel-gateway.md +++ b/docs/adapters/vercel-gateway.md @@ -271,6 +271,27 @@ Pass a key yourself with `createVercelGatewayDecider("typesafe-ai/jev", "vck_... Put Gateway routing on `modelOptions.gateway`, the same as chat. +When the Gateway sends an error response, the thrown error has the HTTP status code in `status`. A rate limit (429) needs a retry, but a bad key (401) does not: + +```typescript +import { decide, boolean } from "@tanstack/ai" +import { vercelGatewayDecider } from "@tanstack/ai-vercel-gateway" + +try { + await decide({ + adapter: vercelGatewayDecider("typesafe-ai/jev"), + state: { body: "Please refund the extra payment." }, + questions: { + refund: boolean({ instructions: "Is the customer asking for a refund?" }), + }, + }) +} catch (error) { + if (error instanceof Error && "status" in error && error.status === 429) { + // Rate limited. Wait, then call decide() again. + } +} +``` + See the [Evaluate guide](../evaluate/evaluate) for question helpers, the result shape, abort, and middleware. diff --git a/docs/config.json b/docs/config.json index c8e0564e23..c5d41bb17e 100644 --- a/docs/config.json +++ b/docs/config.json @@ -1268,7 +1268,8 @@ { "label": "Ollaya", "to": "adapters/ollaya", - "addedAt": "2026-09-29" + "addedAt": "2026-09-29", + "updatedAt": "2026-10-13" }, { "label": "ElevenLabs", @@ -1309,7 +1310,7 @@ "label": "Vercel AI Gateway", "to": "adapters/vercel-gateway", "addedAt": "2026-08-12", - "updatedAt": "2026-10-09" + "updatedAt": "2026-10-13" }, { "label": "Lovable AI Gateway", From 5fb3d8fd356f5c11324e198913ded886c0256f8c Mon Sep 17 00:00:00 2001 From: Chris Trzesniewski Date: Tue, 13 Oct 2026 15:12:35 +0200 Subject: [PATCH 5/5] test(e2e): cover the HTTP status on a failed evaluate request - Add an aimock mount that answers an evaluate request with the HTTP status named in its x-evaluate-http-status header - Mount it at /evaluate-http-status for TypeSafe and at /v4/ai/evaluation-model for Vercel Gateway, which keeps only the origin of its baseURL - Add the /api/evaluate-http-status route, which runs decide() and returns the error name, message and status as JSON - Assert that decide() rejects with status 429 for TypeSafe and 503 for Vercel Gateway and an unchanged message - Add @tanstack/ai-typesafe to the e2e app dependencies --- pnpm-lock.yaml | 3 + testing/e2e/global-setup.ts | 30 ++++++++ testing/e2e/package.json | 1 + testing/e2e/src/routeTree.gen.ts | 21 ++++++ .../src/routes/api.evaluate-http-status.ts | 71 +++++++++++++++++++ .../e2e/tests/evaluate-http-status.spec.ts | 38 ++++++++++ 6 files changed, 164 insertions(+) create mode 100644 testing/e2e/src/routes/api.evaluate-http-status.ts create mode 100644 testing/e2e/tests/evaluate-http-status.spec.ts diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index bb0a8207db..0e00939c2e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -3741,6 +3741,9 @@ importers: '@tanstack/ai-solid': specifier: workspace:* version: link:../../packages/ai-solid + '@tanstack/ai-typesafe': + specifier: workspace:* + version: link:../../packages/ai-typesafe '@tanstack/ai-vercel-gateway': specifier: workspace:* version: link:../../packages/ai-vercel-gateway diff --git a/testing/e2e/global-setup.ts b/testing/e2e/global-setup.ts index 4f06314e99..e810125272 100644 --- a/testing/e2e/global-setup.ts +++ b/testing/e2e/global-setup.ts @@ -186,6 +186,13 @@ export default async function globalSetup() { // asserts `decide()` rejects and still reports that usage. mock.mount('/evaluate-usage-on-failure', evaluateUsageOnFailureMount()) + // Failed evaluate requests. TypeSafe takes a path prefix, but the Vercel + // Gateway adapter keeps only the origin of its baseURL, so that mount sits on + // the real evaluate path. Both answer only requests carrying the + // `x-evaluate-http-status` header and fall through otherwise. + mock.mount('/evaluate-http-status', evaluateHttpStatusMount()) + mock.mount('/v4/ai/evaluation-model', evaluateHttpStatusMount()) + // BytePlus. Ark (chat, Seedream image, Seedance video) serves its whole data // plane under /api/v3, and Seed Speech (TTS/ASR) mounts its own endpoints at // the same prefix on a different host — which collapses onto one prefix here @@ -2304,6 +2311,29 @@ function evaluateUsageOnFailureMount(): Mountable { } } +/** + * Answers an evaluate request with the HTTP status named in its + * `x-evaluate-http-status` header and a small JSON error body. + */ +function evaluateHttpStatusMount(): Mountable { + return { + async handleRequest( + req: http.IncomingMessage, + res: http.ServerResponse, + ): Promise { + const status = Number(req.headers['x-evaluate-http-status']) + if (req.method !== 'POST' || !(status >= 400 && status <= 599)) { + return false + } + await drainBody(req) + res.statusCode = status + res.setHeader('Content-Type', 'application/json') + res.end(JSON.stringify({ error: 'rejected' })) + return true + }, + } +} + function buildToolPlusServerToolEvents(): Array> { const messageId = 'msg_bug_604' const model = 'claude-sonnet-4-5' diff --git a/testing/e2e/package.json b/testing/e2e/package.json index 7001fbcc32..bc599d9a7f 100644 --- a/testing/e2e/package.json +++ b/testing/e2e/package.json @@ -46,6 +46,7 @@ "@tanstack/ai-sandbox-local-process": "workspace:*", "@tanstack/ai-skills": "workspace:*", "@tanstack/ai-solid": "workspace:*", + "@tanstack/ai-typesafe": "workspace:*", "@tanstack/ai-vercel-gateway": "workspace:*", "@tanstack/ai-vertex": "workspace:*", "@tanstack/ai-worldlabs": "workspace:*", diff --git a/testing/e2e/src/routeTree.gen.ts b/testing/e2e/src/routeTree.gen.ts index 780d06b4dc..36178d37f1 100644 --- a/testing/e2e/src/routeTree.gen.ts +++ b/testing/e2e/src/routeTree.gen.ts @@ -84,6 +84,7 @@ import { Route as ApiDevtoolsMemoryRouteImport } from './routes/api.devtools-mem import { Route as ApiDurableDeliveryRouteImport } from './routes/api.durable-delivery' import { Route as ApiDurableTakeoverRouteImport } from './routes/api.durable-takeover' import { Route as ApiEmbeddingRouteImport } from './routes/api.embedding' +import { Route as ApiEvaluateHttpStatusRouteImport } from './routes/api.evaluate-http-status' import { Route as ApiEvaluateUsageOnFailureRouteImport } from './routes/api.evaluate-usage-on-failure' import { Route as ApiFakeTextRouteImport } from './routes/api.fake-text' import { Route as ApiFileSourceWireRouteImport } from './routes/api.file-source-wire' @@ -575,6 +576,11 @@ const ApiEmbeddingRoute = ApiEmbeddingRouteImport.update({ path: '/api/embedding', getParentRoute: () => rootRouteImport, } as any) +const ApiEvaluateHttpStatusRoute = ApiEvaluateHttpStatusRouteImport.update({ + id: '/api/evaluate-http-status', + path: '/api/evaluate-http-status', + getParentRoute: () => rootRouteImport, +} as any) const ApiEvaluateUsageOnFailureRoute = ApiEvaluateUsageOnFailureRouteImport.update({ id: '/api/evaluate-usage-on-failure', @@ -1161,6 +1167,7 @@ export interface FileRoutesByFullPath { '/api/durable-delivery': typeof ApiDurableDeliveryRoute '/api/durable-takeover': typeof ApiDurableTakeoverRoute '/api/embedding': typeof ApiEmbeddingRoute + '/api/evaluate-http-status': typeof ApiEvaluateHttpStatusRoute '/api/evaluate-usage-on-failure': typeof ApiEvaluateUsageOnFailureRoute '/api/fake-text': typeof ApiFakeTextRoute '/api/file-source-wire': typeof ApiFileSourceWireRoute @@ -1335,6 +1342,7 @@ export interface FileRoutesByTo { '/api/durable-delivery': typeof ApiDurableDeliveryRoute '/api/durable-takeover': typeof ApiDurableTakeoverRoute '/api/embedding': typeof ApiEmbeddingRoute + '/api/evaluate-http-status': typeof ApiEvaluateHttpStatusRoute '/api/evaluate-usage-on-failure': typeof ApiEvaluateUsageOnFailureRoute '/api/fake-text': typeof ApiFakeTextRoute '/api/file-source-wire': typeof ApiFileSourceWireRoute @@ -1510,6 +1518,7 @@ export interface FileRoutesById { '/api/durable-delivery': typeof ApiDurableDeliveryRoute '/api/durable-takeover': typeof ApiDurableTakeoverRoute '/api/embedding': typeof ApiEmbeddingRoute + '/api/evaluate-http-status': typeof ApiEvaluateHttpStatusRoute '/api/evaluate-usage-on-failure': typeof ApiEvaluateUsageOnFailureRoute '/api/fake-text': typeof ApiFakeTextRoute '/api/file-source-wire': typeof ApiFileSourceWireRoute @@ -1686,6 +1695,7 @@ export interface FileRouteTypes { | '/api/durable-delivery' | '/api/durable-takeover' | '/api/embedding' + | '/api/evaluate-http-status' | '/api/evaluate-usage-on-failure' | '/api/fake-text' | '/api/file-source-wire' @@ -1860,6 +1870,7 @@ export interface FileRouteTypes { | '/api/durable-delivery' | '/api/durable-takeover' | '/api/embedding' + | '/api/evaluate-http-status' | '/api/evaluate-usage-on-failure' | '/api/fake-text' | '/api/file-source-wire' @@ -2034,6 +2045,7 @@ export interface FileRouteTypes { | '/api/durable-delivery' | '/api/durable-takeover' | '/api/embedding' + | '/api/evaluate-http-status' | '/api/evaluate-usage-on-failure' | '/api/fake-text' | '/api/file-source-wire' @@ -2209,6 +2221,7 @@ export interface RootRouteChildren { ApiDurableDeliveryRoute: typeof ApiDurableDeliveryRoute ApiDurableTakeoverRoute: typeof ApiDurableTakeoverRoute ApiEmbeddingRoute: typeof ApiEmbeddingRoute + ApiEvaluateHttpStatusRoute: typeof ApiEvaluateHttpStatusRoute ApiEvaluateUsageOnFailureRoute: typeof ApiEvaluateUsageOnFailureRoute ApiFakeTextRoute: typeof ApiFakeTextRoute ApiFileSourceWireRoute: typeof ApiFileSourceWireRoute @@ -2831,6 +2844,13 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof ApiEmbeddingRouteImport parentRoute: typeof rootRouteImport } + '/api/evaluate-http-status': { + id: '/api/evaluate-http-status' + path: '/api/evaluate-http-status' + fullPath: '/api/evaluate-http-status' + preLoaderRoute: typeof ApiEvaluateHttpStatusRouteImport + parentRoute: typeof rootRouteImport + } '/api/evaluate-usage-on-failure': { id: '/api/evaluate-usage-on-failure' path: '/api/evaluate-usage-on-failure' @@ -3651,6 +3671,7 @@ const rootRouteChildren: RootRouteChildren = { ApiDurableDeliveryRoute: ApiDurableDeliveryRoute, ApiDurableTakeoverRoute: ApiDurableTakeoverRoute, ApiEmbeddingRoute: ApiEmbeddingRoute, + ApiEvaluateHttpStatusRoute: ApiEvaluateHttpStatusRoute, ApiEvaluateUsageOnFailureRoute: ApiEvaluateUsageOnFailureRoute, ApiFakeTextRoute: ApiFakeTextRoute, ApiFileSourceWireRoute: ApiFileSourceWireRoute, diff --git a/testing/e2e/src/routes/api.evaluate-http-status.ts b/testing/e2e/src/routes/api.evaluate-http-status.ts new file mode 100644 index 0000000000..19817c1726 --- /dev/null +++ b/testing/e2e/src/routes/api.evaluate-http-status.ts @@ -0,0 +1,71 @@ +import { createFileRoute } from '@tanstack/react-router' +import { boolean, decide } from '@tanstack/ai' +import { createTypesafeDecider } from '@tanstack/ai-typesafe' +import { createVercelGatewayDecider } from '@tanstack/ai-vercel-gateway' + +const LLMOCK_DEFAULT_BASE = process.env.LLMOCK_URL || 'http://127.0.0.1:4010' +const DUMMY_KEY = 'sk-e2e-test-dummy-key' + +const STATE = 'The checkout page has been down for an hour.' +const QUESTIONS = { + isUrgent: boolean({ instructions: 'Does this message convey urgency?' }), +} + +export const Route = createFileRoute('/api/evaluate-http-status')({ + server: { + handlers: { + POST: async ({ request }) => { + const params = new URL(request.url).searchParams + const provider = params.get('provider') + const defaultHeaders = { + 'x-evaluate-http-status': params.get('status') ?? '', + } + + try { + if (provider === 'typesafe') { + await decide({ + adapter: createTypesafeDecider('jev-latest', DUMMY_KEY, { + baseURL: `${LLMOCK_DEFAULT_BASE}/evaluate-http-status`, + defaultHeaders, + }), + state: STATE, + questions: QUESTIONS, + }) + } else if (provider === 'vercel-gateway') { + await decide({ + adapter: createVercelGatewayDecider( + 'typesafe-ai/jev', + DUMMY_KEY, + { + baseURL: LLMOCK_DEFAULT_BASE, + defaultHeaders, + }, + ), + state: STATE, + questions: QUESTIONS, + }) + } else { + return Response.json( + { ok: false, error: { message: 'Unsupported provider' } }, + { status: 400 }, + ) + } + } catch (error) { + return Response.json({ + ok: false, + error: + error instanceof Error + ? { + name: error.name, + message: error.message, + status: 'status' in error ? error.status : undefined, + } + : { message: String(error) }, + }) + } + + return Response.json({ ok: true }) + }, + }, + }, +}) diff --git a/testing/e2e/tests/evaluate-http-status.spec.ts b/testing/e2e/tests/evaluate-http-status.spec.ts new file mode 100644 index 0000000000..24f9e1dd6d --- /dev/null +++ b/testing/e2e/tests/evaluate-http-status.spec.ts @@ -0,0 +1,38 @@ +import { test, expect } from './fixtures' + +const CASES = [ + { + provider: 'typesafe', + status: 429, + message: + 'TypeSafe evaluate request failed: 429 Too Many Requests \u2014 {"error":"rejected"}', + }, + { + provider: 'vercel-gateway', + status: 503, + message: + 'Vercel Gateway evaluate request failed: 503 Service Unavailable \u2014 {"error":"rejected"}', + }, +] as const + +test.describe('decide() on a non-ok provider response', () => { + for (const { provider, status, message } of CASES) { + test(`${provider} rejects with the HTTP status on the error`, async ({ + request, + }) => { + const response = await request.post( + `/api/evaluate-http-status?provider=${provider}&status=${status}`, + ) + expect(response.ok()).toBe(true) + + const result = (await response.json()) as { + ok: boolean + error?: { name?: string; message: string; status?: number } + } + expect(result).toEqual({ + ok: false, + error: { name: 'Error', message, status }, + }) + }) + } +})