diff --git a/.changeset/evaluate-http-status.md b/.changeset/evaluate-http-status.md new file mode 100644 index 0000000000..918e7b1593 --- /dev/null +++ b/.changeset/evaluate-http-status.md @@ -0,0 +1,14 @@ +--- +'@tanstack/ai-typesafe': minor +'@tanstack/ai-ollaya': minor +'@tanstack/ai-openrouter': minor +'@tanstack/ai-vercel-gateway': minor +--- + +The TypeSafe, Ollaya, OpenRouter and Vercel Gateway evaluate adapters now set `status` on the error they throw for an error response. `status` is the HTTP status code, so you can tell a rate limit (429) from a bad key (401) without parsing the message. The message does not change. + +```ts +if (error instanceof Error && 'status' in error && error.status === 429) { + // wait, then call decide() again +} +``` diff --git a/docs/adapters/ollaya.md b/docs/adapters/ollaya.md index c005e352e3..490f089099 100644 --- a/docs/adapters/ollaya.md +++ b/docs/adapters/ollaya.md @@ -151,6 +151,29 @@ const adapter = ollayaDecider("laya:latest", { `baseUrl` and `headers` are aliases of `baseURL` and `defaultHeaders`. If you set both forms, `baseURL` and `defaultHeaders` win. A blank `baseURL` keeps the default host. +### Handle a failed request + +A wrong `apiKey` needs a fix, not a retry. When the server sends an error response, the thrown error has the HTTP status code in `status`: + +```typescript +import { decide, boolean } from "@tanstack/ai"; +import { ollayaDecider } from "@tanstack/ai-ollaya"; + +try { + await decide({ + adapter: ollayaDecider("laya:latest"), + state: { body: "Please refund the extra payment." }, + questions: { + refund: boolean({ instructions: "Is the customer asking for a refund?" }), + }, + }); +} catch (error) { + if (error instanceof Error && "status" in error && error.status === 401) { + // The server rejected the apiKey. Check it before you call decide() again. + } +} +``` + ### API reference The adapter sends `POST /v1/systemone` to the base URL with `fetch`. There is no Ollaya SDK. diff --git a/docs/adapters/openrouter.md b/docs/adapters/openrouter.md index 7b1a14562a..1078f639bd 100644 --- a/docs/adapters/openrouter.md +++ b/docs/adapters/openrouter.md @@ -294,6 +294,27 @@ console.log(result.meta.usage); Known ids: `~typesafe/jev-latest`, `typesafe/jev-1.13`, `typesafe/jev-1.13.0`. See the [Evaluate guide](../evaluate/evaluate). +When OpenRouter sends an error response, the thrown error has the HTTP status code in `status`. A rate limit (429) needs a retry, but a bad key (401) does not: + +```typescript +import { decide, boolean } from "@tanstack/ai"; +import { openRouterDecider } from "@tanstack/ai-openrouter"; + +try { + await decide({ + adapter: openRouterDecider("~typesafe/jev-latest"), + state: { body: "Please refund the extra payment." }, + questions: { + refund: boolean({ instructions: "Is the customer asking for a refund?" }), + }, + }); +} catch (error) { + if (error instanceof Error && "status" in error && error.status === 429) { + // Rate limited. Wait, then call decide() again. + } +} +``` + ## Summarization ```typescript diff --git a/docs/adapters/typesafe.md b/docs/adapters/typesafe.md index 236fdf08a6..5fd1b5cbd0 100644 --- a/docs/adapters/typesafe.md +++ b/docs/adapters/typesafe.md @@ -134,6 +134,29 @@ const adapter = createTypesafeDecider("jev-latest", process.env.TYPESAFE_API_KEY `baseUrl` and `headers` are aliases of the same two options. If you set both forms, `baseURL` and `defaultHeaders` win. +### Handle a failed request + +A rate limit needs a retry, but a bad API key does not. When TypeSafe sends an error response, the thrown error has the HTTP status code in `status`: + +```typescript +import { decide, boolean } from "@tanstack/ai"; +import { typesafeDecider } from "@tanstack/ai-typesafe"; + +try { + await decide({ + adapter: typesafeDecider("jev-latest"), + state: { body: "Please refund the extra payment." }, + questions: { + refund: boolean({ instructions: "Is the customer asking for a refund?" }), + }, + }); +} catch (error) { + if (error instanceof Error && "status" in error && error.status === 429) { + // Rate limited. Wait, then call decide() again. + } +} +``` + ### Bring your own key `typesafeByok` from `@tanstack/ai-typesafe/byok` declares TypeSafe for the BYOK store, with `TYPESAFE_API_KEY` as its env var. See [Bring Your Own Key](../advanced/byok). diff --git a/docs/adapters/vercel-gateway.md b/docs/adapters/vercel-gateway.md index f199a90ece..2c89100ab9 100644 --- a/docs/adapters/vercel-gateway.md +++ b/docs/adapters/vercel-gateway.md @@ -270,6 +270,27 @@ console.log(result.refund.value); // true See the [Evaluate guide](../evaluate/evaluate) for the question helpers, the result shape, abort, and middleware. +When the Gateway sends an error response, the thrown error has the HTTP status code in `status`. A rate limit (429) needs a retry, but a bad key (401) does not: + +```typescript +import { decide, boolean } from "@tanstack/ai"; +import { vercelGatewayDecider } from "@tanstack/ai-vercel-gateway"; + +try { + await decide({ + adapter: vercelGatewayDecider("typesafe-ai/jev"), + state: { body: "Please refund the extra payment." }, + questions: { + refund: boolean({ instructions: "Is the customer asking for a refund?" }), + }, + }); +} catch (error) { + if (error instanceof Error && "status" in error && error.status === 429) { + // Rate limited. Wait, then call decide() again. + } +} +``` + ## Not in this package This package does not do video, speech, transcription, or rerank. Use a dedicated adapter for these. diff --git a/packages/ai-ollaya/src/adapters/evaluate.ts b/packages/ai-ollaya/src/adapters/evaluate.ts index 01e8715a39..f769cbab17 100644 --- a/packages/ai-ollaya/src/adapters/evaluate.ts +++ b/packages/ai-ollaya/src/adapters/evaluate.ts @@ -61,6 +61,9 @@ export class OllayaEvaluateAdapter< /** * POST `{ model, state, questions }` to Ollaya and return wire answers plus * mapped token usage. Does not invent unified `.value` fields. + * + * @throws An `Error` with a numeric `status` (the HTTP status code) when + * Ollaya answers with a non-OK response. */ async evaluate(options: EvaluateOptions) { const { model, state, questions, abortSignal, logger } = options @@ -95,10 +98,13 @@ export class OllayaEvaluateAdapter< if (!response.ok) { const detail = await response.text().catch(() => '') - const error = new Error( - `Ollaya evaluate request failed: ${response.status} ${response.statusText}${ - detail ? ` — ${detail}` : '' - }`, + const error = Object.assign( + new Error( + `Ollaya evaluate request failed: ${response.status} ${response.statusText}${ + detail ? ` — ${detail}` : '' + }`, + ), + { status: response.status }, ) logger.errors(`${this.name}.evaluate fatal`, { error, diff --git a/packages/ai-ollaya/tests/evaluate-adapter.test.ts b/packages/ai-ollaya/tests/evaluate-adapter.test.ts index 31c72befa9..75ca848b34 100644 --- a/packages/ai-ollaya/tests/evaluate-adapter.test.ts +++ b/packages/ai-ollaya/tests/evaluate-adapter.test.ts @@ -171,7 +171,7 @@ describe('OllayaEvaluateAdapter', () => { ).toBe('Bearer proxy-token') }) - it('throws with the response body detail on a non-OK status', async () => { + it('throws with status and the response body detail on a non-OK status', async () => { fetchMock.mockResolvedValue( new Response('model not found', { status: 404, @@ -181,7 +181,12 @@ describe('OllayaEvaluateAdapter', () => { await expect( ollayaDecider('laya:latest').evaluate(evaluateOptions()), - ).rejects.toThrow('404') + ).rejects.toMatchObject({ + status: 404, + message: expect.stringMatching( + /^Ollaya evaluate request failed: 404 Not Found .+ model not found$/, + ), + }) }) it('throws when the response shape is unexpected', async () => { diff --git a/packages/ai-openrouter/src/adapters/evaluate.ts b/packages/ai-openrouter/src/adapters/evaluate.ts index 6183fe056f..4b61b321cf 100644 --- a/packages/ai-openrouter/src/adapters/evaluate.ts +++ b/packages/ai-openrouter/src/adapters/evaluate.ts @@ -151,10 +151,13 @@ export class OpenRouterEvaluateAdapter< if (!response.ok) { const detail = await response.text().catch(() => '') - throw new Error( - `OpenRouter evaluate request failed: ${response.status} ${response.statusText}${ - detail ? ` — ${detail}` : '' - }`, + throw Object.assign( + new Error( + `OpenRouter evaluate request failed: ${response.status} ${response.statusText}${ + detail ? ` — ${detail}` : '' + }`, + ), + { status: response.status }, ) } diff --git a/packages/ai-openrouter/tests/evaluate-adapter.test.ts b/packages/ai-openrouter/tests/evaluate-adapter.test.ts index 4a7b2238d0..a3fbafdc5a 100644 --- a/packages/ai-openrouter/tests/evaluate-adapter.test.ts +++ b/packages/ai-openrouter/tests/evaluate-adapter.test.ts @@ -237,7 +237,7 @@ describe('OpenRouterEvaluateAdapter', () => { }) }) - it('throws on a non-OK response', async () => { + it('throws with status on a non-OK response', async () => { fetchMock.mockResolvedValue( new Response('bad request', { status: 400, statusText: 'Bad Request' }), ) @@ -253,6 +253,11 @@ describe('OpenRouterEvaluateAdapter', () => { }), }, }), - ).rejects.toThrow() + ).rejects.toMatchObject({ + status: 400, + message: expect.stringMatching( + /^OpenRouter evaluate request failed: 400 Bad Request .+ bad request$/, + ), + }) }) }) diff --git a/packages/ai-typesafe/src/adapters/evaluate.ts b/packages/ai-typesafe/src/adapters/evaluate.ts index 1e2ce9284d..6fa805486d 100644 --- a/packages/ai-typesafe/src/adapters/evaluate.ts +++ b/packages/ai-typesafe/src/adapters/evaluate.ts @@ -63,6 +63,9 @@ export class TypesafeEvaluateAdapter< /** * POST `{ model, state, questions }` to TypeSafe and return wire answers * plus mapped token usage. Does not invent unified `.value` fields. + * + * @throws An `Error` with a numeric `status` (the HTTP status code) when + * TypeSafe answers with a non-OK response. */ async evaluate(options: EvaluateOptions) { const { model, state, questions, abortSignal, logger } = options @@ -96,10 +99,13 @@ export class TypesafeEvaluateAdapter< if (!response.ok) { const detail = await response.text().catch(() => '') - const error = new Error( - `TypeSafe evaluate request failed: ${response.status} ${response.statusText}${ - detail ? ` — ${detail}` : '' - }`, + const error = Object.assign( + new Error( + `TypeSafe evaluate request failed: ${response.status} ${response.statusText}${ + detail ? ` — ${detail}` : '' + }`, + ), + { status: response.status }, ) logger.errors(`${this.name}.evaluate fatal`, { error, diff --git a/packages/ai-typesafe/tests/evaluate-adapter.test.ts b/packages/ai-typesafe/tests/evaluate-adapter.test.ts index f5477d9e58..33c0b410dd 100644 --- a/packages/ai-typesafe/tests/evaluate-adapter.test.ts +++ b/packages/ai-typesafe/tests/evaluate-adapter.test.ts @@ -137,22 +137,27 @@ describe('TypesafeEvaluateAdapter', () => { }) }) - it('throws on 401', async () => { + it('throws with status on 401', async () => { await expectStatus(401, 'Unauthorized', 'invalid api key') }) - it('throws on 422', async () => { + it('throws with status on 422', async () => { await expectStatus(422, 'Unprocessable Entity', 'malformed question') }) - it('throws on 429', async () => { + it('throws with status on 429', async () => { await expectStatus(429, 'Too Many Requests', 'rate limited') }) }) async function expectStatus(status: number, statusText: string, body: string) { fetchMock.mockResolvedValue(new Response(body, { status, statusText })) - await expect(adapter().evaluate(evaluateOptions())).rejects.toThrow( - String(status), - ) + await expect(adapter().evaluate(evaluateOptions())).rejects.toMatchObject({ + status, + message: expect.stringMatching( + new RegExp( + `^TypeSafe evaluate request failed: ${status} ${statusText} .+ ${body}$`, + ), + ), + }) } diff --git a/packages/ai-vercel-gateway/src/adapters/evaluate.ts b/packages/ai-vercel-gateway/src/adapters/evaluate.ts index 537ff486f7..dafdca6bfd 100644 --- a/packages/ai-vercel-gateway/src/adapters/evaluate.ts +++ b/packages/ai-vercel-gateway/src/adapters/evaluate.ts @@ -302,10 +302,13 @@ export class VercelGatewayEvaluateAdapter< if (!response.ok) { const detail = await response.text().catch(() => '') - throw new Error( - `Vercel Gateway evaluate request failed: ${response.status} ${response.statusText}${ - detail ? ` — ${detail}` : '' - }`, + throw Object.assign( + new Error( + `Vercel Gateway evaluate request failed: ${response.status} ${response.statusText}${ + detail ? ` — ${detail}` : '' + }`, + ), + { status: response.status }, ) } diff --git a/packages/ai-vercel-gateway/tests/evaluate-adapter.test.ts b/packages/ai-vercel-gateway/tests/evaluate-adapter.test.ts index 1105b5fb4a..bac7a8b525 100644 --- a/packages/ai-vercel-gateway/tests/evaluate-adapter.test.ts +++ b/packages/ai-vercel-gateway/tests/evaluate-adapter.test.ts @@ -147,7 +147,7 @@ it('POSTs to /v4/ai/evaluation-model and maps boolean/choice/score answers', asy }) }) -it('throws when the evaluate request is not OK', async () => { +it('throws with status when the evaluate request is not OK', async () => { fetchMock.mockResolvedValue( new Response('gateway down', { status: 503, @@ -157,7 +157,9 @@ it('throws when the evaluate request is not OK', async () => { const adapter = createVercelGatewayDecider('typesafe-ai/jev', 'gw_test_key') - await expect(adapter.evaluate(evaluateOptions())).rejects.toThrow( - 'Vercel Gateway evaluate request failed: 503 Service Unavailable — gateway down', - ) + await expect(adapter.evaluate(evaluateOptions())).rejects.toMatchObject({ + status: 503, + message: + 'Vercel Gateway evaluate request failed: 503 Service Unavailable — gateway down', + }) }) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5ad881287d..eae2381e7a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -4049,6 +4049,9 @@ importers: '@tanstack/ai-solid': specifier: workspace:* version: link:../../packages/ai-solid + '@tanstack/ai-typesafe': + specifier: workspace:* + version: link:../../packages/ai-typesafe '@tanstack/ai-vercel-gateway': specifier: workspace:* version: link:../../packages/ai-vercel-gateway diff --git a/testing/e2e/global-setup.ts b/testing/e2e/global-setup.ts index a6ad6292e3..b0d59d4706 100644 --- a/testing/e2e/global-setup.ts +++ b/testing/e2e/global-setup.ts @@ -181,6 +181,13 @@ export default async function globalSetup() { // asserts `decide()` rejects and still reports that usage. mock.mount('/evaluate-usage-on-failure', evaluateUsageOnFailureMount()) + // Failed evaluate requests. TypeSafe takes a path prefix, but the Vercel + // Gateway adapter keeps only the origin of its baseURL, so that mount sits on + // the real evaluate path. Both answer only requests carrying the + // `x-evaluate-http-status` header and fall through otherwise. + mock.mount('/evaluate-http-status', evaluateHttpStatusMount()) + mock.mount('/v4/ai/evaluation-model', evaluateHttpStatusMount()) + // BytePlus. Ark (chat, Seedream image, Seedance video) serves its whole data // plane under /api/v3, and Seed Speech (TTS/ASR) mounts its own endpoints at // the same prefix on a different host — which collapses onto one prefix here @@ -2226,6 +2233,29 @@ function evaluateUsageOnFailureMount(): Mountable { } } +/** + * Answers an evaluate request with the HTTP status named in its + * `x-evaluate-http-status` header and a small JSON error body. + */ +function evaluateHttpStatusMount(): Mountable { + return { + async handleRequest( + req: http.IncomingMessage, + res: http.ServerResponse, + ): Promise { + const status = Number(req.headers['x-evaluate-http-status']) + if (req.method !== 'POST' || !(status >= 400 && status <= 599)) { + return false + } + await drainBody(req) + res.statusCode = status + res.setHeader('Content-Type', 'application/json') + res.end(JSON.stringify({ error: 'rejected' })) + return true + }, + } +} + function buildToolPlusServerToolEvents(): Array> { const messageId = 'msg_bug_604' const model = 'claude-sonnet-4-5' diff --git a/testing/e2e/package.json b/testing/e2e/package.json index f45f96e0f2..0e0124a173 100644 --- a/testing/e2e/package.json +++ b/testing/e2e/package.json @@ -52,6 +52,7 @@ "@tanstack/ai-sandbox-local-process": "workspace:*", "@tanstack/ai-skills": "workspace:*", "@tanstack/ai-solid": "workspace:*", + "@tanstack/ai-typesafe": "workspace:*", "@tanstack/ai-vercel-gateway": "workspace:*", "@tanstack/ai-vertex": "workspace:*", "@tanstack/ai-worldlabs": "workspace:*", diff --git a/testing/e2e/src/routeTree.gen.ts b/testing/e2e/src/routeTree.gen.ts index e5ec31ba17..8f1b55a77b 100644 --- a/testing/e2e/src/routeTree.gen.ts +++ b/testing/e2e/src/routeTree.gen.ts @@ -90,6 +90,7 @@ import { Route as ApiDevtoolsMemoryRouteImport } from './routes/api.devtools-mem import { Route as ApiDurableDeliveryRouteImport } from './routes/api.durable-delivery' import { Route as ApiDurableTakeoverRouteImport } from './routes/api.durable-takeover' import { Route as ApiEmbeddingRouteImport } from './routes/api.embedding' +import { Route as ApiEvaluateHttpStatusRouteImport } from './routes/api.evaluate-http-status' import { Route as ApiEvaluateUsageOnFailureRouteImport } from './routes/api.evaluate-usage-on-failure' import { Route as ApiFakeTextRouteImport } from './routes/api.fake-text' import { Route as ApiFileSourceWireRouteImport } from './routes/api.file-source-wire' @@ -629,6 +630,11 @@ const ApiEmbeddingRoute = ApiEmbeddingRouteImport.update({ path: '/api/embedding', getParentRoute: () => rootRouteImport, } as any) +const ApiEvaluateHttpStatusRoute = ApiEvaluateHttpStatusRouteImport.update({ + id: '/api/evaluate-http-status', + path: '/api/evaluate-http-status', + getParentRoute: () => rootRouteImport, +} as any) const ApiEvaluateUsageOnFailureRoute = ApiEvaluateUsageOnFailureRouteImport.update({ id: '/api/evaluate-usage-on-failure', @@ -1309,6 +1315,7 @@ export interface FileRoutesByFullPath { '/api/durable-delivery': typeof ApiDurableDeliveryRoute '/api/durable-takeover': typeof ApiDurableTakeoverRoute '/api/embedding': typeof ApiEmbeddingRoute + '/api/evaluate-http-status': typeof ApiEvaluateHttpStatusRoute '/api/evaluate-usage-on-failure': typeof ApiEvaluateUsageOnFailureRoute '/api/fake-text': typeof ApiFakeTextRoute '/api/file-source-wire': typeof ApiFileSourceWireRoute @@ -1506,6 +1513,7 @@ export interface FileRoutesByTo { '/api/durable-delivery': typeof ApiDurableDeliveryRoute '/api/durable-takeover': typeof ApiDurableTakeoverRoute '/api/embedding': typeof ApiEmbeddingRoute + '/api/evaluate-http-status': typeof ApiEvaluateHttpStatusRoute '/api/evaluate-usage-on-failure': typeof ApiEvaluateUsageOnFailureRoute '/api/fake-text': typeof ApiFakeTextRoute '/api/file-source-wire': typeof ApiFileSourceWireRoute @@ -1704,6 +1712,7 @@ export interface FileRoutesById { '/api/durable-delivery': typeof ApiDurableDeliveryRoute '/api/durable-takeover': typeof ApiDurableTakeoverRoute '/api/embedding': typeof ApiEmbeddingRoute + '/api/evaluate-http-status': typeof ApiEvaluateHttpStatusRoute '/api/evaluate-usage-on-failure': typeof ApiEvaluateUsageOnFailureRoute '/api/fake-text': typeof ApiFakeTextRoute '/api/file-source-wire': typeof ApiFileSourceWireRoute @@ -1903,6 +1912,7 @@ export interface FileRouteTypes { | '/api/durable-delivery' | '/api/durable-takeover' | '/api/embedding' + | '/api/evaluate-http-status' | '/api/evaluate-usage-on-failure' | '/api/fake-text' | '/api/file-source-wire' @@ -2100,6 +2110,7 @@ export interface FileRouteTypes { | '/api/durable-delivery' | '/api/durable-takeover' | '/api/embedding' + | '/api/evaluate-http-status' | '/api/evaluate-usage-on-failure' | '/api/fake-text' | '/api/file-source-wire' @@ -2297,6 +2308,7 @@ export interface FileRouteTypes { | '/api/durable-delivery' | '/api/durable-takeover' | '/api/embedding' + | '/api/evaluate-http-status' | '/api/evaluate-usage-on-failure' | '/api/fake-text' | '/api/file-source-wire' @@ -2495,6 +2507,7 @@ export interface RootRouteChildren { ApiDurableDeliveryRoute: typeof ApiDurableDeliveryRoute ApiDurableTakeoverRoute: typeof ApiDurableTakeoverRoute ApiEmbeddingRoute: typeof ApiEmbeddingRoute + ApiEvaluateHttpStatusRoute: typeof ApiEvaluateHttpStatusRoute ApiEvaluateUsageOnFailureRoute: typeof ApiEvaluateUsageOnFailureRoute ApiFakeTextRoute: typeof ApiFakeTextRoute ApiFileSourceWireRoute: typeof ApiFileSourceWireRoute @@ -3176,6 +3189,13 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof ApiEmbeddingRouteImport parentRoute: typeof rootRouteImport } + '/api/evaluate-http-status': { + id: '/api/evaluate-http-status' + path: '/api/evaluate-http-status' + fullPath: '/api/evaluate-http-status' + preLoaderRoute: typeof ApiEvaluateHttpStatusRouteImport + parentRoute: typeof rootRouteImport + } '/api/evaluate-usage-on-failure': { id: '/api/evaluate-usage-on-failure' path: '/api/evaluate-usage-on-failure' @@ -4121,6 +4141,7 @@ const rootRouteChildren: RootRouteChildren = { ApiDurableDeliveryRoute: ApiDurableDeliveryRoute, ApiDurableTakeoverRoute: ApiDurableTakeoverRoute, ApiEmbeddingRoute: ApiEmbeddingRoute, + ApiEvaluateHttpStatusRoute: ApiEvaluateHttpStatusRoute, ApiEvaluateUsageOnFailureRoute: ApiEvaluateUsageOnFailureRoute, ApiFakeTextRoute: ApiFakeTextRoute, ApiFileSourceWireRoute: ApiFileSourceWireRoute, diff --git a/testing/e2e/src/routes/api.evaluate-http-status.ts b/testing/e2e/src/routes/api.evaluate-http-status.ts new file mode 100644 index 0000000000..19817c1726 --- /dev/null +++ b/testing/e2e/src/routes/api.evaluate-http-status.ts @@ -0,0 +1,71 @@ +import { createFileRoute } from '@tanstack/react-router' +import { boolean, decide } from '@tanstack/ai' +import { createTypesafeDecider } from '@tanstack/ai-typesafe' +import { createVercelGatewayDecider } from '@tanstack/ai-vercel-gateway' + +const LLMOCK_DEFAULT_BASE = process.env.LLMOCK_URL || 'http://127.0.0.1:4010' +const DUMMY_KEY = 'sk-e2e-test-dummy-key' + +const STATE = 'The checkout page has been down for an hour.' +const QUESTIONS = { + isUrgent: boolean({ instructions: 'Does this message convey urgency?' }), +} + +export const Route = createFileRoute('/api/evaluate-http-status')({ + server: { + handlers: { + POST: async ({ request }) => { + const params = new URL(request.url).searchParams + const provider = params.get('provider') + const defaultHeaders = { + 'x-evaluate-http-status': params.get('status') ?? '', + } + + try { + if (provider === 'typesafe') { + await decide({ + adapter: createTypesafeDecider('jev-latest', DUMMY_KEY, { + baseURL: `${LLMOCK_DEFAULT_BASE}/evaluate-http-status`, + defaultHeaders, + }), + state: STATE, + questions: QUESTIONS, + }) + } else if (provider === 'vercel-gateway') { + await decide({ + adapter: createVercelGatewayDecider( + 'typesafe-ai/jev', + DUMMY_KEY, + { + baseURL: LLMOCK_DEFAULT_BASE, + defaultHeaders, + }, + ), + state: STATE, + questions: QUESTIONS, + }) + } else { + return Response.json( + { ok: false, error: { message: 'Unsupported provider' } }, + { status: 400 }, + ) + } + } catch (error) { + return Response.json({ + ok: false, + error: + error instanceof Error + ? { + name: error.name, + message: error.message, + status: 'status' in error ? error.status : undefined, + } + : { message: String(error) }, + }) + } + + return Response.json({ ok: true }) + }, + }, + }, +}) diff --git a/testing/e2e/tests/evaluate-http-status.spec.ts b/testing/e2e/tests/evaluate-http-status.spec.ts new file mode 100644 index 0000000000..24f9e1dd6d --- /dev/null +++ b/testing/e2e/tests/evaluate-http-status.spec.ts @@ -0,0 +1,38 @@ +import { test, expect } from './fixtures' + +const CASES = [ + { + provider: 'typesafe', + status: 429, + message: + 'TypeSafe evaluate request failed: 429 Too Many Requests \u2014 {"error":"rejected"}', + }, + { + provider: 'vercel-gateway', + status: 503, + message: + 'Vercel Gateway evaluate request failed: 503 Service Unavailable \u2014 {"error":"rejected"}', + }, +] as const + +test.describe('decide() on a non-ok provider response', () => { + for (const { provider, status, message } of CASES) { + test(`${provider} rejects with the HTTP status on the error`, async ({ + request, + }) => { + const response = await request.post( + `/api/evaluate-http-status?provider=${provider}&status=${status}`, + ) + expect(response.ok()).toBe(true) + + const result = (await response.json()) as { + ok: boolean + error?: { name?: string; message: string; status?: number } + } + expect(result).toEqual({ + ok: false, + error: { name: 'Error', message, status }, + }) + }) + } +})