diff --git a/.agent/domain-map.json b/.agent/domain-map.json index db20795f76..b09f64e809 100644 --- a/.agent/domain-map.json +++ b/.agent/domain-map.json @@ -25,6 +25,7 @@ "packages/ai-llmgateway", "packages/ai-lovable", "packages/ai-mistral", + "packages/ai-neon", "packages/ai-ollaya", "packages/ai-openrouter", "packages/ai-perplexity", diff --git a/.changeset/neon-adapter.md b/.changeset/neon-adapter.md new file mode 100644 index 0000000000..26c914c98f --- /dev/null +++ b/.changeset/neon-adapter.md @@ -0,0 +1,7 @@ +--- +'@tanstack/ai': patch +'@tanstack/ai-neon': minor +'@tanstack/openai-base': patch +--- + +Add `@tanstack/ai-neon` for the Neon AI Gateway: `neonText` sends Claude to the Anthropic Messages route, codex and `*-pro` models to the Responses route, and every other model to Chat Completions, with `neonEmbedding` and `neonSummarize`. `@tanstack/openai-base` gets a `streamOptions(model)` hook, so a Chat Completions backend that rejects `stream_options` can leave it out. diff --git a/.github/workflows/sync-models.yml b/.github/workflows/sync-models.yml index e67073cbd3..1b45a811ac 100644 --- a/.github/workflows/sync-models.yml +++ b/.github/workflows/sync-models.yml @@ -67,7 +67,7 @@ jobs: run: | git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" - git add packages/ scripts/openrouter.models.json scripts/openrouter.video-models.json scripts/vercel-gateway.models.json scripts/lovable-gateway.models.json scripts/llmgateway.models.json scripts/.sync-models-last-run .changeset/ + git add packages/ scripts/openrouter.models.json scripts/openrouter.video-models.json scripts/vercel-gateway.models.json scripts/lovable-gateway.models.json scripts/llmgateway.models.json scripts/neon.models.json scripts/.sync-models-last-run .changeset/ git commit -m "chore: sync model metadata" # GITHUB_TOKEN pushes do not start the PR Test / E2E workflows. # After this job, a maintainer must run those checks from the @@ -87,12 +87,13 @@ jobs: if [ -z "$EXISTING_PR" ] || [ "$EXISTING_PR" = "null" ]; then BODY=$(cat <<'PRBODY' - Automated daily sync of model metadata from OpenRouter, Vercel AI Gateway, Lovable AI Gateway, LLM Gateway, and modelschemas (ElevenLabs text-to-speech, music, sound effects, and voice design). + Automated daily sync of model metadata from OpenRouter, Vercel AI Gateway, Lovable AI Gateway, LLM Gateway, Neon AI Gateway, and modelschemas (ElevenLabs text-to-speech, music, sound effects, and voice design). - Fetches the latest model list from OpenRouter (chat + `GET /api/v1/videos/models`) - Fetches the latest model list from Vercel AI Gateway - Fetches the latest model list from Lovable AI Gateway - Fetches the latest model list from LLM Gateway (`GET /v1/models`) + - Fetches the latest model list from Neon AI Gateway - Converts to the internal adapter format - Syncs provider-specific model metadata for affected packages - Inserts new ElevenLabs text-to-speech, music, sound-effect, and voice-design models from modelschemas diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 10e7f5a23e..a0c9e34a46 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -53,8 +53,8 @@ For deeper architecture details (adapter system, isomorphic tools, framework int `pnpm generate:models` is the maintainer command behind the daily **Sync Model Metadata** workflow (branch `automated/sync-models`). It: -1. Fetches OpenRouter, Vercel AI Gateway, and Lovable AI Gateway catalogs. -2. Regenerates `packages/ai-openrouter/src/model-meta.ts` and the Vercel Gateway model list. +1. Fetches OpenRouter, Vercel AI Gateway, Lovable AI Gateway, and Neon AI Gateway catalogs. +2. Regenerates `packages/ai-openrouter/src/model-meta.ts` and the Vercel Gateway and Neon model lists. 3. Inserts **new** native-provider models into `packages/ai-openai`, `ai-anthropic`, `ai-gemini`, and `ai-grok`. 4. Inserts **new** ElevenLabs text-to-speech, music, sound-effect, and voice-design ids from [modelschemas](https://modelschemas.com/v1/schemas/elevenlabs) into `packages/ai-elevenlabs/src/model-meta.ts`. Transcription ids stay hand-maintained. 5. Writes a patch changeset for the packages that changed. diff --git a/README.md b/README.md index 40f036e06e..2838653dd1 100644 --- a/README.md +++ b/README.md @@ -244,6 +244,7 @@ Official adapters include: | [`@tanstack/ai-reactor`](https://tanstack.com/ai/latest/docs/adapters/reactor) | Reactor live world and video generation (Orbis, Happy Oyster, LingBot, Helios, FastH3) | | [`@tanstack/ai-worldlabs`](https://tanstack.com/ai/latest/docs/adapters/worldlabs) | World Labs Marble persistent 3D world generation | | [`@tanstack/ai-cloudflare`](https://tanstack.com/ai/latest/docs/adapters/cloudflare) | Cloudflare Workers AI chat, embeddings, image, speech, transcription, and AI Gateway | +| [`@tanstack/ai-neon`](https://tanstack.com/ai/latest/docs/adapters/neon) | Neon AI Gateway chat and embeddings for Claude, GPT, Gemini, and open models | The adapter system is tree-shakeable by activity. Import `openaiText` for chat, `openaiImage` for images, `falVideo` for video, `geminiSpeech` for TTS, and so diff --git a/docs/adapters/neon.md b/docs/adapters/neon.md new file mode 100644 index 0000000000..1a037a15fe --- /dev/null +++ b/docs/adapters/neon.md @@ -0,0 +1,246 @@ +--- +title: Neon AI Gateway +id: neon-adapter +description: "Call Claude, GPT, Gemini, and open models with one Neon credential through the Neon AI Gateway. Chat, embeddings, and summarize work with @tanstack/ai-neon." +keywords: + - tanstack ai + - neon + - ai gateway + - adapter + - claude + - gemini + - embeddings + - pgvector +--- + +You want Claude, GPT, and Gemini in your app, and you do not want an account and a key for each provider. The Neon AI Gateway gives your Neon branch one credential for every model in its catalog. + +Install `@tanstack/ai-neon`. Then call `neonText` with a model id such as `claude-opus-5-5`. + +## Installation + + + +react: @tanstack/ai-neon +vue: @tanstack/ai-neon +solid: @tanstack/ai-neon +svelte: @tanstack/ai-neon +preact: @tanstack/ai-neon +angular: @tanstack/ai-neon +vanilla: @tanstack/ai-neon +octane: @tanstack/ai-neon + + + +## Auth + +The adapter reads two variables: + +- `NEON_AI_GATEWAY_TOKEN`: the Neon credential (`nt_live_...`). +- `NEON_AI_GATEWAY_BASE_URL`: the gateway host of your branch, with no path. + +`neon env pull` writes both to `.env`. A deployed Neon Function gets both automatically. + +```bash +NEON_AI_GATEWAY_TOKEN=nt_live_... +NEON_AI_GATEWAY_BASE_URL=https://br-winter-pond-aptw82ef-api.ai.c-2.us-east-2.aws.neon.tech +``` + +You can also pass the token to a `create*` factory. `baseURL` is the same bare host: + +```typescript +import { createNeonText } from "@tanstack/ai-neon" + +const adapter = createNeonText("claude-opus-5-5", process.env.NEON_AI_GATEWAY_TOKEN!, { + baseURL: process.env.NEON_AI_GATEWAY_BASE_URL, +}) +``` + +## Chat + +Model ids are the catalog ids, with no provider prefix. + +**Server.** An endpoint that streams the reply over SSE: + +```typescript +import { + chat, + chatParamsFromRequest, + toServerSentEventsResponse, +} from "@tanstack/ai" +import { neonText } from "@tanstack/ai-neon" + +export async function POST(request: Request) { + const { messages, threadId, runId } = await chatParamsFromRequest(request) + + const stream = chat({ + adapter: neonText("claude-opus-5-5"), + messages, + threadId, + runId, + }) + + return toServerSentEventsResponse(stream) +} +``` + +**Client.** The same `useChat` hook as every other provider: + +```tsx +import { useState } from "react" +import { fetchServerSentEvents, useChat } from "@tanstack/ai-react" + +export function Chat() { + const [input, setInput] = useState("") + + const { messages, sendMessage, isLoading } = useChat({ + connection: fetchServerSentEvents("/api/chat"), + threadId: "support-chat", + }) + + return ( +
+ {messages.map((message) => ( +
+ {message.role} + {message.parts.map((part, index) => + part.type === "text" ?

{part.content}

: null, + )} +
+ ))} + +
{ + event.preventDefault() + if (!input.trim() || isLoading) return + sendMessage(input) + setInput("") + }} + > + setInput(event.target.value)} /> + +
+
+ ) +} +``` + +To change the model, change the id. The rest of the code stays the same. + +## Which endpoint each model uses + +The gateway has one endpoint per API style. `neonText` sends each model to the endpoint that serves it best: + +| Models | Endpoint | What you get | +| --- | --- | --- | +| Claude (`claude-*`) | `/anthropic/v1/messages` | The [Anthropic adapter](./anthropic), with thinking budgets and prompt caching | +| `gpt-5-3-codex`, `gpt-5-5-pro` | `/openai/v1/responses` | The OpenAI Responses API, the only API that serves these models | +| All other models | `/v1/chat/completions` | Chat Completions, which serves every model in the catalog | + +The types follow the endpoint. For example, a Claude model takes the Anthropic model options. + +Chat Completions accepts text and images through this adapter. For documents, use Claude or a GPT model with `api: "responses"`. Audio and video inputs are not supported. + +Other GPT models can also use the Responses API. Pass `{ api: "responses" }`: + +```typescript +import { chat } from "@tanstack/ai" +import { neonText } from "@tanstack/ai-neon" + +const stream = chat({ + adapter: neonText("gpt-6-1-sol", { api: "responses" }), + messages: [{ role: "user", content: "Hello" }], + threadId: "thread-1", +}) +``` + +`NEON_RESPONSES_MODELS` lists the models that take `api: "responses"`. TypeScript rejects it for any other model. + +Need a header on each request, or a log of each model call? The chat adapters on this page support [`wrapFetch`](../advanced/middleware#change-the-http-requests-of-a-call). + +## Reasoning + +Set `reasoning` on `chat()`. The types take the levels of each model: + +```typescript +import { chat } from "@tanstack/ai" +import { neonText } from "@tanstack/ai-neon" + +const stream = chat({ + adapter: neonText("gemini-3-8-flash"), + messages: [{ role: "user", content: "Plan a database migration." }], + threadId: "thread-1", + reasoning: "high", +}) +``` + +- Chat Completions sends `reasoning_effort`. Responses sends `reasoning.effort`. Neither takes `budgetTokens`. +- Claude takes the same levels as on [Anthropic](./anthropic#reasoning), and older Claude models also take `budgetTokens`. + +## Embeddings + +Embed text with the same credential. Then keep the vectors in your branch, next to the rows they describe: + +```typescript +import { embed } from "@tanstack/ai" +import { neonEmbedding } from "@tanstack/ai-neon" + +const result = await embed({ + adapter: neonEmbedding("qwen3-embedding-0-6b"), + input: ["Neon branches a database like code."], +}) + +console.log(result.embeddings[0]?.vector.length) // 1024 +``` + +All three embedding models (`qwen3-embedding-0-6b`, `gte-large-en`, `bge-large-en`) return 1024 numbers. A pgvector column of the same size stores them: + +```sql +CREATE EXTENSION IF NOT EXISTS vector; +CREATE TABLE documents (id serial PRIMARY KEY, body text, embedding vector(1024)); +``` + +## Summarize + +```typescript +import { summarize } from "@tanstack/ai" +import { neonSummarize } from "@tanstack/ai-neon" + +const result = await summarize({ + adapter: neonSummarize("gpt-6-1-sol"), + threadId: "thread-1", + text: "The Fender Stratocaster is a versatile electric guitar.", + maxLength: 60, +}) +``` + +Summarize uses the same endpoint and route options as `neonText` for the model. With `api: "responses"`, `modelOptions` accepts Responses options such as `max_output_tokens`. + +## Models + +Pass a model id from the catalog. TypeScript rejects an id that is not on a list. + +- `NEON_CHAT_MODELS`: every chat model. +- `NEON_ANTHROPIC_MODELS`: the Claude models that go to the Anthropic endpoint. +- `NEON_RESPONSES_MODELS`: the models that take `api: "responses"`. +- `NEON_EMBEDDING_MODELS`: the embedding models. + +The lists come from the [Neon model catalog](https://neon.com/models). Your account can have a smaller set. To see the models your branch can call, ask the gateway: + +```bash +curl "$NEON_AI_GATEWAY_BASE_URL/v1/models" \ + -H "Authorization: Bearer $NEON_AI_GATEWAY_TOKEN" +``` + +A model with `"enabled": false` in that list is in the catalog, but your account cannot call it yet. + +## Errors + +- `401`: the token is missing, wrong, or revoked. +- `403 credential not authorized for ai gateway`: the credential has no `ai_gateway:invoke` scope. Make a new credential with that scope. +- `403 model requires a verified account`: request access to the model on the AI Gateway page of your branch in the Neon Console. +- `429`: your account quota or the upstream provider limit is reached. The response has a `Retry-After` header. + +The gateway needs a paid Neon plan and a project in a supported region. See [Neon AI Gateway](https://neon.com/docs/ai-gateway/overview) for regions, pricing, and limits. diff --git a/docs/adapters/openai-compatible.md b/docs/adapters/openai-compatible.md index 0f73318eed..0cab3cc6d6 100644 --- a/docs/adapters/openai-compatible.md +++ b/docs/adapters/openai-compatible.md @@ -28,6 +28,7 @@ If a dedicated adapter exists, use it, because it has curated metadata for each - [OpenAI](./openai), [Grok](./grok), [Groq](./groq), and [OpenRouter](./openrouter). - Vercel AI Gateway: `vercelGatewayText` from `@tanstack/ai-vercel-gateway`. See [Vercel AI Gateway](./vercel-gateway.md). - Lovable AI Gateway: `lovableText` (plus image, video, embeddings, and speech factories) from `@tanstack/ai-lovable`. See [Lovable AI Gateway](./lovable.md). +- Neon AI Gateway: `neonText` from `@tanstack/ai-neon`. See [Neon AI Gateway](./neon.md). Perplexity Sonar chat uses this adapter. [`@tanstack/ai-perplexity`](./perplexity.md) is for search and grounding only. To send the `X-Pplx-Integration` attribution header of Perplexity, pass `defaultHeaders: getPerplexityIntegrationHeaders()` from that package. diff --git a/docs/chat/reasoning.md b/docs/chat/reasoning.md index ae6ad21efc..a5ba2f0b83 100644 --- a/docs/chat/reasoning.md +++ b/docs/chat/reasoning.md @@ -169,9 +169,9 @@ const lowEffort: ChatMiddleware = { | Adapter | Sent as | | --- | --- | -| OpenAI, Azure OpenAI, Grok, Bedrock Responses | `reasoning: { effort, summary }` | -| OpenAI Chat Completions, Groq, LLM Gateway, Lovable, Bedrock chat, Cloudflare | `reasoning_effort` | -| Anthropic | adaptive thinking with the effort, or a thinking budget | +| OpenAI, Azure OpenAI, Grok, Bedrock Responses, Neon Responses | `reasoning: { effort, summary }` | +| OpenAI Chat Completions, Groq, LLM Gateway, Lovable, Bedrock chat, Cloudflare, Neon Chat Completions | `reasoning_effort` | +| Anthropic, Neon Claude | adaptive thinking with the effort, or a thinking budget | | Gemini | `thinkingConfig.thinkingLevel` or `thinkingConfig.thinkingBudget` | | OpenRouter, Vercel AI Gateway | the gateway's `reasoning` object | | Mistral | `reasoning_effort` or `prompt_mode` | diff --git a/docs/config.json b/docs/config.json index 5d4255995c..359b32483f 100644 --- a/docs/config.json +++ b/docs/config.json @@ -1829,6 +1829,11 @@ "addedAt": "2026-09-03", "updatedAt": "2026-10-22" }, + { + "label": "Neon AI Gateway", + "to": "adapters/neon", + "addedAt": "2026-10-21" + }, { "label": "Claude Code", "to": "adapters/claude-code", diff --git a/docs/embeddings.md b/docs/embeddings.md index 011769c831..9ac3e8a916 100644 --- a/docs/embeddings.md +++ b/docs/embeddings.md @@ -2,7 +2,7 @@ title: Embeddings id: embeddings order: 1 -description: "Turn text and images into vectors with embed(). Use them for semantic search, RAG, and clustering with OpenAI, Cohere, Gemini, Mistral, Amazon Bedrock, Ollama, or Vercel AI Gateway." +description: "Turn text and images into vectors with embed(). Use them for semantic search, RAG, and clustering with OpenAI, Cohere, Gemini, Mistral, Amazon Bedrock, Ollama, Vercel AI Gateway, or Neon AI Gateway." keywords: - tanstack ai - embeddings @@ -18,6 +18,7 @@ keywords: - bedrock - ollama - vercel ai gateway + - neon ai gateway --- `embed()` turns text into a vector. Search, RAG, and clustering then compare those vectors. @@ -83,6 +84,7 @@ You now have one `number[]` vector per input string. Store them in your vector d | Amazon Bedrock | `bedrockEmbedding` (`@tanstack/ai-bedrock`) | Titan Text Embeddings V2 (text), Titan Multimodal Embeddings G1 (text + image), Cohere Embed v3 (text) | text, or text + image | | Ollama | `ollamaEmbedding` (`@tanstack/ai-ollama`) | `nomic-embed-text`, `mxbai-embed-large`, and any local embedding model | text | | Vercel AI Gateway | `vercelGatewayEmbedding` (`@tanstack/ai-vercel-gateway`) | OpenAI-compatible models such as `openai/text-embedding-3-small` | text | +| Neon AI Gateway | `neonEmbedding` (`@tanstack/ai-neon`) | `qwen3-embedding-0-6b`, `gte-large-en`, `bge-large-en` (1024 dimensions) | text | Vercel AI Gateway takes a `creator/model` id: @@ -96,6 +98,18 @@ const result = await embed({ }); ``` +Neon AI Gateway uses the same call. The vectors fit a `vector(1024)` pgvector column in your Neon branch: + +```typescript +import { embed } from "@tanstack/ai"; +import { neonEmbedding } from "@tanstack/ai-neon"; + +const result = await embed({ + adapter: neonEmbedding("qwen3-embedding-0-6b"), + input: "a red guitar", +}); +``` + ## Pick the vector size Some models (Matryoshka models) let you choose the number of dimensions. Pass `dimensions`: diff --git a/docs/getting-started/overview.md b/docs/getting-started/overview.md index 13f716d851..a6a08355d6 100644 --- a/docs/getting-started/overview.md +++ b/docs/getting-started/overview.md @@ -139,6 +139,7 @@ Each adapter is its own package. Install only the ones you use. | `@tanstack/ai-worldlabs` | World Labs Marble (persistent 3D world generation). | | `@tanstack/ai-llmgateway` | LLM Gateway. Hundreds of models through one OpenAI-compatible endpoint. You can host it yourself. | | `@tanstack/ai-cloudflare` | Cloudflare Workers AI (chat, embeddings, image, speech, transcription, evaluate) and AI Gateway routing. | +| `@tanstack/ai-neon` | Neon AI Gateway (Claude, GPT, Gemini, and open models, plus embeddings, through one Neon credential). | | `@tanstack/ai-typesafe` | TypeSafe Jev (typed evaluate decisions). | | `@tanstack/ai-ollaya` | Local Ollaya `laya` models for typed evaluate decisions. No API key. | | `@tanstack/ai-lovable` | Lovable AI Gateway (Google and OpenAI chat, image, video, embeddings, and speech through one project key). | diff --git a/docs/structured-outputs/streaming.md b/docs/structured-outputs/streaming.md index 67a3192c55..cd55788948 100644 --- a/docs/structured-outputs/streaming.md +++ b/docs/structured-outputs/streaming.md @@ -215,6 +215,7 @@ Streaming structured output works with every adapter. Only some send the JSON as | `@tanstack/ai-byteplus` | Native stream in one request on supported models. Other models emit `RUN_ERROR`. | | `@tanstack/ai-llmgateway` | Native stream in one request (Chat Completions, `response_format: json_schema`) | | `@tanstack/ai-cloudflare` | Native stream in one request (Chat Completions, `response_format: json_schema`) | +| `@tanstack/ai-neon` | Native stream in one request (Chat Completions, `response_format: json_schema`). Claude uses the Anthropic adapter. | | `@tanstack/ai-lovable` | Native stream in one request (Responses or Chat Completions) | | `@tanstack/ai-gemini` | Native stream in one request (`generateContentStream`, `responseSchema`) | | Other adapters (Anthropic, Ollama, and more) | Runs `structuredOutput` without streaming, then emits the object as one `structured-output.complete` event | diff --git a/package.json b/package.json index 2a12ce279a..3197d56c3a 100644 --- a/package.json +++ b/package.json @@ -48,8 +48,8 @@ "generate:fal-image-fields": "tsx scripts/generate-fal-image-field-map.ts", "generate:models": "pnpm generate:models:fetch && pnpm regenerate:models && tsx scripts/sync-elevenlabs-models.ts && tsx scripts/sync-provider-models.ts && pnpm format", "generate:models:elevenlabs": "tsx scripts/sync-elevenlabs-models.ts", - "generate:models:fetch": "tsx scripts/fetch-openrouter-models.ts && tsx scripts/fetch-vercel-gateway-models.ts && tsx scripts/fetch-lovable-gateway-models.ts && node scripts/fetch-llmgateway-models.ts", - "regenerate:models": "tsx scripts/convert-openrouter-models.ts && tsx scripts/convert-vercel-gateway-models.ts && node scripts/convert-llmgateway-models.ts", + "generate:models:fetch": "tsx scripts/fetch-openrouter-models.ts && tsx scripts/fetch-vercel-gateway-models.ts && tsx scripts/fetch-lovable-gateway-models.ts && node scripts/fetch-llmgateway-models.ts && tsx scripts/fetch-neon-models.ts", + "regenerate:models": "tsx scripts/convert-openrouter-models.ts && tsx scripts/convert-vercel-gateway-models.ts && node scripts/convert-llmgateway-models.ts && tsx scripts/convert-neon-models.ts", "sync-docs-config": "node scripts/sync-docs-config.ts", "copy:readme": "node scripts/copy-readme.js", "changeset": "changeset", diff --git a/packages/ai-neon/LICENSE b/packages/ai-neon/LICENSE new file mode 100644 index 0000000000..308cb68dc0 --- /dev/null +++ b/packages/ai-neon/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 Tanner Linsley + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/packages/ai-neon/README.md b/packages/ai-neon/README.md new file mode 100644 index 0000000000..4b571a9968 --- /dev/null +++ b/packages/ai-neon/README.md @@ -0,0 +1,57 @@ +
+ + + + TanStack AI + +
+ +
+ +# @tanstack/ai-neon + +Neon AI Gateway adapter for TanStack AI. One Neon credential reaches Claude, GPT, Gemini, and open models. + +## Installation + +```bash +npm install @tanstack/ai-neon +``` + +## Auth + +Set `NEON_AI_GATEWAY_TOKEN` and `NEON_AI_GATEWAY_BASE_URL`. `neon env pull` writes both. + +```bash +NEON_AI_GATEWAY_TOKEN=nt_live_... +NEON_AI_GATEWAY_BASE_URL=https://-api.ai...aws.neon.tech +``` + +You can also pass the token and `baseURL` to a `create*` factory. + +## Usage + +```typescript +import { chat } from '@tanstack/ai' +import { neonText } from '@tanstack/ai-neon' + +const stream = chat({ + adapter: neonText('claude-opus-5-5'), + messages: [{ role: 'user', content: 'Hello' }], + threadId: 'thread-1', +}) +``` + +`neonText` sends Claude to the Anthropic Messages endpoint, `gpt-5-3-codex` and `gpt-5-5-pro` to the Responses endpoint, and every other model to Chat Completions. Pass `{ api: 'responses' }` to use Responses for another GPT model. + +See [docs/adapters/neon.md](../../docs/adapters/neon.md) for chat, reasoning, embeddings, and summarize. diff --git a/packages/ai-neon/package.json b/packages/ai-neon/package.json new file mode 100644 index 0000000000..d60a07e5f0 --- /dev/null +++ b/packages/ai-neon/package.json @@ -0,0 +1,70 @@ +{ + "name": "@tanstack/ai-neon", + "version": "0.0.1", + "description": "Neon AI Gateway adapter for TanStack AI chat, embeddings, and summarization.", + "author": "Tanner Linsley", + "license": "MIT", + "homepage": "https://tanstack.com/ai", + "repository": { + "type": "git", + "url": "git+https://github.com/TanStack/ai.git", + "directory": "packages/ai-neon" + }, + "bugs": { + "url": "https://github.com/TanStack/ai/issues" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/tannerlinsley" + }, + "type": "module", + "module": "./dist/esm/index.js", + "types": "./dist/esm/index.d.ts", + "exports": { + ".": { + "types": "./dist/esm/index.d.ts", + "import": "./dist/esm/index.js" + } + }, + "files": [ + "dist", + "src" + ], + "scripts": { + "build": "vite build", + "clean": "premove ./build ./dist", + "lint:fix": "oxlint src --type-aware --fix", + "test:build": "publint --strict", + "test:oxlint": "oxlint src --type-aware", + "test:coverage": "vitest run --coverage --coverage.include='src/**' --coverage.reporter=text-summary --coverage.reporter=json-summary", + "test:lib": "vitest run", + "test:lib:dev": "pnpm test:lib --watch", + "test:types": "tsc" + }, + "keywords": [ + "ai", + "ai-sdk", + "typescript", + "tanstack", + "neon", + "ai-gateway", + "adapter", + "llm", + "chat", + "embeddings" + ], + "devDependencies": { + "@tanstack/ai": "workspace:*", + "@vitest/coverage-v8": "4.1.10", + "vite": "^8.2.1" + }, + "peerDependencies": { + "@tanstack/ai": "workspace:^" + }, + "dependencies": { + "@tanstack/ai-anthropic": "workspace:^", + "@tanstack/ai-utils": "workspace:^", + "@tanstack/openai-base": "workspace:^", + "openai": "^7.30.0" + } +} diff --git a/packages/ai-neon/src/adapters/anthropic-text.ts b/packages/ai-neon/src/adapters/anthropic-text.ts new file mode 100644 index 0000000000..b4454df813 --- /dev/null +++ b/packages/ai-neon/src/adapters/anthropic-text.ts @@ -0,0 +1,35 @@ +import { AnthropicTextAdapter } from '@tanstack/ai-anthropic' +import { dropRejectedSampling } from './text' +import type { NeonAnthropicModel } from '../model-meta' + +/** Apply Neon's sampling restrictions before Anthropic validates options. */ +export class NeonAnthropicTextAdapter< + TModel extends NeonAnthropicModel, +> extends AnthropicTextAdapter { + override chatStream( + options: Parameters['chatStream']>[0], + ) { + return super.chatStream({ + ...options, + modelOptions: dropRejectedSampling( + options.model, + Object.assign({}, options.modelOptions), + ), + }) + } + + override structuredOutput( + options: Parameters['structuredOutput']>[0], + ) { + return super.structuredOutput({ + ...options, + chatOptions: { + ...options.chatOptions, + modelOptions: dropRejectedSampling( + options.chatOptions.model, + Object.assign({}, options.chatOptions.modelOptions), + ), + }, + }) + } +} diff --git a/packages/ai-neon/src/adapters/embedding.ts b/packages/ai-neon/src/adapters/embedding.ts new file mode 100644 index 0000000000..a3d1ec57c6 --- /dev/null +++ b/packages/ai-neon/src/adapters/embedding.ts @@ -0,0 +1,114 @@ +import OpenAI from 'openai' +import { BaseEmbeddingAdapter } from '@tanstack/ai/adapters' +import { toRunErrorPayload } from '@tanstack/ai/adapter-internals' +import { generateId } from '@tanstack/ai-utils' +import { requireTextOnlyEmbeddingInput } from '@tanstack/ai' +import { getNeonTokenFromEnv, withNeonDefaults } from '../utils/client' +import type { + EmbeddingOptions, + EmbeddingResult, + TokenUsage, +} from '@tanstack/ai' +import type OpenAI_SDK from 'openai' +import type { NeonEmbeddingModel } from '../model-meta' +import type { NeonClientConfig } from '../utils/client' + +export interface NeonEmbeddingConfig extends NeonClientConfig {} + +export interface NeonEmbeddingProviderOptions { + user?: string +} + +type ByName = { [K in NeonEmbeddingModel]: T } + +/** + * Neon AI Gateway embedding adapter. Talks to `POST /v1/embeddings`. + * + * ponytail: Neon's own provider sends at most 150 inputs per call; chunk the + * input here if a caller hits a gateway limit. + */ +export class NeonEmbeddingAdapter< + TModel extends NeonEmbeddingModel, +> extends BaseEmbeddingAdapter< + TModel, + NeonEmbeddingProviderOptions, + ByName, + ByName +> { + readonly name = 'neon' as const + + protected client: OpenAI + + constructor(config: NeonEmbeddingConfig, model: TModel) { + super(model, {}) + this.client = new OpenAI(withNeonDefaults(config, 'chat')) + } + + async createEmbeddings( + options: EmbeddingOptions, + ): Promise { + const { model, logger } = options + const texts = requireTextOnlyEmbeddingInput(options.input, this.name, model) + + try { + const request: OpenAI_SDK.EmbeddingCreateParams = { + ...options.modelOptions, + model, + input: texts, + encoding_format: 'float', + } + if (options.dimensions !== undefined) { + request.dimensions = options.dimensions + } + + logger.request( + `activity=embed provider=${this.name} model=${model} inputs=${texts.length}`, + { provider: this.name, model }, + ) + + const response = await this.client.embeddings.create(request) + + const usage: TokenUsage = { + promptTokens: response.usage.prompt_tokens, + completionTokens: 0, + totalTokens: response.usage.total_tokens, + } + + return { + id: generateId(this.name), + model, + embeddings: response.data.map((item) => ({ + vector: item.embedding, + index: item.index, + })), + usage, + } + } catch (error: unknown) { + logger.errors(`${this.name}.createEmbeddings fatal`, { + error: toRunErrorPayload(error, `${this.name}.createEmbeddings failed`), + source: `${this.name}.createEmbeddings`, + }) + throw error + } + } +} + +/** Creates a Neon AI Gateway embedding adapter with an explicit token. */ +export function createNeonEmbedding( + model: TModel, + apiKey: string, + config?: Omit, +): NeonEmbeddingAdapter { + return new NeonEmbeddingAdapter({ apiKey, ...config }, model) +} + +/** + * Creates a Neon AI Gateway embedding adapter with the token from + * `NEON_AI_GATEWAY_TOKEN` and the host from `NEON_AI_GATEWAY_BASE_URL`. + */ +export function neonEmbedding( + model: TModel, + config?: Omit, +): NeonEmbeddingAdapter { + return createNeonEmbedding(model, getNeonTokenFromEnv(), config) +} diff --git a/packages/ai-neon/src/adapters/factory.ts b/packages/ai-neon/src/adapters/factory.ts new file mode 100644 index 0000000000..02e7aeaf83 --- /dev/null +++ b/packages/ai-neon/src/adapters/factory.ts @@ -0,0 +1,186 @@ +import { + NEON_ANTHROPIC_MODELS, + NEON_RESPONSES_MODELS, + NEON_RESPONSES_ONLY_MODELS, +} from '../model-meta' +import { + NEON_ROUTES, + getNeonTokenFromEnv, + resolveNeonHost, +} from '../utils/client' +import { NeonAnthropicTextAdapter } from './anthropic-text' +import { NeonTextAdapter } from './text' +import { NeonResponsesTextAdapter } from './responses-text' +import type { AnthropicTextConfig } from '@tanstack/ai-anthropic' +import type { + NeonAnthropicModel, + NeonChatModel, + NeonResponsesModel, + NeonResponsesOnlyModel, +} from '../model-meta' +import type { NeonTextConfig } from './text' +import type { NeonResponsesTextConfig } from './responses-text' + +export type { NeonAnthropicTextAdapter } from './anthropic-text' + +/** + * Config for a Claude model. It goes to the Anthropic Messages route, so it + * takes the Anthropic SDK's client options. `baseURL` is the branch host. + */ +export type NeonAnthropicTextConfig = Omit< + AnthropicTextConfig, + 'apiKey' | 'auth' | 'baseURL' | 'provider' +> & { baseURL?: string; api?: never } + +/** Config for the Chat Completions route, the default. */ +export type NeonChatApiConfig = Omit & { + api?: 'chat' +} + +/** Config for the Responses route. Opt in with `api: 'responses'`. */ +export type NeonResponsesApiConfig = Omit & { + api: 'responses' +} + +export type AnyNeonTextConfig = + | NeonAnthropicTextConfig + | NeonChatApiConfig + | (Omit & { api?: 'responses' }) + +const anthropicModels = new Set(NEON_ANTHROPIC_MODELS) +const responsesOnlyModels = new Set(NEON_RESPONSES_ONLY_MODELS) +const responsesModels = new Set(NEON_RESPONSES_MODELS) + +type ChatCompletionsModel = Exclude< + NeonChatModel, + NeonAnthropicModel | NeonResponsesOnlyModel +> + +/** Client options for automatic routing, including a runtime model selection. */ +export type NeonDefaultTextConfig = [ + TModel, +] extends [NeonAnthropicModel] + ? NeonAnthropicTextConfig + : [TModel] extends [NeonResponsesOnlyModel] + ? Omit & { api?: 'responses' } + : [TModel] extends [ChatCompletionsModel] + ? NeonChatApiConfig + : Omit & NeonAnthropicTextConfig + +/** Each member of a model union keeps the adapter of its default route. */ +export type NeonDefaultTextAdapter = + TModel extends NeonAnthropicModel + ? NeonAnthropicTextAdapter + : TModel extends NeonResponsesOnlyModel + ? NeonResponsesTextAdapter + : NeonTextAdapter + +function isAnthropicModel(model: string): model is NeonAnthropicModel { + return anthropicModels.has(model) +} + +/** The adapter of any route, for callers that don't know the model's type. */ +export type AnyNeonTextAdapter = + | NeonAnthropicTextAdapter + | NeonResponsesTextAdapter + | NeonTextAdapter + +/** The routing behind `createNeonText`, without the per-model overloads. */ +export function buildNeonText( + model: NeonChatModel, + apiKey: string, + config: AnyNeonTextConfig = {}, +): AnyNeonTextAdapter { + // The overloads pair each model with its SDK's options. Only the shared + // fields are needed here; SDK-specific headers pass through the spread. + const { + api, + ...rest + }: { + api?: 'chat' | 'responses' + baseURL?: string + fetch?: typeof globalThis.fetch + } = config + if (isAnthropicModel(model)) { + if (api !== undefined) { + throw new Error(`${model} uses the Anthropic Messages API; omit api.`) + } + return new NeonAnthropicTextAdapter( + { + ...rest, + apiKey, + baseURL: resolveNeonHost(rest.baseURL) + NEON_ROUTES.anthropic, + auth: 'bearer', + provider: 'neon', + }, + model, + ) + } + if (api === 'chat' && responsesOnlyModels.has(model)) { + throw new Error( + `${model} requires the Responses API; omit api or use api: 'responses'.`, + ) + } + if (api === 'responses' && !responsesModels.has(model)) { + throw new Error(`${model} does not support the Responses API.`) + } + if (api === 'responses' || responsesOnlyModels.has(model)) { + // The overloads only let a Responses-capable model get here. + return new NeonResponsesTextAdapter( + { ...rest, apiKey }, + model as NeonResponsesModel, + ) + } + return new NeonTextAdapter({ ...rest, apiKey }, model) +} + +/** + * Creates a Neon AI Gateway text adapter with an explicit token. Each model + * goes to the route that serves it best: + * - Claude: the Anthropic Messages route, through `@tanstack/ai-anthropic`. + * - Models only Responses serves (codex, `*-pro`): the Responses route. + * - Everything else: the Chat Completions route. `api: 'responses'` moves a + * Responses-capable model to the Responses route. + * + * `config.baseURL` is the branch host. It defaults to + * `NEON_AI_GATEWAY_BASE_URL`. + */ +export function createNeonText( + model: TModel, + apiKey: string, + config: NeonResponsesApiConfig, +): NeonResponsesTextAdapter +export function createNeonText( + model: TModel, + apiKey: string, + config?: NeonDefaultTextConfig>, +): NeonDefaultTextAdapter +export function createNeonText( + model: NeonChatModel, + apiKey: string, + config?: AnyNeonTextConfig, +) { + return buildNeonText(model, apiKey, config) +} + +/** + * Creates a Neon AI Gateway text adapter with the token from + * `NEON_AI_GATEWAY_TOKEN` and the host from `NEON_AI_GATEWAY_BASE_URL`. + * See {@link createNeonText} for how each model is routed. + * + * @example + * ```ts + * const adapter = neonText('gemini-3-8-flash') + * ``` + */ +export function neonText( + model: TModel, + config: NeonResponsesApiConfig, +): NeonResponsesTextAdapter +export function neonText( + model: TModel, + config?: NeonDefaultTextConfig>, +): NeonDefaultTextAdapter +export function neonText(model: NeonChatModel, config?: AnyNeonTextConfig) { + return buildNeonText(model, getNeonTokenFromEnv(), config) +} diff --git a/packages/ai-neon/src/adapters/responses-text.ts b/packages/ai-neon/src/adapters/responses-text.ts new file mode 100644 index 0000000000..57b7387e0f --- /dev/null +++ b/packages/ai-neon/src/adapters/responses-text.ts @@ -0,0 +1,68 @@ +import OpenAI from 'openai' +import { OpenAIBaseResponsesTextAdapter } from '@tanstack/openai-base' +import { + NEON_MODEL_INPUT_MODALITIES, + NEON_MODEL_REASONING, +} from '../model-meta' +import { withNeonDefaults } from '../utils/client' +import { dropRejectedSampling } from './text' +import type { + DefaultMessageMetadataByModality, + Modality, + TextOptions, +} from '@tanstack/ai' +import type { NeonResponsesModel } from '../model-meta' +import type { NeonResponsesProviderOptions } from '../text/text-provider-options' +import type { ResolveInputModalities, ResolveReasoning } from './text' +import type { NeonClientConfig } from '../utils/client' +import type { OpenAIBaseTextAdapterOptions } from '@tanstack/openai-base' + +export interface NeonResponsesTextConfig + extends NeonClientConfig, OpenAIBaseTextAdapterOptions {} + +/** + * Neon AI Gateway adapter for the OpenAI Responses route + * (`/openai/v1/responses`). `neonText` picks it for the models only that + * route serves (codex and `*-pro`), and for `api: 'responses'`. + */ +export class NeonResponsesTextAdapter< + TModel extends NeonResponsesModel, + TProviderOptions extends Record = NeonResponsesProviderOptions, + TInputModalities extends ReadonlyArray = + ResolveInputModalities, +> extends OpenAIBaseResponsesTextAdapter< + TModel, + TProviderOptions, + TInputModalities, + DefaultMessageMetadataByModality, + readonly [], + ResolveReasoning +> { + override readonly kind = 'text' as const + override readonly name = 'neon' as const + override readonly inputModalities = NEON_MODEL_INPUT_MODALITIES[this.model] + /** The output cap on this route, for `summarize({ maxLength })`. */ + readonly maxTokensKey = 'max_output_tokens' + + constructor(config: NeonResponsesTextConfig, model: TModel) { + super( + model, + 'neon', + new OpenAI(withNeonDefaults(config, 'responses')), + config, + ) + } + + protected override modelReasoning(model: string) { + return NEON_MODEL_REASONING[model] + } + + protected override mapOptionsToRequest( + options: TextOptions, + ) { + return dropRejectedSampling( + options.model, + super.mapOptionsToRequest(options), + ) + } +} diff --git a/packages/ai-neon/src/adapters/summarize.ts b/packages/ai-neon/src/adapters/summarize.ts new file mode 100644 index 0000000000..7117161246 --- /dev/null +++ b/packages/ai-neon/src/adapters/summarize.ts @@ -0,0 +1,95 @@ +import { ChatStreamSummarizeAdapter } from '@tanstack/ai/adapters' +import { NEON_ANTHROPIC_MODELS } from '../model-meta' +import { getNeonTokenFromEnv } from '../utils/client' +import { buildNeonText } from './factory' +import type { InferTextProviderOptions } from '@tanstack/ai/adapters' +import type { + AnyNeonTextConfig, + NeonAnthropicTextAdapter, + NeonDefaultTextConfig, + NeonResponsesApiConfig, +} from './factory' +import type { + NeonAnthropicModel, + NeonChatModel, + NeonResponsesModel, + NeonResponsesOnlyModel, +} from '../model-meta' +import type { + NeonResponsesProviderOptions, + NeonTextProviderOptions, +} from '../text/text-provider-options' + +export type NeonSummarizeConfig = + | NeonDefaultTextConfig + | ([TModel] extends [NeonResponsesModel] ? NeonResponsesApiConfig : never) + +/** The model options of the route the model runs on. */ +export type NeonSummarizeProviderOptions< + TModel extends NeonChatModel, + TApi extends 'chat' | 'responses' | undefined = undefined, +> = TApi extends 'responses' + ? NeonResponsesProviderOptions + : TModel extends NeonAnthropicModel + ? InferTextProviderOptions> + : TModel extends NeonResponsesOnlyModel + ? NeonResponsesProviderOptions + : NeonTextProviderOptions + +const anthropicModels = new Set(NEON_ANTHROPIC_MODELS) + +/** Creates a summarize adapter with the same route and options as `createNeonText`. */ +export function createNeonSummarize( + model: TModel, + apiKey: string, + config: NeonResponsesApiConfig, +): ChatStreamSummarizeAdapter +export function createNeonSummarize( + model: TModel, + apiKey: string, + config?: NeonDefaultTextConfig>, +): ChatStreamSummarizeAdapter> +export function createNeonSummarize( + model: NeonChatModel, + apiKey: string, + config?: AnyNeonTextConfig, +) { + return buildNeonSummarize(model, apiKey, config) +} + +function buildNeonSummarize( + model: NeonChatModel, + apiKey: string, + config?: AnyNeonTextConfig, +): ChatStreamSummarizeAdapter< + NeonChatModel, + NeonSummarizeProviderOptions +> { + // Claude runs on the Anthropic adapter, so `maxLength` and the default + // temperature go where that adapter reads them. + const name = anthropicModels.has(model) ? 'anthropic' : 'neon' + return new ChatStreamSummarizeAdapter( + buildNeonText(model, apiKey, config), + model, + name, + ) +} + +/** + * Creates a Neon AI Gateway summarize adapter with the token from + * `NEON_AI_GATEWAY_TOKEN` and the host from `NEON_AI_GATEWAY_BASE_URL`. + */ +export function neonSummarize( + model: TModel, + config: NeonResponsesApiConfig, +): ChatStreamSummarizeAdapter +export function neonSummarize( + model: TModel, + config?: NeonDefaultTextConfig>, +): ChatStreamSummarizeAdapter> +export function neonSummarize( + model: NeonChatModel, + config?: AnyNeonTextConfig, +) { + return buildNeonSummarize(model, getNeonTokenFromEnv(), config) +} diff --git a/packages/ai-neon/src/adapters/text.ts b/packages/ai-neon/src/adapters/text.ts new file mode 100644 index 0000000000..2cf33f4b4b --- /dev/null +++ b/packages/ai-neon/src/adapters/text.ts @@ -0,0 +1,128 @@ +import OpenAI from 'openai' +import { OpenAIBaseChatCompletionsTextAdapter } from '@tanstack/openai-base' +import { + NEON_MODELS_REJECTING_SAMPLING, + NEON_MODELS_REJECTING_STREAM_OPTIONS, + NEON_MODEL_INPUT_MODALITIES, + NEON_MODEL_REASONING, +} from '../model-meta' +import { withNeonDefaults } from '../utils/client' +import type { + DefaultMessageMetadataByModality, + Modality, + TextOptions, +} from '@tanstack/ai' +import type { + NeonChatModel, + NeonModelInputModalitiesByName, + NeonModelReasoningByName, +} from '../model-meta' +import type { NeonTextProviderOptions } from '../text/text-provider-options' +import type { NeonClientConfig } from '../utils/client' +import type { OpenAIBaseTextAdapterOptions } from '@tanstack/openai-base' + +/** The reasoning levels of a model, for `chat({ reasoning })`. `never`: none. */ +export type ResolveReasoning = + TModel extends keyof NeonModelReasoningByName + ? NeonModelReasoningByName[TModel] + : never + +export type ResolveInputModalities = + TModel extends keyof NeonModelInputModalitiesByName + ? NeonModelInputModalitiesByName[TModel] + : readonly ['text'] + +// The Chat Completions converter handles text and images. The catalog also +// lists media that needs a native route, so constrain this route's types. +type ResolveChatInputModalities = + 'image' extends ResolveInputModalities[number] + ? readonly ['text', 'image'] + : readonly ['text'] + +/** + * Drop `temperature` and `top_p` for the models that 400 on them. Callers + * (and the summarize adapter's low-temperature default) can't know a given + * model rejects them, and stripping only ever averts a guaranteed 400. + */ +export function dropRejectedSampling( + model: string, + request: TRequest, +): TRequest { + if (NEON_MODELS_REJECTING_SAMPLING.has(model)) { + Reflect.deleteProperty(request, 'temperature') + Reflect.deleteProperty(request, 'top_p') + } + return request +} + +export interface NeonTextConfig + extends NeonClientConfig, OpenAIBaseTextAdapterOptions {} + +/** + * Neon AI Gateway adapter for the unified Chat Completions route + * (`/v1/chat/completions`), which serves every chat model in the catalog. + * `neonText` picks this adapter unless the model needs another route. + */ +export class NeonTextAdapter< + TModel extends NeonChatModel, + TProviderOptions extends Record = NeonTextProviderOptions, + TInputModalities extends ReadonlyArray = + ResolveChatInputModalities, +> extends OpenAIBaseChatCompletionsTextAdapter< + TModel, + TProviderOptions, + TInputModalities, + DefaultMessageMetadataByModality, + readonly [], + ResolveReasoning +> { + override readonly kind = 'text' as const + override readonly name = 'neon' as const + override readonly inputModalities = NEON_MODEL_INPUT_MODALITIES[ + this.model + ].filter((modality) => modality === 'text' || modality === 'image') + /** The gateway's output cap on this route, for `summarize({ maxLength })`. */ + readonly maxTokensKey = 'max_tokens' + + constructor(config: NeonTextConfig, model: TModel) { + const options = withNeonDefaults(config, 'chat') + // The base wraps `options.fetch` for `wrapFetch`, so it must be the + // normalizing fetch the client uses. + super(model, 'neon', new OpenAI(options), { + ...config, + fetch: options.fetch, + }) + } + + protected override modelReasoning(model: string) { + return NEON_MODEL_REASONING[model] + } + + /** Gemini models 400 on `stream_options` here, and stream usage anyway. */ + protected override streamOptions(model: string) { + return NEON_MODELS_REJECTING_STREAM_OPTIONS.has(model) + ? undefined + : super.streamOptions(model) + } + + protected override mapOptionsToRequest(options: TextOptions) { + return dropRejectedSampling( + options.model, + super.mapOptionsToRequest(options), + ) + } + + /** + * Reasoning deltas arrive as `delta.reasoning_content` (also where the + * content normalizer puts array `reasoning` parts) or `delta.reasoning`. + */ + protected override extractReasoning( + chunk: OpenAI.Chat.Completions.ChatCompletionChunk, + ): { text: string } | undefined { + const delta = chunk.choices?.[0]?.delta as + | { reasoning?: unknown; reasoning_content?: unknown } + | undefined + const raw = delta?.reasoning_content ?? delta?.reasoning + return typeof raw === 'string' && raw.length > 0 ? { text: raw } : undefined + } +} diff --git a/packages/ai-neon/src/index.ts b/packages/ai-neon/src/index.ts new file mode 100644 index 0000000000..8bfc74d10f --- /dev/null +++ b/packages/ai-neon/src/index.ts @@ -0,0 +1,66 @@ +/** + * @module @tanstack/ai-neon + * + * Neon AI Gateway adapter for TanStack AI. One Neon credential reaches every + * model in the gateway catalog; `neonText` sends each model to the route + * that serves it. + */ + +export { + createNeonText, + neonText, + type NeonAnthropicTextAdapter, + type NeonAnthropicTextConfig, + type NeonChatApiConfig, + type NeonResponsesApiConfig, +} from './adapters/factory' + +export { NeonTextAdapter, type NeonTextConfig } from './adapters/text' + +export { + NeonResponsesTextAdapter, + type NeonResponsesTextConfig, +} from './adapters/responses-text' + +export { + NeonEmbeddingAdapter, + createNeonEmbedding, + neonEmbedding, + type NeonEmbeddingConfig, + type NeonEmbeddingProviderOptions, +} from './adapters/embedding' + +export { + createNeonSummarize, + neonSummarize, + type NeonSummarizeConfig, + type NeonSummarizeProviderOptions, +} from './adapters/summarize' + +export type { + NeonResponsesProviderOptions, + NeonTextProviderOptions, +} from './text/text-provider-options' + +export { + NEON_ANTHROPIC_MODELS, + NEON_CHAT_MODELS, + NEON_EMBEDDING_MODELS, + NEON_RESPONSES_MODELS, + NEON_RESPONSES_ONLY_MODELS, +} from './model-meta' +export type { + NeonAnthropicModel, + NeonChatModel, + NeonEmbeddingModel, + NeonModelInputModalitiesByName, + NeonModelReasoningByName, + NeonResponsesModel, + NeonResponsesOnlyModel, +} from './model-meta' + +export { + getNeonTokenFromEnv, + resolveNeonHost, + type NeonClientConfig, +} from './utils/client' diff --git a/packages/ai-neon/src/model-meta.ts b/packages/ai-neon/src/model-meta.ts new file mode 100644 index 0000000000..a3c9ce29f8 --- /dev/null +++ b/packages/ai-neon/src/model-meta.ts @@ -0,0 +1,367 @@ +/** + * This file is generated by scripts/convert-neon-models.ts. Do not edit by hand. + */ + +import type { ModelReasoning } from '@tanstack/ai' +import type { ModelReasoningCapability } from '@tanstack/ai/adapter-internals' + +/** Every chat model the gateway serves, on any route. */ +export const NEON_CHAT_MODELS = [ + 'claude-fable-5', + 'claude-fable-5-1', + 'claude-haiku-4-5', + 'claude-haiku-5-5', + 'claude-opus-4-1', + 'claude-opus-4-5', + 'claude-opus-4-6', + 'claude-opus-4-7', + 'claude-opus-4-8', + 'claude-opus-5', + 'claude-opus-5-5', + 'claude-sonnet-4-5', + 'claude-sonnet-4-6', + 'claude-sonnet-5', + 'claude-sonnet-5-5', + 'deepseek-v4-1-flash', + 'gemini-3-1-flash-lite', + 'gemini-3-1-pro', + 'gemini-3-5-flash', + 'gemini-3-5-flash-lite', + 'gemini-3-6-flash', + 'gemini-3-7-flash', + 'gemini-3-8-flash', + 'gemini-3-flash', + 'gemma-3-12b', + 'glm-5-2', + 'glm-5-3', + 'glm-5-3-flash', + 'gpt-5', + 'gpt-5-1', + 'gpt-5-2', + 'gpt-5-3-codex', + 'gpt-5-4', + 'gpt-5-4-mini', + 'gpt-5-4-nano', + 'gpt-5-5', + 'gpt-5-5-pro', + 'gpt-5-6-luna', + 'gpt-5-6-sol', + 'gpt-5-6-terra', + 'gpt-5-mini', + 'gpt-5-nano', + 'gpt-6-1-sol', + 'gpt-6-astra', + 'gpt-6-luna', + 'gpt-6-sol', + 'gpt-oss-120b', + 'gpt-oss-20b', + 'grok-4-6', + 'grok-4-7', + 'inkling', + 'kimi-k3', + 'llama-4-maverick', + 'meta-llama-3-1-8b-instruct', + 'meta-llama-3-3-70b-instruct', + 'qwen3-next-80b-a3b-instruct', + 'qwen35-122b-a10b', +] as const + +export type NeonChatModel = (typeof NEON_CHAT_MODELS)[number] + +/** Claude models. `neonText` sends them to the Anthropic Messages route. */ +export const NEON_ANTHROPIC_MODELS = [ + 'claude-fable-5', + 'claude-fable-5-1', + 'claude-haiku-4-5', + 'claude-haiku-5-5', + 'claude-opus-4-5', + 'claude-opus-4-6', + 'claude-opus-4-7', + 'claude-opus-4-8', + 'claude-opus-5', + 'claude-opus-5-5', + 'claude-sonnet-4-5', + 'claude-sonnet-4-6', + 'claude-sonnet-5', + 'claude-sonnet-5-5', +] as const + +export type NeonAnthropicModel = (typeof NEON_ANTHROPIC_MODELS)[number] + +/** Models the Responses route serves. They take `api: 'responses'`. */ +export const NEON_RESPONSES_MODELS = [ + 'gpt-5', + 'gpt-5-1', + 'gpt-5-2', + 'gpt-5-3-codex', + 'gpt-5-4', + 'gpt-5-4-mini', + 'gpt-5-4-nano', + 'gpt-5-5', + 'gpt-5-5-pro', + 'gpt-5-6-luna', + 'gpt-5-6-sol', + 'gpt-5-6-terra', + 'gpt-5-mini', + 'gpt-5-nano', + 'gpt-6-1-sol', + 'gpt-6-astra', + 'gpt-6-luna', + 'gpt-6-sol', + 'grok-4-6', + 'grok-4-7', +] as const + +export type NeonResponsesModel = (typeof NEON_RESPONSES_MODELS)[number] + +/** Models only the Responses route serves. `neonText` always sends them there. */ +export const NEON_RESPONSES_ONLY_MODELS = [ + 'gpt-5-3-codex', + 'gpt-5-5-pro', +] as const + +export type NeonResponsesOnlyModel = (typeof NEON_RESPONSES_ONLY_MODELS)[number] + +export const NEON_EMBEDDING_MODELS = [ + 'bge-large-en', + 'gte-large-en', + 'qwen3-embedding-0-6b', +] as const + +export type NeonEmbeddingModel = (typeof NEON_EMBEDDING_MODELS)[number] + +/** Models that reject `temperature` and `top_p`. The adapters drop both. */ +export const NEON_MODELS_REJECTING_SAMPLING: ReadonlySet = new Set([ + 'claude-fable-5', + 'claude-fable-5-1', + 'claude-haiku-5-5', + 'claude-opus-4-7', + 'claude-opus-4-8', + 'claude-opus-5', + 'claude-opus-5-5', + 'claude-sonnet-5', + 'claude-sonnet-5-5', + 'gemini-3-6-flash', + 'gemini-3-7-flash', + 'gemini-3-8-flash', + 'gpt-5', + 'gpt-5-5', + 'gpt-5-5-pro', + 'gpt-5-6-luna', + 'gpt-5-6-sol', + 'gpt-5-6-terra', + 'gpt-5-mini', + 'gpt-5-nano', + 'gpt-6-1-sol', + 'gpt-6-astra', + 'gpt-6-luna', + 'gpt-6-sol', +] as const) + +/** + * Models that reject `stream_options` on Chat Completions. They stream + * usage without it. + */ +export const NEON_MODELS_REJECTING_STREAM_OPTIONS: ReadonlySet = + new Set([ + 'gemini-3-1-flash-lite', + 'gemini-3-1-pro', + 'gemini-3-5-flash', + 'gemini-3-5-flash-lite', + 'gemini-3-6-flash', + 'gemini-3-7-flash', + 'gemini-3-8-flash', + 'gemini-3-flash', + ] as const) + +/** The input modalities of each chat model. */ +export const NEON_MODEL_INPUT_MODALITIES = { + 'claude-fable-5': ['text', 'image', 'document'], + 'claude-fable-5-1': ['text', 'image', 'document'], + 'claude-haiku-4-5': ['text', 'image', 'document'], + 'claude-haiku-5-5': ['text', 'image', 'document'], + 'claude-opus-4-1': ['text', 'image', 'document'], + 'claude-opus-4-5': ['text', 'image', 'document'], + 'claude-opus-4-6': ['text', 'image', 'document'], + 'claude-opus-4-7': ['text', 'image', 'document'], + 'claude-opus-4-8': ['text', 'image', 'document'], + 'claude-opus-5': ['text', 'image', 'document'], + 'claude-opus-5-5': ['text', 'image', 'document'], + 'claude-sonnet-4-5': ['text', 'image', 'document'], + 'claude-sonnet-4-6': ['text', 'image', 'document'], + 'claude-sonnet-5': ['text', 'image', 'document'], + 'claude-sonnet-5-5': ['text', 'image', 'document'], + 'deepseek-v4-1-flash': ['text', 'image'], + 'gemini-3-1-flash-lite': ['text', 'image', 'document', 'audio', 'video'], + 'gemini-3-1-pro': ['text', 'image', 'document', 'audio', 'video'], + 'gemini-3-5-flash': ['text', 'image', 'document', 'audio', 'video'], + 'gemini-3-5-flash-lite': ['text', 'image', 'document', 'audio', 'video'], + 'gemini-3-6-flash': ['text', 'image', 'document', 'audio', 'video'], + 'gemini-3-7-flash': ['text', 'image', 'document', 'audio', 'video'], + 'gemini-3-8-flash': ['text', 'image', 'document', 'audio', 'video'], + 'gemini-3-flash': ['text', 'image', 'document', 'audio', 'video'], + 'gemma-3-12b': ['text', 'image'], + 'glm-5-2': ['text'], + 'glm-5-3': ['text'], + 'glm-5-3-flash': ['text', 'image'], + 'gpt-5': ['text', 'image'], + 'gpt-5-1': ['text', 'image'], + 'gpt-5-2': ['text', 'image'], + 'gpt-5-3-codex': ['text', 'image', 'document'], + 'gpt-5-4': ['text', 'image', 'document'], + 'gpt-5-4-mini': ['text', 'image'], + 'gpt-5-4-nano': ['text', 'image'], + 'gpt-5-5': ['text', 'image', 'document'], + 'gpt-5-5-pro': ['text', 'image', 'document'], + 'gpt-5-6-luna': ['text', 'image', 'document'], + 'gpt-5-6-sol': ['text', 'image', 'document'], + 'gpt-5-6-terra': ['text', 'image', 'document'], + 'gpt-5-mini': ['text', 'image'], + 'gpt-5-nano': ['text', 'image'], + 'gpt-6-1-sol': ['text', 'image', 'document'], + 'gpt-6-astra': ['text', 'image', 'document'], + 'gpt-6-luna': ['text', 'image', 'document'], + 'gpt-6-sol': ['text', 'image', 'document'], + 'gpt-oss-120b': ['text'], + 'gpt-oss-20b': ['text'], + 'grok-4-6': ['text', 'image'], + 'grok-4-7': ['text'], + inkling: ['text', 'image', 'audio'], + 'kimi-k3': ['text', 'image', 'video'], + 'llama-4-maverick': ['text', 'image'], + 'meta-llama-3-1-8b-instruct': ['text'], + 'meta-llama-3-3-70b-instruct': ['text'], + 'qwen3-next-80b-a3b-instruct': ['text'], + 'qwen35-122b-a10b': ['text'], +} as const satisfies Record> + +export type NeonModelInputModalitiesByName = typeof NEON_MODEL_INPUT_MODALITIES + +// Reasoning +// +// Each model's data for `chat({ reasoning })`: the levels it has, with the +// effort value the gateway takes for each. Each model points at one shape. + +const EFFORT_LOW_HIGH_MAX = { + levels: { low: 'low', high: 'high', max: 'max' }, + budget: false, +} as const +const EFFORT_LOW_TO_HIGH = { + levels: { low: 'low', medium: 'medium', high: 'high' }, + budget: false, +} as const +const EFFORT_LOW_TO_MAX = { + levels: { + low: 'low', + medium: 'medium', + high: 'high', + xhigh: 'xhigh', + max: 'max', + }, + budget: false, +} as const +const EFFORT_LOW_TO_XHIGH = { + levels: { low: 'low', medium: 'medium', high: 'high', xhigh: 'xhigh' }, + budget: false, +} as const +const EFFORT_MEDIUM_TO_XHIGH = { + levels: { medium: 'medium', high: 'high', xhigh: 'xhigh' }, + budget: false, +} as const +const EFFORT_MINIMAL_TO_HIGH = { + levels: { minimal: 'minimal', low: 'low', medium: 'medium', high: 'high' }, + budget: false, +} as const +const EFFORT_NONE_LOW_TO_HIGH = { + levels: { off: 'none', low: 'low', medium: 'medium', high: 'high' }, + budget: false, +} as const +const EFFORT_NONE_LOW_TO_MAX = { + levels: { + off: 'none', + low: 'low', + medium: 'medium', + high: 'high', + xhigh: 'xhigh', + max: 'max', + }, + budget: false, +} as const +const EFFORT_NONE_LOW_TO_XHIGH = { + levels: { + off: 'none', + low: 'low', + medium: 'medium', + high: 'high', + xhigh: 'xhigh', + }, + budget: false, +} as const +const EFFORT_NONE_MINIMAL_TO_HIGH_MAX = { + levels: { + off: 'none', + minimal: 'minimal', + low: 'low', + medium: 'medium', + high: 'high', + max: 'max', + }, + budget: false, +} as const + +const NEON_REASONING = { + 'gemini-3-1-flash-lite': EFFORT_LOW_TO_HIGH, + 'gemini-3-1-pro': EFFORT_LOW_TO_HIGH, + 'gemini-3-5-flash': EFFORT_MINIMAL_TO_HIGH, + 'gemini-3-5-flash-lite': EFFORT_MINIMAL_TO_HIGH, + 'gemini-3-6-flash': EFFORT_MINIMAL_TO_HIGH, + 'gemini-3-7-flash': EFFORT_MINIMAL_TO_HIGH, + 'gemini-3-8-flash': EFFORT_MINIMAL_TO_HIGH, + 'gemini-3-flash': EFFORT_LOW_TO_HIGH, + 'glm-5-2': EFFORT_NONE_MINIMAL_TO_HIGH_MAX, + 'glm-5-3': EFFORT_LOW_HIGH_MAX, + 'glm-5-3-flash': EFFORT_LOW_HIGH_MAX, + 'gpt-5': EFFORT_MINIMAL_TO_HIGH, + 'gpt-5-1': EFFORT_NONE_LOW_TO_HIGH, + 'gpt-5-2': EFFORT_NONE_LOW_TO_XHIGH, + 'gpt-5-3-codex': EFFORT_NONE_LOW_TO_XHIGH, + 'gpt-5-4': EFFORT_NONE_LOW_TO_XHIGH, + 'gpt-5-4-mini': EFFORT_NONE_LOW_TO_XHIGH, + 'gpt-5-4-nano': EFFORT_NONE_LOW_TO_XHIGH, + 'gpt-5-5': EFFORT_NONE_LOW_TO_XHIGH, + 'gpt-5-5-pro': EFFORT_MEDIUM_TO_XHIGH, + 'gpt-5-6-luna': EFFORT_NONE_LOW_TO_MAX, + 'gpt-5-6-sol': EFFORT_NONE_LOW_TO_MAX, + 'gpt-5-6-terra': EFFORT_NONE_LOW_TO_MAX, + 'gpt-5-mini': EFFORT_MINIMAL_TO_HIGH, + 'gpt-5-nano': EFFORT_MINIMAL_TO_HIGH, + 'gpt-6-1-sol': EFFORT_LOW_TO_MAX, + 'gpt-6-astra': EFFORT_LOW_TO_MAX, + 'gpt-6-luna': EFFORT_LOW_TO_MAX, + 'gpt-6-sol': EFFORT_LOW_TO_MAX, + 'gpt-oss-120b': EFFORT_LOW_TO_HIGH, + 'gpt-oss-20b': EFFORT_LOW_TO_HIGH, + 'grok-4-6': EFFORT_LOW_TO_XHIGH, + 'grok-4-7': EFFORT_LOW_TO_XHIGH, + inkling: EFFORT_NONE_MINIMAL_TO_HIGH_MAX, + 'kimi-k3': EFFORT_NONE_MINIMAL_TO_HIGH_MAX, + 'qwen35-122b-a10b': EFFORT_NONE_LOW_TO_HIGH, +} satisfies Partial> + +/** + * Each chat model's reasoning levels, for `chat({ reasoning })`. A model + * that is not here does not reason. + */ +export type NeonModelReasoningByName = { + [K in keyof typeof NEON_REASONING]: ModelReasoningCapability< + (typeof NEON_REASONING)[K] + > +} + +/** + * Runtime map from chat model id to its reasoning data, for the text + * adapters. An unknown id gives `undefined`: the adapter sends no reasoning + * field. + */ +export const NEON_MODEL_REASONING: Readonly> = + NEON_REASONING diff --git a/packages/ai-neon/src/text/text-provider-options.ts b/packages/ai-neon/src/text/text-provider-options.ts new file mode 100644 index 0000000000..270dbdd659 --- /dev/null +++ b/packages/ai-neon/src/text/text-provider-options.ts @@ -0,0 +1,33 @@ +import type { ChatCompletionCreateParamsBase } from 'openai/resources/chat/completions' +import type { ResponseCreateParamsBase } from 'openai/resources/responses/responses' + +/** + * Model options for the Chat Completions route (`/v1/chat/completions`): the + * standard Chat Completions parameters. The gateway caps output with + * `max_tokens`. `chat({ reasoning })` sets `reasoning_effort`, and + * `chat({ toolChoice })` sets `tool_choice`. + */ +export type NeonTextProviderOptions = Pick< + ChatCompletionCreateParamsBase, + | 'frequency_penalty' + | 'max_completion_tokens' + | 'max_tokens' + | 'parallel_tool_calls' + | 'presence_penalty' + | 'seed' + | 'stop' + | 'temperature' + | 'top_p' + | 'user' +> + +/** Model options for the Responses route (`/openai/v1/responses`). */ +export type NeonResponsesProviderOptions = Pick< + ResponseCreateParamsBase, + | 'max_output_tokens' + | 'parallel_tool_calls' + | 'temperature' + | 'top_p' + | 'truncation' + | 'user' +> diff --git a/packages/ai-neon/src/utils/client.ts b/packages/ai-neon/src/utils/client.ts new file mode 100644 index 0000000000..47c4c7a337 --- /dev/null +++ b/packages/ai-neon/src/utils/client.ts @@ -0,0 +1,77 @@ +import { getApiKeyFromEnv } from '@tanstack/ai-utils' +import { normalizeNeonContent } from './normalize-content' +import type { ClientOptions } from 'openai' + +/** + * The gateway routes on the branch host. Chat Completions serves every chat + * model, Responses serves the OpenAI models, and Anthropic serves Claude. The + * Anthropic SDK appends `/v1/messages` itself. + */ +export const NEON_ROUTES = { + chat: '/v1', // also serves embeddings + responses: '/openai/v1', + anthropic: '/anthropic', +} as const + +export interface NeonClientConfig extends Omit< + ClientOptions, + 'apiKey' | 'baseURL' +> { + /** The Neon credential (`nt_live_...`), sent as `Authorization: Bearer`. */ + apiKey: string + /** + * The bare branch gateway host, with no path, for example + * `https://-api.ai...aws.neon.tech`. The adapter + * appends each route's path. Defaults to `NEON_AI_GATEWAY_BASE_URL`. + */ + baseURL?: string +} + +/** + * Gets the Neon credential from `NEON_AI_GATEWAY_TOKEN`. + * @throws Error if it is not set + */ +export function getNeonTokenFromEnv(): string { + try { + return getApiKeyFromEnv('NEON_AI_GATEWAY_TOKEN') + } catch (cause) { + throw new Error( + 'NEON_AI_GATEWAY_TOKEN is required. Set it in the environment (`neon env pull` writes it) or pass the token to a create* factory.', + { cause }, + ) + } +} + +/** + * The branch gateway host: `baseURL`, else `NEON_AI_GATEWAY_BASE_URL`, with + * any trailing slash removed. + * @throws Error if neither is set + */ +export function resolveNeonHost(baseURL?: string): string { + let host = baseURL + if (!host) { + try { + host = getApiKeyFromEnv('NEON_AI_GATEWAY_BASE_URL') + } catch (cause) { + throw new Error( + 'NEON_AI_GATEWAY_BASE_URL is required. Set it in the environment (`neon env pull` writes it) or pass baseURL.', + { cause }, + ) + } + } + return host.replace(/\/+$/, '') +} + +/** OpenAI client options for one route of the gateway. */ +export function withNeonDefaults( + config: NeonClientConfig, + route: 'chat' | 'responses', +): ClientOptions { + const { baseURL, fetch, ...rest } = config + return { + ...rest, + baseURL: resolveNeonHost(baseURL) + NEON_ROUTES[route], + // Only Chat Completions returns the array `content` that needs fixing. + fetch: route === 'chat' ? normalizeNeonContent(fetch) : fetch, + } +} diff --git a/packages/ai-neon/src/utils/normalize-content.ts b/packages/ai-neon/src/utils/normalize-content.ts new file mode 100644 index 0000000000..625f5599a9 --- /dev/null +++ b/packages/ai-neon/src/utils/normalize-content.ts @@ -0,0 +1,143 @@ +import type { ClientOptions } from 'openai' + +type Fetch = NonNullable + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value) +} + +/** The text of a `reasoning` part: its `text`, `summary[].text` and `content[].text`. */ +function reasoningText(part: Record): string { + let text = typeof part.text === 'string' ? part.text : '' + for (const key of ['summary', 'content']) { + const entries = part[key] + if (!Array.isArray(entries)) continue + for (const entry of entries) { + if (isRecord(entry) && typeof entry.text === 'string') text += entry.text + } + } + return text +} + +/** + * Turn an array `content` on a message or delta into the string the Chat + * Completions spec requires. `text` parts become `content`; `reasoning` parts + * go to `reasoning_content`, where the adapter reads reasoning. + */ +function flattenContent(target: unknown): boolean { + if (!isRecord(target) || !Array.isArray(target.content)) return false + let text = '' + let reasoning = '' + for (const part of target.content) { + if (!isRecord(part)) continue + if (part.type === 'text' && typeof part.text === 'string') text += part.text + else if (part.type === 'reasoning') reasoning += reasoningText(part) + } + target.content = text + if (reasoning) { + const existing = + typeof target.reasoning_content === 'string' + ? target.reasoning_content + : '' + target.reasoning_content = existing + reasoning + } + return true +} + +/** Repair Neon content and usage fields, or return `undefined` when unchanged. */ +function rewrite(json: string): string | undefined { + let body: unknown + try { + body = JSON.parse(json) + } catch { + return undefined + } + if (!isRecord(body) || !Array.isArray(body.choices)) return undefined + let changed = false + for (const choice of body.choices) { + if (!isRecord(choice)) continue + if (flattenContent(choice.message)) changed = true + if (flattenContent(choice.delta)) changed = true + } + // Gemini reports reasoning tokens at the root of usage. The inherited + // Chat Completions parser reads the standard nested field, including on + // usage-only chunks whose choices array is empty. + if (isRecord(body.usage) && typeof body.usage.reasoning_tokens === 'number') { + const details = isRecord(body.usage.completion_tokens_details) + ? body.usage.completion_tokens_details + : {} + body.usage.completion_tokens_details = { + ...details, + reasoning_tokens: details.reasoning_tokens ?? body.usage.reasoning_tokens, + } + changed = true + } + return changed ? JSON.stringify(body) : undefined +} + +/** Rewrites each SSE `data:` line, buffering a line split across chunks. */ +function sseRewriter(): TransformStream { + let buffer = '' + const rewriteLine = (line: string) => { + if (!line.startsWith('data:')) return line + const fixed = rewrite(line.slice(5)) + return fixed === undefined ? line : `data: ${fixed}` + } + return new TransformStream({ + transform(chunk, controller) { + buffer += chunk + const lines = buffer.split('\n') + buffer = lines.pop() ?? '' + if (lines.length > 0) { + controller.enqueue(lines.map(rewriteLine).join('\n') + '\n') + } + }, + flush(controller) { + if (buffer) controller.enqueue(rewriteLine(buffer)) + }, + }) +} + +function withBody(response: Response, body: BodyInit): Response { + const headers = new Headers(response.headers) + // The body is already decoded, and a rewrite changes its length. + headers.delete('content-length') + headers.delete('content-encoding') + return new Response(body, { + status: response.status, + statusText: response.statusText, + headers, + }) +} + +/** + * A fetch for the Chat Completions route. For some models the gateway sends + * `content` as an array of `{ type: 'reasoning' | 'text' }` parts instead of a + * string, in JSON and in each streamed delta. This fetch rewrites those to the + * spec shape before the OpenAI SDK reads them, so `chatStream`, + * `structuredOutput` and `structuredOutputStream` all see strings. A body + * with string content passes through unchanged. + * + * ponytail: remove when Neon returns string content (neondatabase/neon-pkgs#308). + */ +export function normalizeNeonContent(baseFetch?: Fetch): Fetch { + return async (input, init) => { + const response = await (baseFetch ?? globalThis.fetch)(input, init) + if (!response.ok || !response.body) return response + const type = response.headers.get('content-type') ?? '' + if (type.includes('text/event-stream')) { + return withBody( + response, + response.body + .pipeThrough(new TextDecoderStream()) + .pipeThrough(sseRewriter()) + .pipeThrough(new TextEncoderStream()), + ) + } + if (type.includes('application/json')) { + const text = await response.text() + return withBody(response, rewrite(text) ?? text) + } + return response + } +} diff --git a/packages/ai-neon/tests/adapters.test.ts b/packages/ai-neon/tests/adapters.test.ts new file mode 100644 index 0000000000..dc92dac5cb --- /dev/null +++ b/packages/ai-neon/tests/adapters.test.ts @@ -0,0 +1,132 @@ +import { describe, expect, it } from 'vitest' +import { resolveDebugOption } from '@tanstack/ai/adapter-internals' +import { createNeonEmbedding } from '../src/adapters/embedding' +import { createNeonSummarize } from '../src/adapters/summarize' + +const logger = resolveDebugOption(false) +const HOST = 'https://br-test-api.ai.c-1.us-east-2.aws.neon.tech' + +describe('neon embedding adapter', () => { + it('posts to /v1/embeddings and returns the vectors', async () => { + const sent: Array<{ url: string; body: any }> = [] + const adapter = createNeonEmbedding('qwen3-embedding-0-6b', 'nt-test', { + baseURL: HOST, + fetch: async (input, init) => { + sent.push({ + url: new Request(input).url, + body: JSON.parse(String(init?.body)), + }) + return Response.json({ + object: 'list', + data: [{ object: 'embedding', embedding: [0.1, 0.2], index: 0 }], + model: 'qwen3-embedding-0-6b', + usage: { prompt_tokens: 3, total_tokens: 3 }, + }) + }, + }) + + const result = await adapter.createEmbeddings({ + model: 'qwen3-embedding-0-6b', + input: ['a red guitar'], + dimensions: 1024, + logger, + }) + + expect(result.embeddings[0]?.vector).toEqual([0.1, 0.2]) + expect(sent[0]?.url).toBe(`${HOST}/v1/embeddings`) + expect(sent[0]?.body).toMatchObject({ + model: 'qwen3-embedding-0-6b', + input: ['a red guitar'], + dimensions: 1024, + }) + }) +}) + +describe('neon summarize adapter', () => { + /** Runs one summarize call and gives back the request body. */ + async function summarizeBody( + model: Parameters[0], + ) { + const bodies: Array = [] + const adapter = createNeonSummarize(model, 'nt-test', { + baseURL: HOST, + fetch: async (_input, init) => { + bodies.push(JSON.parse(String(init?.body))) + return new Response('', { + headers: { 'content-type': 'text/event-stream' }, + }) + }, + }) + await adapter + .summarize({ + model, + threadId: 'thread-1', + text: 'A long text.', + maxLength: 50, + logger, + }) + .catch(() => { + // An empty stream fails; only the request matters here. + }) + return { adapter, body: bodies[0] } + } + + it('caps output with max_tokens on Chat Completions', async () => { + const { adapter, body } = await summarizeBody('gpt-5-4') + expect(adapter.name).toBe('neon') + expect(body.max_tokens).toBeGreaterThan(0) + expect(body.temperature).toBe(0.3) + }) + + it('uses Responses token options for an explicit Responses route', async () => { + const bodies: Array = [] + const adapter = createNeonSummarize('gpt-6-1-sol', 'nt-test', { + api: 'responses', + baseURL: HOST, + fetch: async (input, init) => { + expect(new Request(input).url).toBe(`${HOST}/openai/v1/responses`) + bodies.push(JSON.parse(String(init?.body))) + return new Response('', { + headers: { 'content-type': 'text/event-stream' }, + }) + }, + }) + await adapter.summarize({ + model: 'gpt-6-1-sol', + threadId: 'thread-1', + text: 'A long text.', + maxLength: 50, + logger, + modelOptions: { max_output_tokens: 100 }, + }) + await adapter.summarize({ + model: 'gpt-6-1-sol', + threadId: 'thread-1', + text: 'A long text.', + maxLength: 50, + logger, + }) + expect(bodies.map((body) => body.max_output_tokens)).toEqual([100, 50]) + for (const body of bodies) { + expect(body).not.toHaveProperty('max_tokens') + expect(body).not.toHaveProperty('temperature') + } + }) + + it('drops the default temperature for a model that rejects it', async () => { + const { body } = await summarizeBody('gpt-5-5') + expect(body).not.toHaveProperty('temperature') + }) + + it('runs Claude on the Anthropic adapter', async () => { + const { adapter, body } = await summarizeBody('claude-sonnet-5-5') + expect(adapter.name).toBe('anthropic') + expect(body.max_tokens).toBeGreaterThan(0) + }) + + it('omits the default temperature for Claude models that reject sampling', async () => { + const { body } = await summarizeBody('claude-opus-5-5') + expect(body).not.toHaveProperty('temperature') + expect(body).not.toHaveProperty('top_p') + }) +}) diff --git a/packages/ai-neon/tests/normalize-content.test.ts b/packages/ai-neon/tests/normalize-content.test.ts new file mode 100644 index 0000000000..10b3f55456 --- /dev/null +++ b/packages/ai-neon/tests/normalize-content.test.ts @@ -0,0 +1,184 @@ +import { describe, expect, it } from 'vitest' +import { EventType } from '@tanstack/ai' +import { resolveDebugOption } from '@tanstack/ai/adapter-internals' +import { normalizeNeonContent } from '../src/utils/normalize-content' +import { createNeonText } from '../src/adapters/factory' +import type { FetchWrapper } from '@tanstack/ai' + +const logger = resolveDebugOption(false) +const HOST = 'https://br-test-api.ai.c-1.us-east-2.aws.neon.tech' + +const encoder = new TextEncoder() + +/** An SSE response whose body arrives in the given raw chunks. */ +function sseResponse(chunks: Array): Response { + const body = new ReadableStream({ + start(controller) { + for (const chunk of chunks) controller.enqueue(encoder.encode(chunk)) + controller.close() + }, + }) + return new Response(body, { + headers: { 'content-type': 'text/event-stream' }, + }) +} + +function chunk(delta: Record, finish: string | null = null) { + return `data: ${JSON.stringify({ + id: 'chatcmpl-1', + object: 'chat.completion.chunk', + created: 1, + model: 'gemini-3-8-flash', + choices: [{ index: 0, delta, finish_reason: finish }], + })}\n\n` +} + +const arrayDelta = chunk({ + role: 'assistant', + content: [ + { + type: 'reasoning', + summary: [{ type: 'summary_text', text: 'Thinking.' }], + }, + { type: 'text', text: 'Hello', thoughtSignature: 'sig' }, + ], +}) + +describe('normalizeNeonContent', () => { + it('rewrites an array delta, even when a line is split across chunks', async () => { + const stream = arrayDelta + chunk({}, 'stop') + 'data: [DONE]\n\n' + const split = [stream.slice(0, 40), stream.slice(40, 90), stream.slice(90)] + const fetch = normalizeNeonContent(async () => sseResponse(split)) + + const text = await (await fetch('https://x/v1/chat/completions')).text() + const events = text + .split('\n') + .filter((line) => line.startsWith('data: {')) + .map((line) => JSON.parse(line.slice(6))) + + expect(events[0].choices[0].delta.content).toBe('Hello') + expect(events[0].choices[0].delta.reasoning_content).toBe('Thinking.') + expect(text).toContain('data: [DONE]') + }) + + it('rewrites an array message in a JSON body', async () => { + const fetch = normalizeNeonContent(async () => + Response.json({ + choices: [ + { + index: 0, + message: { + role: 'assistant', + content: [ + { type: 'reasoning', text: 'Hmm.' }, + { type: 'text', text: '{"a":1}' }, + ], + }, + }, + ], + }), + ) + + const body = await (await fetch('https://x/v1/chat/completions')).json() + + expect(body.choices[0].message.content).toBe('{"a":1}') + expect(body.choices[0].message.reasoning_content).toBe('Hmm.') + }) + + it.each([0, 272])( + 'normalizes top-level reasoning usage (%s tokens)', + async (reasoningTokens) => { + const usage = { + prompt_tokens: 180, + completion_tokens: 7, + total_tokens: 187 + reasoningTokens, + reasoning_tokens: reasoningTokens, + completion_tokens_details: { audio_tokens: 2 }, + } + const fetch = normalizeNeonContent(async () => + Response.json({ choices: [], usage }), + ) + const body = await (await fetch('https://x')).json() + expect(body.usage.completion_tokens_details).toEqual({ + audio_tokens: 2, + reasoning_tokens: reasoningTokens, + }) + }, + ) + + it('preserves standard nested reasoning usage when both fields exist', async () => { + const fetch = normalizeNeonContent(async () => + Response.json({ + choices: [], + usage: { + reasoning_tokens: 272, + completion_tokens_details: { reasoning_tokens: 0, audio_tokens: 2 }, + }, + }), + ) + const body = await (await fetch('https://x')).json() + expect(body.usage.completion_tokens_details).toEqual({ + reasoning_tokens: 0, + audio_tokens: 2, + }) + }) + + it('leaves string content and error responses alone', async () => { + const plain = chunk({ content: 'Hi' }) + const fetch = normalizeNeonContent(async () => sseResponse([plain])) + expect(await (await fetch('https://x')).text()).toBe(plain) + + const error = new Response('{"error":{"message":"no"}}', { status: 403 }) + const failing = normalizeNeonContent(async () => error) + expect(await failing('https://x')).toBe(error) + }) + + it('gives chatStream text and reasoning, also through wrapFetch', async () => { + const wrapFetch: FetchWrapper = (next) => (input, init) => next(input, init) + const adapter = createNeonText('gemini-3-8-flash', 'nt-test', { + baseURL: HOST, + fetch: async () => + sseResponse([ + arrayDelta, + chunk({}, 'stop'), + `data: ${JSON.stringify({ + choices: [], + usage: { + prompt_tokens: 180, + completion_tokens: 7, + total_tokens: 459, + reasoning_tokens: 272, + }, + })}\n\n`, + 'data: [DONE]\n\n', + ]), + }) + + const events: Array<{ type: string; delta?: string }> = [] + for await (const event of adapter.chatStream({ + logger, + model: 'gemini-3-8-flash', + threadId: 'thread-1', + messages: [{ role: 'user', content: 'Hi' }], + wrapFetch, + })) { + events.push(event as { type: string; delta?: string }) + } + + const types = events.map((event) => event.type) + expect(types).not.toContain(EventType.RUN_ERROR) + expect( + events.find((event) => event.type === EventType.RUN_FINISHED), + ).toMatchObject({ + usage: { completionTokensDetails: { reasoningTokens: 272 } }, + }) + expect( + events.find((event) => event.type === EventType.TEXT_MESSAGE_CONTENT) + ?.delta, + ).toBe('Hello') + expect( + events.find((event) => event.type === EventType.REASONING_MESSAGE_CONTENT) + ?.delta, + ).toBe('Thinking.') + }) +}) diff --git a/packages/ai-neon/tests/routing.test.ts b/packages/ai-neon/tests/routing.test.ts new file mode 100644 index 0000000000..2443a22215 --- /dev/null +++ b/packages/ai-neon/tests/routing.test.ts @@ -0,0 +1,330 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { resolveDebugOption } from '@tanstack/ai/adapter-internals' +import { AnthropicTextAdapter } from '@tanstack/ai-anthropic' +import { + buildNeonText, + createNeonText, + neonText, +} from '../src/adapters/factory' +import { NeonTextAdapter } from '../src/adapters/text' +import { NeonResponsesTextAdapter } from '../src/adapters/responses-text' +import type { TextOptions } from '@tanstack/ai' + +const logger = resolveDebugOption(false) +const messages = [{ role: 'user' as const, content: 'Hi' }] +const HOST = 'https://br-test-api.ai.c-1.us-east-2.aws.neon.tech' + +/** A fetch that records each request and answers with an empty stream. */ +function captureFetch() { + const sent: Array<{ url: string; headers: Headers; body: any }> = [] + const fetch: typeof globalThis.fetch = async (input, init) => { + const request = new Request(input, init) + sent.push({ + url: request.url, + headers: request.headers, + body: init?.body ? JSON.parse(String(init.body)) : undefined, + }) + return new Response('', { + headers: { 'content-type': 'text/event-stream' }, + }) + } + return { fetch, sent } +} + +async function drain( + adapter: { chatStream: (options: TextOptions) => AsyncIterable }, + model: string, + extra: Partial = {}, +) { + const events: Array = [] + for await (const event of adapter.chatStream({ + logger, + model, + threadId: 'thread-1', + messages, + ...extra, + })) { + events.push(event) + } + return events +} + +describe('neon text routing', () => { + afterEach(() => { + vi.unstubAllEnvs() + }) + + it('sends Claude to the Anthropic Messages route with a bearer token', async () => { + const { fetch, sent } = captureFetch() + const adapter = createNeonText('claude-sonnet-5-5', 'nt-test', { + baseURL: HOST, + fetch, + }) + expect(adapter).toBeInstanceOf(AnthropicTextAdapter) + expect(adapter.provider).toBe('neon') + + await drain(adapter, 'claude-sonnet-5-5') + + expect(sent).toHaveLength(1) + // The SDK's beta client adds `?beta=true`. + expect(sent[0]?.url).toMatch( + new RegExp( + `^${HOST.replaceAll('.', '\\.')}/anthropic/v1/messages(\\?|$)`, + ), + ) + expect(sent[0]?.headers.get('authorization')).toBe('Bearer nt-test') + expect(sent[0]?.headers.get('x-api-key')).toBeNull() + }) + + it('sends a Responses-only model to the Responses route', async () => { + const { fetch, sent } = captureFetch() + const adapter = createNeonText('gpt-5-3-codex', 'nt-test', { + baseURL: HOST, + fetch, + }) + expect(adapter).toBeInstanceOf(NeonResponsesTextAdapter) + + await drain(adapter, 'gpt-5-3-codex') + + expect(sent[0]?.url).toBe(`${HOST}/openai/v1/responses`) + expect(sent[0]?.headers.get('authorization')).toBe('Bearer nt-test') + }) + + it('sends a GPT model to Chat Completions unless api is responses', async () => { + const chat = captureFetch() + const chatAdapter = createNeonText('gpt-5-4', 'nt-test', { + baseURL: HOST, + fetch: chat.fetch, + }) + expect(chatAdapter).toBeInstanceOf(NeonTextAdapter) + await drain(chatAdapter, 'gpt-5-4') + expect(chat.sent[0]?.url).toBe(`${HOST}/v1/chat/completions`) + + const responses = captureFetch() + const responsesAdapter = createNeonText('gpt-5-4', 'nt-test', { + api: 'responses', + baseURL: HOST, + fetch: responses.fetch, + }) + expect(responsesAdapter).toBeInstanceOf(NeonResponsesTextAdapter) + await drain(responsesAdapter, 'gpt-5-4') + expect(responses.sent[0]?.url).toBe(`${HOST}/openai/v1/responses`) + }) + + it('sends every other model to Chat Completions and tolerates a trailing slash', async () => { + const { fetch, sent } = captureFetch() + await drain( + createNeonText('gemini-3-8-flash', 'nt-test', { + baseURL: `${HOST}/`, + fetch, + }), + 'gemini-3-8-flash', + ) + expect(sent[0]?.url).toBe(`${HOST}/v1/chat/completions`) + }) + + it('reads the token and host from the environment', async () => { + vi.stubEnv('NEON_AI_GATEWAY_TOKEN', 'nt-env') + vi.stubEnv('NEON_AI_GATEWAY_BASE_URL', HOST) + const { fetch, sent } = captureFetch() + + await drain(neonText('gemini-3-8-flash', { fetch }), 'gemini-3-8-flash') + + expect(sent[0]?.url).toBe(`${HOST}/v1/chat/completions`) + expect(sent[0]?.headers.get('authorization')).toBe('Bearer nt-env') + }) + + it('names the missing variable', () => { + vi.stubEnv('NEON_AI_GATEWAY_TOKEN', '') + vi.stubEnv('NEON_AI_GATEWAY_BASE_URL', '') + expect(() => neonText('gemini-3-8-flash')).toThrow(/NEON_AI_GATEWAY_TOKEN/) + expect(() => createNeonText('gemini-3-8-flash', 'nt-test')).toThrow( + /NEON_AI_GATEWAY_BASE_URL/, + ) + }) +}) + +describe('neon request body', () => { + it('reports only the input modalities supported by each route', () => { + const config = { baseURL: HOST } + expect( + createNeonText('gemini-3-8-flash', 'nt-test', config).inputModalities, + ).toEqual(['text', 'image']) + expect( + createNeonText('meta-llama-3-3-70b-instruct', 'nt-test', config) + .inputModalities, + ).toEqual(['text']) + expect( + createNeonText('gpt-5-4', 'nt-test', { ...config, api: 'responses' }) + .inputModalities, + ).toContain('document') + expect( + createNeonText('claude-opus-5-5', 'nt-test', config).inputModalities, + ).toContain('document') + }) + + it('rejects a route that cannot serve the model, including untyped callers', () => { + expect(() => + buildNeonText('gpt-5-3-codex', 'nt-test', { baseURL: HOST, api: 'chat' }), + ).toThrow(/requires the Responses API/) + expect(() => + buildNeonText('gemini-3-8-flash', 'nt-test', { + baseURL: HOST, + api: 'responses', + }), + ).toThrow(/does not support the Responses API/) + expect(() => + buildNeonText('claude-opus-5-5', 'nt-test', { + baseURL: HOST, + api: 'chat', + }), + ).toThrow(/uses the Anthropic Messages API/) + }) + + it('drops rejected Claude sampling settings, including through wrapFetch', async () => { + const { fetch, sent } = captureFetch() + const modelOptions = { temperature: 0.2, top_p: 0.9 } + const events = await drain( + createNeonText('claude-opus-5-5', 'nt-test', { baseURL: HOST, fetch }), + 'claude-opus-5-5', + { + modelOptions, + wrapFetch: (next) => (input, init) => next(input, init), + }, + ) + expect(events).not.toContainEqual( + expect.objectContaining({ type: 'RUN_ERROR' }), + ) + expect(sent).toHaveLength(1) + expect(sent[0]?.body).not.toHaveProperty('temperature') + expect(sent[0]?.body).not.toHaveProperty('top_p') + expect(modelOptions).toEqual({ temperature: 0.2, top_p: 0.9 }) + + const accepts = captureFetch() + await drain( + createNeonText('claude-sonnet-4-5', 'nt-test', { + baseURL: HOST, + fetch: accepts.fetch, + }), + 'claude-sonnet-4-5', + { modelOptions: { temperature: 0.2 } }, + ) + expect(accepts.sent[0]?.body.temperature).toBe(0.2) + }) + + it('drops rejected Claude sampling settings on structured output', async () => { + const bodies: Array> = [] + const model = 'claude-opus-5-5' + const adapter = createNeonText(model, 'nt-test', { + baseURL: HOST, + fetch: async (_input, init) => { + bodies.push(JSON.parse(String(init?.body))) + return Response.json({ + id: 'msg-neon', + type: 'message', + role: 'assistant', + model, + content: [ + { + type: 'tool_use', + id: 'tool-neon', + name: 'structured_output', + input: { ok: true }, + }, + ], + stop_reason: 'tool_use', + usage: { input_tokens: 3, output_tokens: 5 }, + }) + }, + }) + const modelOptions = { max_tokens: 73, temperature: 0.2, top_p: 0.9 } + const result = await adapter.structuredOutput({ + outputSchema: { type: 'object', properties: { ok: { type: 'boolean' } } }, + chatOptions: { + model, + threadId: 'thread-1', + messages, + logger, + modelOptions, + }, + }) + expect(result.data).toEqual({ ok: true }) + expect(bodies).toHaveLength(1) + expect(bodies[0]?.max_tokens).toBe(73) + expect(bodies[0]).not.toHaveProperty('temperature') + expect(bodies[0]).not.toHaveProperty('top_p') + expect(modelOptions).toEqual({ + max_tokens: 73, + temperature: 0.2, + top_p: 0.9, + }) + }) + + it('sends the model reasoning effort on Chat Completions', async () => { + const { fetch, sent } = captureFetch() + await drain( + createNeonText('gemini-3-8-flash', 'nt-test', { baseURL: HOST, fetch }), + 'gemini-3-8-flash', + { reasoning: { level: 'high', summary: true } }, + ) + expect(sent[0]?.body.reasoning_effort).toBe('high') + }) + + it('sends stream_options except to Gemini models', async () => { + const gemini = captureFetch() + await drain( + createNeonText('gemini-3-8-flash', 'nt-test', { + baseURL: HOST, + fetch: gemini.fetch, + }), + 'gemini-3-8-flash', + ) + expect(gemini.sent[0]?.body).not.toHaveProperty('stream_options') + + const gpt = captureFetch() + await drain( + createNeonText('gpt-5-4', 'nt-test', { baseURL: HOST, fetch: gpt.fetch }), + 'gpt-5-4', + ) + expect(gpt.sent[0]?.body.stream_options).toEqual({ include_usage: true }) + }) + + it('drops temperature and top_p for a model that rejects them', async () => { + const modelOptions = { temperature: 0.2, top_p: 0.9 } + + const rejects = captureFetch() + await drain( + createNeonText('gpt-5-5', 'nt-test', { + baseURL: HOST, + fetch: rejects.fetch, + }), + 'gpt-5-5', + { modelOptions }, + ) + expect(rejects.sent[0]?.body).not.toHaveProperty('temperature') + expect(rejects.sent[0]?.body).not.toHaveProperty('top_p') + + const responses = captureFetch() + await drain( + createNeonText('gpt-5-5-pro', 'nt-test', { + baseURL: HOST, + fetch: responses.fetch, + }), + 'gpt-5-5-pro', + { modelOptions }, + ) + expect(responses.sent[0]?.body).not.toHaveProperty('temperature') + + const accepts = captureFetch() + await drain( + createNeonText('gpt-5-4', 'nt-test', { + baseURL: HOST, + fetch: accepts.fetch, + }), + 'gpt-5-4', + { modelOptions }, + ) + expect(accepts.sent[0]?.body.temperature).toBe(0.2) + expect(accepts.sent[0]?.body.top_p).toBe(0.9) + }) +}) diff --git a/packages/ai-neon/tests/types.test-d.ts b/packages/ai-neon/tests/types.test-d.ts new file mode 100644 index 0000000000..eed74c444f --- /dev/null +++ b/packages/ai-neon/tests/types.test-d.ts @@ -0,0 +1,165 @@ +import { chat, summarize } from '@tanstack/ai' +import { expectTypeOf } from 'vitest' +import { + createNeonText, + createNeonSummarize, + neonText, + neonSummarize, + NEON_CHAT_MODELS, +} from '../src' +import type { + NeonAnthropicTextAdapter, + NeonResponsesTextAdapter, + NeonTextAdapter, +} from '../src' + +const messages = [{ role: 'user' as const, content: 'hi' }] + +// Each model gets the adapter of its route. +expectTypeOf(createNeonText('claude-sonnet-5-5', 'k')).toEqualTypeOf< + NeonAnthropicTextAdapter<'claude-sonnet-5-5'> +>() +expectTypeOf(createNeonText('gpt-5-3-codex', 'k')).toEqualTypeOf< + NeonResponsesTextAdapter<'gpt-5-3-codex'> +>() +expectTypeOf( + createNeonText('gpt-5-4', 'k', { api: 'responses' }), +).toEqualTypeOf>() +expectTypeOf(createNeonText('gemini-3-8-flash', 'k')).toEqualTypeOf< + NeonTextAdapter<'gemini-3-8-flash'> +>() + +// @ts-expect-error the Responses route does not serve Gemini +createNeonText('gemini-3-8-flash', 'k', { api: 'responses' }) + +// @ts-expect-error a Responses-only model cannot select Chat Completions +createNeonText('gpt-5-3-codex', 'k', { api: 'chat' }) +// @ts-expect-error the environment factory has the same route restriction +neonText('gpt-5-5-pro', { api: 'chat' }) +// @ts-expect-error Claude always uses the Anthropic Messages route +createNeonText('claude-opus-5-5', 'k', { api: 'chat' }) +// @ts-expect-error Claude cannot take OpenAI route options +neonText('claude-opus-5-5', { api: 'responses' }) + +expectTypeOf( + createNeonText('gemini-3-8-flash', 'k')['~types']['inputModalities'], +).toEqualTypeOf() +expectTypeOf( + createNeonText('gpt-5-4', 'k', { api: 'responses' })['~types'][ + 'inputModalities' + ], +).toEqualTypeOf() + +chat({ + adapter: createNeonText('gpt-5-3-codex', 'k'), + messages, + threadId: 'thread-1', + // @ts-expect-error the Responses API uses max_output_tokens + modelOptions: { max_tokens: 10 }, +}) + +chat({ + adapter: createNeonText('gemini-3-8-flash', 'k'), + messages, + threadId: 'thread-1', + reasoning: 'high', +}) + +chat({ + adapter: createNeonText('gemini-3-8-flash', 'k'), + messages, + threadId: 'thread-1', + // @ts-expect-error gemini-3-8-flash has no max level + reasoning: 'max', +}) + +chat({ + adapter: createNeonText('gpt-5-4', 'k'), + messages, + threadId: 'thread-1', + // @ts-expect-error the gateway's OpenAI routes take no token budget + reasoning: { level: 'high', budgetTokens: 4000 }, +}) + +chat({ + adapter: createNeonText('meta-llama-3-3-70b-instruct', 'k'), + messages, + threadId: 'thread-1', + // @ts-expect-error the model does not reason + reasoning: 'low', +}) + +// Claude keeps the Anthropic adapter's token budget. +chat({ + adapter: createNeonText('claude-sonnet-4-5', 'k'), + messages, + threadId: 'thread-1', + reasoning: { level: 'high', budgetTokens: 4000 }, +}) + +// Each route's summarize adapter fits `summarize()`. +void summarize({ + adapter: neonSummarize('gpt-6-1-sol'), + threadId: 'thread-1', + text: 'A long text.', +}) +void summarize({ + adapter: neonSummarize('gpt-5-3-codex'), + threadId: 'thread-1', + text: 'A long text.', +}) +void summarize({ + adapter: neonSummarize('claude-opus-5-5'), + threadId: 'thread-1', + text: 'A long text.', +}) + +// A runtime selection from the full catalog keeps automatic routing. +for (const model of NEON_CHAT_MODELS) { + chat({ adapter: createNeonText(model, 'k'), messages, threadId: 'thread-1' }) + chat({ adapter: neonText(model), messages, threadId: 'thread-1' }) + void summarize({ + adapter: neonSummarize(model), + threadId: 'thread-1', + text: 'A long text.', + }) + // @ts-expect-error not every member of the model union supports Chat Completions + createNeonText(model, 'k', { api: 'chat' }) + // @ts-expect-error not every member of the model union supports Responses + neonText(model, { api: 'responses' }) +} + +// Summarize enforces the same model/route pairs as chat. +// @ts-expect-error Gemini does not support Responses +createNeonSummarize('gemini-3-8-flash', 'k', { api: 'responses' }) +// @ts-expect-error Claude uses Anthropic Messages +neonSummarize('claude-opus-5-5', { api: 'responses' }) +// @ts-expect-error a Responses-only model cannot select Chat Completions +neonSummarize('gpt-5-3-codex', { api: 'chat' }) + +void summarize({ + adapter: createNeonSummarize('gpt-6-1-sol', 'k', { api: 'responses' }), + threadId: 'thread-1', + text: 'A long text.', + modelOptions: { max_output_tokens: 100, truncation: 'auto' }, +}) +void summarize({ + adapter: neonSummarize('gpt-6-1-sol', { api: 'responses' }), + threadId: 'thread-1', + text: 'A long text.', + // @ts-expect-error Responses uses max_output_tokens + modelOptions: { max_tokens: 100 }, +}) +void summarize({ + adapter: neonSummarize('gpt-6-1-sol'), + threadId: 'thread-1', + text: 'A long text.', + modelOptions: { max_tokens: 100 }, +}) +void summarize({ + adapter: neonSummarize('gpt-6-1-sol'), + threadId: 'thread-1', + text: 'A long text.', + // @ts-expect-error Chat Completions uses max_tokens + modelOptions: { max_output_tokens: 100 }, +}) diff --git a/packages/ai-neon/tsconfig.json b/packages/ai-neon/tsconfig.json new file mode 100644 index 0000000000..c38689f4ea --- /dev/null +++ b/packages/ai-neon/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "dist" + }, + "include": ["src", "tests"], + "exclude": ["node_modules", "dist"] +} diff --git a/packages/ai-neon/vite.config.ts b/packages/ai-neon/vite.config.ts new file mode 100644 index 0000000000..77bcc2e60b --- /dev/null +++ b/packages/ai-neon/vite.config.ts @@ -0,0 +1,36 @@ +import { defineConfig, mergeConfig } from 'vitest/config' +import { tanstackViteConfig } from '@tanstack/vite-config' +import packageJson from './package.json' + +const config = defineConfig({ + test: { + name: packageJson.name, + dir: './', + watch: false, + globals: true, + environment: 'node', + include: ['tests/**/*.test.ts'], + coverage: { + provider: 'v8', + reporter: ['text', 'json', 'html', 'lcov'], + exclude: [ + 'node_modules/', + 'dist/', + 'tests/', + '**/*.test.ts', + '**/*.config.ts', + '**/types.ts', + ], + include: ['src/**/*.ts'], + }, + }, +}) + +export default mergeConfig( + config, + tanstackViteConfig({ + entry: ['./src/index.ts'], + srcDir: './src', + cjs: false, + }), +) diff --git a/packages/ai/skills/ai-core/adapter-configuration/SKILL.md b/packages/ai/skills/ai-core/adapter-configuration/SKILL.md index 6697116346..3d56591c2a 100644 --- a/packages/ai/skills/ai-core/adapter-configuration/SKILL.md +++ b/packages/ai/skills/ai-core/adapter-configuration/SKILL.md @@ -91,6 +91,7 @@ The text adapter is the primary one for chat/completions: | BytePlus | `@tanstack/ai-byteplus` | `byteplusText` | `ARK_API_KEY` (falls back to `BYTEPLUS_API_KEY`) | | OpenAI-compatible | `@tanstack/ai-openai/compatible` | `openaiCompatible` / `openaiCompatibleText` | provider-specific (passed via `apiKey`) | | Cloudflare | `@tanstack/ai-cloudflare` | `cloudflareText` | `CLOUDFLARE_ACCOUNT_ID` + `CLOUDFLARE_API_TOKEN`, or `{ binding: env.AI }` in a Worker | +| Neon AI Gateway | `@tanstack/ai-neon` | `neonText` | `NEON_AI_GATEWAY_TOKEN` + `NEON_AI_GATEWAY_BASE_URL` (branch host) | > **BytePlus uses two keys.** `byteplusText` / `byteplusVideo` / > `byteplusImage` read `ARK_API_KEY` (ModelArk, `Authorization: Bearer`), but diff --git a/packages/openai-base/src/adapters/chat-completions-text.ts b/packages/openai-base/src/adapters/chat-completions-text.ts index 6cad6dc60c..1268d75209 100644 --- a/packages/openai-base/src/adapters/chat-completions-text.ts +++ b/packages/openai-base/src/adapters/chat-completions-text.ts @@ -175,6 +175,7 @@ export abstract class OpenAIBaseChatCompletionsTextAdapter< `activity=chat provider=${this.name} model=${this.model} messages=${options.messages.length} tools=${options.tools?.length ?? 0} stream=true`, { provider: this.name, model: this.model }, ) + const streamOptions = this.streamOptions(options.model) const stream = await clientFor( this.client, options, @@ -183,7 +184,7 @@ export abstract class OpenAIBaseChatCompletionsTextAdapter< { ...requestParams, stream: true, - stream_options: { include_usage: true }, + ...(streamOptions && { stream_options: streamOptions }), }, extractRequestOptions(options.request), ) @@ -500,6 +501,7 @@ export abstract class OpenAIBaseChatCompletionsTextAdapter< { provider: this.name, model: this.model }, ) + const streamOptions = this.streamOptions(chatOptions.model) const stream = await clientFor( this.client, chatOptions, @@ -508,7 +510,7 @@ export abstract class OpenAIBaseChatCompletionsTextAdapter< { ...cleanParams, stream: true, - stream_options: { include_usage: true }, + ...(streamOptions && { stream_options: streamOptions }), response_format: responseFormat, }, extractRequestOptions(chatOptions.request), @@ -1437,6 +1439,17 @@ export abstract class OpenAIBaseChatCompletionsTextAdapter< return undefined } + /** + * The `stream_options` of a streaming request. The default asks for the + * usage-only tail chunk. A subclass returns `undefined` for a backend that + * rejects the field (it then sends usage on its own, or none). + */ + protected streamOptions( + _model: string, + ): OpenAI.Chat.Completions.ChatCompletionStreamOptions | undefined { + return { include_usage: true } + } + /** * Maps common TextOptions to Chat Completions API request format. * Override this in subclasses to add provider-specific options. diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index eae2381e7a..cb2f1b2686 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -3092,6 +3092,31 @@ importers: specifier: ^8.2.1 version: 8.2.1(@types/node@24.10.3)(esbuild@0.28.1)(jiti@2.7.0)(less@4.6.6)(sass@1.101.0)(terser@5.44.1)(tsx@4.23.12)(yaml@2.9.0) + packages/ai-neon: + dependencies: + '@tanstack/ai-anthropic': + specifier: workspace:^ + version: link:../ai-anthropic + '@tanstack/ai-utils': + specifier: workspace:^ + version: link:../ai-utils + '@tanstack/openai-base': + specifier: workspace:^ + version: link:../openai-base + openai: + specifier: ^7.30.0 + version: 7.30.0(@aws-sdk/credential-provider-node@3.972.58)(@smithy/signature-v4@5.5.2)(undici@8.10.2)(ws@8.21.3)(zod@4.3.6) + devDependencies: + '@tanstack/ai': + specifier: workspace:* + version: link:../ai + '@vitest/coverage-v8': + specifier: 4.1.10 + version: 4.1.10(vitest@4.1.11) + vite: + specifier: ^8.2.1 + version: 8.3.0(@types/node@24.13.6)(esbuild@0.28.1)(jiti@2.7.0)(less@4.6.6)(sass@1.101.0)(terser@5.44.1)(tsx@4.23.12)(yaml@2.9.0) + packages/ai-octane: dependencies: '@tanstack/ai-client': @@ -4019,6 +4044,9 @@ importers: '@tanstack/ai-mistral': specifier: workspace:* version: link:../../packages/ai-mistral + '@tanstack/ai-neon': + specifier: workspace:* + version: link:../../packages/ai-neon '@tanstack/ai-ollama': specifier: workspace:* version: link:../../packages/ai-ollama @@ -7401,6 +7429,7 @@ packages: '@opentelemetry/propagator-jaeger@2.7.1': resolution: {integrity: sha512-KMjVBHzP4N60bOzxja76M1F1hZZ43lGPga5ix+mkv9+kk1nx9SbkxSvJsMbuVUxdPQmsPTqGShmhN8ulrMOg6Q==} engines: {node: ^18.19.0 || >=20.6.0} + deprecated: Package no longer supported. See https://github.com/open-telemetry/opentelemetry-js/pull/7077 peerDependencies: '@opentelemetry/api': '>=1.0.0 <1.10.0' @@ -7451,6 +7480,7 @@ packages: '@opentelemetry/sdk-trace-node@2.7.1': resolution: {integrity: sha512-pCpQxU68lV+I9s9svqMyVu5iHdDDUnqUpSxqwyCU8A9ejEsSnMPCbearwsUO4yk08ZJzAIUCFuReMdVQvHrdvg==} engines: {node: ^18.19.0 || >=20.6.0} + deprecated: Use @opentelemetry/sdk-trace package peerDependencies: '@opentelemetry/api': '>=1.0.0 <1.10.0' diff --git a/scripts/convert-neon-models.ts b/scripts/convert-neon-models.ts new file mode 100644 index 0000000000..4d51e81703 --- /dev/null +++ b/scripts/convert-neon-models.ts @@ -0,0 +1,257 @@ +/** + * Converts Neon AI Gateway catalog rows into packages/ai-neon/src/model-meta.ts. + * + * Usage: + * pnpm tsx scripts/convert-neon-models.ts + */ + +import { writeFile } from 'node:fs/promises' +import { dirname, resolve } from 'node:path' +import { fileURLToPath } from 'node:url' +import { ANTHROPIC_MODELS } from '../packages/ai-anthropic/src/model-meta' +import { models } from './neon.models' +import { + reasoningShapeName, + reasoningShapeTable, +} from './model-sync/reasoning-shapes' +import type { NeonCatalogModel } from './neon.models' +import type { ReasoningLevel, ReasoningLevels } from '@tanstack/ai' +import type { ReasoningShape } from './model-sync/reasoning-shapes' + +const __dirname = dirname(fileURLToPath(import.meta.url)) +const OUTPUT_PATH = resolve(__dirname, '../packages/ai-neon/src/model-meta.ts') + +type InputModality = 'text' | 'image' | 'audio' | 'video' | 'document' + +const INPUT_MODALITY_ORDER: Array = [ + 'text', + 'image', + 'document', + 'audio', + 'video', +] + +const MODALITY_MAP: Record = { + text: 'text', + image: 'image', + audio: 'audio', + video: 'video', + pdf: 'document', + file: 'document', + document: 'document', +} + +// The effort values that are TanStack levels of the same name. `none` is `off`. +const LEVELS = new Set([ + 'minimal', + 'low', + 'medium', + 'high', + 'xhigh', + 'max', +]) + +function formatStringArray(ids: Array): string { + if (ids.length === 0) return '[] as const' + return `[\n${ids.map((id) => ` ${JSON.stringify(id)},`).join('\n')}\n] as const` +} + +function inputModalitiesFor(model: NeonCatalogModel): Array { + const mapped = new Set(['text']) + for (const modality of model.modalities?.input ?? []) { + const known = MODALITY_MAP[modality.toLowerCase()] + if (known) mapped.add(known) + } + return INPUT_MODALITY_ORDER.filter((modality) => mapped.has(modality)) +} + +/** + * The model's `chat({ reasoning })` data, from its `effort` values. Both + * OpenAI-shaped routes send one effort value (`reasoning_effort`, or + * `reasoning.effort` on Responses), so no model takes a token budget here. + */ +function reasoningFor(model: NeonCatalogModel): ReasoningShape | undefined { + if (!model.reasoning) return undefined + const effort = model.reasoning_options?.find( + (option) => option.type === 'effort', + ) + const levels: ReasoningLevels = {} + for (const value of effort?.values ?? []) { + if (value === 'none') levels.off = 'none' + else if (LEVELS.has(value)) levels[value as ReasoningLevel] = value + } + if (Object.keys(levels).length === 0) return undefined + return { levels, budget: false } +} + +function convertModels(): string { + const anthropicIds = new Set(ANTHROPIC_MODELS) + const chat: Array = [] + const embedding: Array = [] + let unreleased = 0 + let unserved = 0 + + for (const model of models) { + if (model.released !== true) { + unreleased += 1 + continue + } + const capabilities = model.capabilities ?? {} + if (model.type === 'embedding') { + if (capabilities.embeddings) embedding.push(model.id) + continue + } + if (capabilities.chat !== 'not-served' || capabilities.responses) { + chat.push(model) + } else { + unserved += 1 + } + } + + chat.sort((a, b) => a.id.localeCompare(b.id)) + embedding.sort((a, b) => a.localeCompare(b)) + + if (chat.length === 0) { + throw new Error( + 'The Neon catalog produced no chat models. Refusing to overwrite model-meta.ts.', + ) + } + if (embedding.length === 0) { + throw new Error( + 'The Neon catalog produced no embedding models. Refusing to overwrite model-meta.ts.', + ) + } + + // A Claude id that ai-anthropic does not know yet stays on Chat Completions + // until the provider-model sync adds it there. + const anthropic = chat.filter( + (model) => + model.capabilities?.native_dialect === 'anthropic' && + anthropicIds.has(model.id), + ) + const anthropicSet = new Set(anthropic.map((model) => model.id)) + const responses = chat.filter((model) => model.capabilities?.responses) + const responsesOnly = responses.filter( + (model) => model.capabilities?.chat === 'not-served', + ) + const rejectsSampling = chat.filter((model) => model.temperature === false) + // Measured against the gateway: every Gemini-dialect model 400s on + // `stream_options` on Chat Completions, and streams usage without it. + const rejectsStreamOptions = chat.filter( + (model) => model.capabilities?.native_dialect === 'gemini', + ) + + console.log( + `Classified ${models.length} models: chat=${chat.length} (anthropic=${anthropic.length} responses=${responses.length} responses-only=${responsesOnly.length}) embedding=${embedding.length} unreleased=${unreleased} unserved=${unserved}`, + ) + + const shapes: Array = [] + const reasoningEntries: Array = [] + for (const model of chat) { + // ai-anthropic owns the reasoning data of the models it serves. + if (anthropicSet.has(model.id)) continue + const shape = reasoningFor(model) + if (!shape) continue + shapes.push(shape) + reasoningEntries.push( + ` ${JSON.stringify(model.id)}: ${reasoningShapeName(shape)},`, + ) + } + + const ids = (rows: Array) => rows.map((model) => model.id) + const modalityEntries = chat + .map( + (model) => + ` ${JSON.stringify(model.id)}: [${inputModalitiesFor(model) + .map((modality) => `'${modality}'`) + .join(', ')}],`, + ) + .join('\n') + + return `/** + * This file is generated by scripts/convert-neon-models.ts. Do not edit by hand. + */ + +import type { ModelReasoning } from '@tanstack/ai' +import type { ModelReasoningCapability } from '@tanstack/ai/adapter-internals' + +/** Every chat model the gateway serves, on any route. */ +export const NEON_CHAT_MODELS = ${formatStringArray(ids(chat))} + +export type NeonChatModel = (typeof NEON_CHAT_MODELS)[number] + +/** Claude models. \`neonText\` sends them to the Anthropic Messages route. */ +export const NEON_ANTHROPIC_MODELS = ${formatStringArray(ids(anthropic))} + +export type NeonAnthropicModel = (typeof NEON_ANTHROPIC_MODELS)[number] + +/** Models the Responses route serves. They take \`api: 'responses'\`. */ +export const NEON_RESPONSES_MODELS = ${formatStringArray(ids(responses))} + +export type NeonResponsesModel = (typeof NEON_RESPONSES_MODELS)[number] + +/** Models only the Responses route serves. \`neonText\` always sends them there. */ +export const NEON_RESPONSES_ONLY_MODELS = ${formatStringArray(ids(responsesOnly))} + +export type NeonResponsesOnlyModel = (typeof NEON_RESPONSES_ONLY_MODELS)[number] + +export const NEON_EMBEDDING_MODELS = ${formatStringArray(embedding)} + +export type NeonEmbeddingModel = (typeof NEON_EMBEDDING_MODELS)[number] + +/** Models that reject \`temperature\` and \`top_p\`. The adapters drop both. */ +export const NEON_MODELS_REJECTING_SAMPLING: ReadonlySet = new Set(${formatStringArray(ids(rejectsSampling))}) + +/** + * Models that reject \`stream_options\` on Chat Completions. They stream + * usage without it. + */ +export const NEON_MODELS_REJECTING_STREAM_OPTIONS: ReadonlySet = new Set(${formatStringArray(ids(rejectsStreamOptions))}) + +/** The input modalities of each chat model. */ +export const NEON_MODEL_INPUT_MODALITIES = { +${modalityEntries} +} as const satisfies Record> + +export type NeonModelInputModalitiesByName = typeof NEON_MODEL_INPUT_MODALITIES + +// Reasoning +// +// Each model's data for \`chat({ reasoning })\`: the levels it has, with the +// effort value the gateway takes for each. Each model points at one shape. + +${reasoningShapeTable(shapes).code} + +const NEON_REASONING = { +${reasoningEntries.join('\n')} +} satisfies Partial> + +/** + * Each chat model's reasoning levels, for \`chat({ reasoning })\`. A model + * that is not here does not reason. + */ +export type NeonModelReasoningByName = { + [K in keyof typeof NEON_REASONING]: ModelReasoningCapability< + (typeof NEON_REASONING)[K] + > +} + +/** + * Runtime map from chat model id to its reasoning data, for the text + * adapters. An unknown id gives \`undefined\`: the adapter sends no reasoning + * field. + */ +export const NEON_MODEL_REASONING: Readonly> = + NEON_REASONING +` +} + +async function main() { + await writeFile(OUTPUT_PATH, convertModels(), 'utf-8') + console.log(`Model meta file written to ${OUTPUT_PATH}`) +} + +main().catch((error) => { + console.error(error) + process.exit(1) +}) diff --git a/scripts/fetch-neon-models.ts b/scripts/fetch-neon-models.ts new file mode 100644 index 0000000000..98e30b6730 --- /dev/null +++ b/scripts/fetch-neon-models.ts @@ -0,0 +1,85 @@ +/** + * Fetches the Neon AI Gateway model catalog and writes it to + * neon.models.json. + * + * Usage: + * pnpm tsx scripts/fetch-neon-models.ts + * + * The catalog is public. `https://neon.com/models` serves it as JSON when the + * request asks for JSON. That shape carries `capabilities` (which endpoint + * serves each model); `neon.com/models.json` (the models.dev shape) does not. + * + * The output is plain JSON so a malicious or compromised upstream response + * cannot smuggle executable code into the build. The committed wrapper at + * `neon.models.ts` re-exports this JSON typed as `Array`. + */ + +import { writeFile } from 'node:fs/promises' +import { dirname, resolve } from 'node:path' +import { fileURLToPath } from 'node:url' + +const __dirname = dirname(fileURLToPath(import.meta.url)) +const OUTPUT_PATH = resolve(__dirname, 'neon.models.json') +const API_URL = 'https://neon.com/models' +const SCHEMA = 'https://neon.com/schemas/ai-gateway-models.json' + +interface ApiModel { + id: string + [key: string]: unknown +} + +function isValidModel(model: unknown): model is ApiModel { + return ( + typeof model === 'object' && + model !== null && + typeof (model as { id?: unknown }).id === 'string' && + (model as { id: string }).id.length > 0 + ) +} + +async function main() { + console.log(`Fetching models from ${API_URL}...`) + + const response = await fetch(API_URL, { + headers: { Accept: 'application/json' }, + signal: AbortSignal.timeout(30_000), + }) + if (!response.ok) { + throw new Error( + `Failed to fetch the Neon AI Gateway catalog: ${response.status} ${response.statusText}`, + ) + } + + const json = (await response.json()) as { + $schema?: unknown + models?: unknown + } + if (json.$schema !== SCHEMA) { + throw new Error( + `The Neon catalog has $schema ${JSON.stringify(json.$schema)}, expected ${SCHEMA}`, + ) + } + if (!Array.isArray(json.models)) { + throw new Error('The Neon catalog is missing a models array') + } + + const validModels = json.models.filter(isValidModel) + const skipped = json.models.length - validModels.length + if (skipped > 0) { + console.log(`Skipped ${skipped} models missing a string id`) + } + + // ponytail: `examples` is copy-paste code for the website, not model data. + const rows = validModels + .map(({ examples: _examples, ...model }) => model) + .sort((a, b) => a.id.localeCompare(b.id)) + + await writeFile(OUTPUT_PATH, JSON.stringify(rows, null, 2) + '\n', 'utf-8') + console.log(`Fetched ${rows.length} models`) + console.log(`Written to ${OUTPUT_PATH}`) +} + +main().catch((error) => { + console.error(error) + process.exit(1) +}) diff --git a/scripts/neon.models.json b/scripts/neon.models.json new file mode 100644 index 0000000000..23d55bfe65 --- /dev/null +++ b/scripts/neon.models.json @@ -0,0 +1,3014 @@ +[ + { + "id": "bge-large-en", + "name": "BGE Large (En)", + "provider": "baai", + "family": "text-embedding", + "released": true, + "type": "embedding", + "dimensions": 1024, + "open_weights": true, + "release_date": "2023-09-12", + "last_updated": "2023-09-12", + "cost": { + "input": 0.1 + }, + "capabilities": { + "entitled": true, + "chat": "not-served", + "deviations": [], + "native_dialect": "none", + "responses": false, + "web_search": false, + "image_generation": false, + "embeddings": true + }, + "use_case": "chat", + "unsupported": "bge-large-en does not serve chat completions. It is an embedding model — see the embeddings use case." + }, + { + "id": "claude-fable-5", + "name": "Claude Fable 5", + "provider": "anthropic", + "family": "claude-fable", + "released": true, + "attachment": true, + "reasoning": true, + "reasoning_options": [ + { + "type": "effort", + "values": ["low", "medium", "high", "xhigh", "max"] + } + ], + "tool_call": true, + "temperature": false, + "structured_output": true, + "open_weights": false, + "knowledge": "2026-01-31", + "release_date": "2026-06-09", + "last_updated": "2026-06-09", + "modalities": { + "input": ["text", "image", "pdf"], + "output": ["text"] + }, + "limit": { + "context": 1000000, + "output": 128000 + }, + "cost": { + "input": 10, + "output": 50, + "cache_read": 1, + "cache_write": 12.5 + }, + "blended_cost": 20, + "capabilities": { + "entitled": true, + "chat": "conforms", + "deviations": [], + "native_dialect": "anthropic", + "responses": false, + "web_search": false, + "image_generation": false, + "embeddings": false + }, + "use_case": "chat" + }, + { + "id": "claude-fable-5-1", + "name": "Claude Fable 5.1", + "provider": "anthropic", + "family": "claude-fable", + "released": true, + "attachment": true, + "reasoning": true, + "reasoning_options": [ + { + "type": "effort", + "values": ["low", "medium", "high", "xhigh", "max"] + } + ], + "tool_call": true, + "temperature": false, + "structured_output": false, + "open_weights": false, + "knowledge": "2026-06", + "release_date": "2026-09-01", + "last_updated": "2026-09-01", + "modalities": { + "input": ["text", "image", "pdf"], + "output": ["text"] + }, + "limit": { + "context": 1000000, + "output": 128000 + }, + "cost": { + "input": 10, + "output": 50, + "cache_read": 0.25, + "cache_write": 12.5 + }, + "blended_cost": 20, + "capabilities": { + "entitled": true, + "chat": "conforms", + "deviations": [], + "native_dialect": "anthropic", + "responses": false, + "web_search": false, + "image_generation": false, + "embeddings": false + }, + "use_case": "chat" + }, + { + "id": "claude-haiku-4-5", + "name": "Claude Haiku 4.5", + "provider": "anthropic", + "family": "claude-haiku", + "released": true, + "attachment": true, + "reasoning": true, + "reasoning_options": [ + { + "type": "toggle" + }, + { + "type": "budget_tokens", + "min": 1024, + "max": 63999 + } + ], + "tool_call": true, + "temperature": true, + "structured_output": true, + "open_weights": false, + "knowledge": "2025-02-28", + "release_date": "2025-10-15", + "last_updated": "2025-10-15", + "modalities": { + "input": ["text", "image", "pdf"], + "output": ["text"] + }, + "limit": { + "context": 200000, + "output": 64000 + }, + "cost": { + "input": 1, + "output": 5, + "cache_read": 0.1, + "cache_write": 1.25 + }, + "blended_cost": 2, + "capabilities": { + "entitled": true, + "chat": "conforms", + "deviations": [], + "native_dialect": "anthropic", + "responses": false, + "web_search": false, + "image_generation": false, + "embeddings": false + }, + "use_case": "chat" + }, + { + "id": "claude-haiku-5-5", + "name": "Claude Haiku 5.5", + "provider": "anthropic", + "family": "claude-haiku", + "released": true, + "attachment": true, + "reasoning": true, + "reasoning_options": [ + { + "type": "toggle" + }, + { + "type": "budget_tokens", + "min": 1024, + "max": 63999 + } + ], + "tool_call": true, + "temperature": false, + "structured_output": true, + "open_weights": false, + "release_date": "2026-10-07", + "last_updated": "2026-10-07", + "modalities": { + "input": ["text", "image", "pdf"], + "output": ["text"] + }, + "limit": { + "context": 200000, + "output": 64000 + }, + "cost": { + "input": 0.1, + "output": 0.5, + "cache_read": 0.01, + "cache_write": 0.125, + "tiers": [ + { + "input": 0.5, + "output": 2.5, + "cache_read": 0.05, + "cache_write": 0.625, + "tier": { + "type": "context", + "size": 100000 + } + } + ], + "context_over_200k": { + "input": 0.5, + "output": 2.5, + "cache_read": 0.05, + "cache_write": 0.625 + } + }, + "blended_cost": 0.2, + "capabilities": { + "entitled": true, + "chat": "array-content", + "deviations": [ + "`content` is an array[2] of {reasoning, text} with keys {summary, text, type}; the spec requires a string" + ], + "native_dialect": "anthropic", + "responses": false, + "web_search": false, + "image_generation": false, + "embeddings": false + }, + "use_case": "chat" + }, + { + "id": "claude-opus-4-1", + "name": "Claude Opus 4.1", + "provider": "anthropic", + "family": "claude-opus", + "released": true, + "attachment": true, + "reasoning": true, + "reasoning_options": [ + { + "type": "toggle" + }, + { + "type": "budget_tokens", + "min": 1024, + "max": 31999 + } + ], + "tool_call": true, + "temperature": true, + "structured_output": true, + "open_weights": false, + "knowledge": "2025-03-31", + "release_date": "2025-08-05", + "last_updated": "2025-08-05", + "modalities": { + "input": ["text", "image", "pdf"], + "output": ["text"] + }, + "limit": { + "context": 200000, + "output": 32000 + }, + "cost": { + "input": 15, + "output": 75, + "cache_read": 1.5, + "cache_write": 18.75 + }, + "blended_cost": 30, + "capabilities": { + "entitled": true, + "chat": "conforms", + "deviations": [], + "native_dialect": "anthropic", + "responses": false, + "web_search": false, + "image_generation": false, + "embeddings": false + }, + "use_case": "chat" + }, + { + "id": "claude-opus-4-5", + "name": "Claude Opus 4.5", + "provider": "anthropic", + "family": "claude-opus", + "released": true, + "attachment": true, + "reasoning": true, + "reasoning_options": [ + { + "type": "toggle" + }, + { + "type": "budget_tokens", + "min": 1024, + "max": 63999 + } + ], + "tool_call": true, + "temperature": true, + "structured_output": true, + "open_weights": false, + "knowledge": "2025-05", + "release_date": "2025-11-24", + "last_updated": "2025-11-24", + "modalities": { + "input": ["text", "image", "pdf"], + "output": ["text"] + }, + "limit": { + "context": 200000, + "output": 64000 + }, + "cost": { + "input": 5, + "output": 25, + "cache_read": 0.5, + "cache_write": 6.25 + }, + "blended_cost": 10, + "capabilities": { + "entitled": true, + "chat": "conforms", + "deviations": [], + "native_dialect": "anthropic", + "responses": false, + "web_search": false, + "image_generation": false, + "embeddings": false + }, + "use_case": "chat" + }, + { + "id": "claude-opus-4-6", + "name": "Claude Opus 4.6", + "provider": "anthropic", + "family": "claude-opus", + "released": true, + "attachment": true, + "reasoning": true, + "reasoning_options": [ + { + "type": "toggle" + }, + { + "type": "budget_tokens", + "min": 1024, + "max": 127999 + } + ], + "tool_call": true, + "temperature": true, + "structured_output": true, + "open_weights": false, + "knowledge": "2025-05-31", + "release_date": "2026-02-05", + "last_updated": "2026-03-13", + "modalities": { + "input": ["text", "image", "pdf"], + "output": ["text"] + }, + "limit": { + "context": 1000000, + "output": 128000 + }, + "cost": { + "input": 5, + "output": 25, + "cache_read": 0.5, + "cache_write": 6.25 + }, + "blended_cost": 10, + "capabilities": { + "entitled": true, + "chat": "conforms", + "deviations": [], + "native_dialect": "anthropic", + "responses": false, + "web_search": false, + "image_generation": false, + "embeddings": false + }, + "use_case": "chat" + }, + { + "id": "claude-opus-4-7", + "name": "Claude Opus 4.7", + "provider": "anthropic", + "family": "claude-opus", + "released": true, + "attachment": true, + "reasoning": true, + "reasoning_options": [ + { + "type": "toggle" + }, + { + "type": "effort", + "values": ["low", "medium", "high", "xhigh", "max"] + } + ], + "tool_call": true, + "temperature": false, + "structured_output": true, + "open_weights": false, + "knowledge": "2026-01-31", + "release_date": "2026-04-16", + "last_updated": "2026-04-16", + "modalities": { + "input": ["text", "image", "pdf"], + "output": ["text"] + }, + "limit": { + "context": 1000000, + "output": 128000 + }, + "cost": { + "input": 5, + "output": 25, + "cache_read": 0.5, + "cache_write": 6.25 + }, + "blended_cost": 10, + "capabilities": { + "entitled": true, + "chat": "conforms", + "deviations": [], + "native_dialect": "anthropic", + "responses": false, + "web_search": false, + "image_generation": false, + "embeddings": false + }, + "use_case": "chat" + }, + { + "id": "claude-opus-4-8", + "name": "Claude Opus 4.8", + "provider": "anthropic", + "family": "claude-opus", + "released": true, + "attachment": true, + "reasoning": true, + "reasoning_options": [ + { + "type": "toggle" + }, + { + "type": "effort", + "values": ["low", "medium", "high", "xhigh", "max"] + } + ], + "tool_call": true, + "temperature": false, + "structured_output": true, + "open_weights": false, + "knowledge": "2026-01", + "release_date": "2026-05-28", + "last_updated": "2026-05-28", + "modalities": { + "input": ["text", "image", "pdf"], + "output": ["text"] + }, + "limit": { + "context": 1000000, + "output": 128000 + }, + "cost": { + "input": 5, + "output": 25, + "cache_read": 0.5, + "cache_write": 6.25 + }, + "blended_cost": 10, + "capabilities": { + "entitled": true, + "chat": "conforms", + "deviations": [], + "native_dialect": "anthropic", + "responses": false, + "web_search": false, + "image_generation": false, + "embeddings": false + }, + "use_case": "chat" + }, + { + "id": "claude-opus-5", + "name": "Claude Opus 5", + "provider": "anthropic", + "family": "claude-opus", + "released": true, + "attachment": true, + "reasoning": true, + "reasoning_options": [ + { + "type": "toggle" + }, + { + "type": "effort", + "values": ["low", "medium", "high", "xhigh", "max"] + } + ], + "tool_call": true, + "temperature": false, + "structured_output": true, + "open_weights": false, + "knowledge": "2026-05", + "release_date": "2026-07-24", + "last_updated": "2026-07-24", + "modalities": { + "input": ["text", "image", "pdf"], + "output": ["text"] + }, + "limit": { + "context": 1000000, + "output": 128000 + }, + "cost": { + "input": 5, + "output": 25, + "cache_read": 0.5, + "cache_write": 6.25 + }, + "blended_cost": 10, + "capabilities": { + "entitled": true, + "chat": "conforms", + "deviations": [], + "native_dialect": "anthropic", + "responses": false, + "web_search": false, + "image_generation": false, + "embeddings": false + }, + "use_case": "chat" + }, + { + "id": "claude-opus-5-5", + "name": "Claude Opus 5.5", + "provider": "anthropic", + "family": "claude-opus", + "released": true, + "attachment": true, + "reasoning": true, + "reasoning_options": [ + { + "type": "effort", + "values": ["low", "medium", "high", "xhigh", "max"] + } + ], + "tool_call": true, + "temperature": false, + "structured_output": false, + "open_weights": false, + "knowledge": "2026-06", + "release_date": "2026-09-22", + "last_updated": "2026-09-22", + "modalities": { + "input": ["text", "image", "pdf"], + "output": ["text"] + }, + "limit": { + "context": 1000000, + "output": 128000 + }, + "cost": { + "input": 4, + "output": 20, + "cache_read": 0.2, + "cache_write": 5 + }, + "blended_cost": 8, + "capabilities": { + "entitled": true, + "chat": "array-content", + "deviations": [ + "`content` is an array[2] of {reasoning, text} with keys {summary, text, type}; the spec requires a string" + ], + "native_dialect": "anthropic", + "responses": false, + "web_search": false, + "image_generation": false, + "embeddings": false + }, + "use_case": "chat" + }, + { + "id": "claude-sonnet-4-5", + "name": "Claude Sonnet 4.5", + "provider": "anthropic", + "family": "claude-sonnet", + "released": true, + "attachment": true, + "reasoning": true, + "reasoning_options": [ + { + "type": "toggle" + }, + { + "type": "budget_tokens", + "min": 1024, + "max": 63999 + } + ], + "tool_call": true, + "temperature": true, + "structured_output": true, + "open_weights": false, + "knowledge": "2025-07-31", + "release_date": "2025-09-29", + "last_updated": "2025-09-29", + "modalities": { + "input": ["text", "image", "pdf"], + "output": ["text"] + }, + "limit": { + "context": 200000, + "output": 64000 + }, + "cost": { + "input": 3, + "output": 15, + "cache_read": 0.3, + "cache_write": 3.75 + }, + "blended_cost": 6, + "capabilities": { + "entitled": true, + "chat": "conforms", + "deviations": [], + "native_dialect": "anthropic", + "responses": false, + "web_search": false, + "image_generation": false, + "embeddings": false + }, + "use_case": "chat" + }, + { + "id": "claude-sonnet-4-6", + "name": "Claude Sonnet 4.6", + "provider": "anthropic", + "family": "claude-sonnet", + "released": true, + "attachment": true, + "reasoning": true, + "reasoning_options": [ + { + "type": "toggle" + }, + { + "type": "budget_tokens", + "min": 1024, + "max": 63999 + } + ], + "tool_call": true, + "temperature": true, + "structured_output": true, + "open_weights": false, + "knowledge": "2025-08-31", + "release_date": "2026-02-17", + "last_updated": "2026-03-13", + "modalities": { + "input": ["text", "image", "pdf"], + "output": ["text"] + }, + "limit": { + "context": 1000000, + "output": 64000 + }, + "cost": { + "input": 3, + "output": 15, + "cache_read": 0.3, + "cache_write": 3.75 + }, + "blended_cost": 6, + "capabilities": { + "entitled": true, + "chat": "conforms", + "deviations": [], + "native_dialect": "anthropic", + "responses": false, + "web_search": false, + "image_generation": false, + "embeddings": false + }, + "use_case": "chat" + }, + { + "id": "claude-sonnet-5", + "name": "Claude Sonnet 5", + "provider": "anthropic", + "family": "claude-sonnet", + "released": true, + "attachment": true, + "reasoning": true, + "reasoning_options": [ + { + "type": "toggle" + }, + { + "type": "effort", + "values": ["low", "medium", "high", "xhigh", "max"] + } + ], + "tool_call": true, + "temperature": false, + "structured_output": true, + "open_weights": false, + "knowledge": "2026-01-31", + "release_date": "2026-06-30", + "last_updated": "2026-06-30", + "modalities": { + "input": ["text", "image", "pdf"], + "output": ["text"] + }, + "limit": { + "context": 1000000, + "output": 128000 + }, + "cost": { + "input": 2, + "output": 10, + "cache_read": 0.2, + "cache_write": 2.5 + }, + "blended_cost": 4, + "capabilities": { + "entitled": true, + "chat": "conforms", + "deviations": [], + "native_dialect": "anthropic", + "responses": false, + "web_search": false, + "image_generation": false, + "embeddings": false + }, + "use_case": "chat" + }, + { + "id": "claude-sonnet-5-5", + "name": "Claude Sonnet 5.5", + "provider": "anthropic", + "family": "claude-sonnet", + "released": true, + "attachment": true, + "reasoning": true, + "reasoning_options": [ + { + "type": "toggle" + }, + { + "type": "effort", + "values": ["low", "medium", "high", "xhigh", "max"] + } + ], + "tool_call": true, + "temperature": false, + "structured_output": true, + "open_weights": false, + "release_date": "2026-09-28", + "last_updated": "2026-09-28", + "modalities": { + "input": ["text", "image", "pdf"], + "output": ["text"] + }, + "limit": { + "context": 1000000, + "output": 128000 + }, + "cost": { + "input": 2, + "output": 10, + "cache_read": 0.2, + "cache_write": 2.5 + }, + "blended_cost": 4, + "capabilities": { + "entitled": true, + "chat": "conforms", + "deviations": [], + "native_dialect": "anthropic", + "responses": false, + "web_search": false, + "image_generation": false, + "embeddings": false + }, + "use_case": "chat" + }, + { + "id": "deepseek-v4-1-flash", + "name": "DeepSeek V4.1 Flash", + "provider": "deepseek", + "family": "deepseek-flash", + "released": true, + "attachment": true, + "reasoning": true, + "tool_call": true, + "temperature": true, + "structured_output": false, + "open_weights": true, + "release_date": "2026-09-10", + "last_updated": "2026-09-10", + "modalities": { + "input": ["text", "image"], + "output": ["text"] + }, + "limit": { + "context": 128000, + "output": 8192 + }, + "cost": { + "input": 0.3, + "output": 1.2, + "cache_read": 0.03 + }, + "blended_cost": 0.525, + "capabilities": { + "entitled": true, + "chat": "conforms", + "deviations": [], + "native_dialect": "none", + "responses": false, + "web_search": false, + "image_generation": false, + "embeddings": false + }, + "use_case": "chat" + }, + { + "id": "gemini-3-1-flash-lite", + "name": "Gemini 3.1 Flash Lite", + "provider": "google", + "family": "gemini-flash-lite", + "released": true, + "attachment": true, + "reasoning": true, + "reasoning_options": [ + { + "type": "effort", + "values": ["low", "medium", "high"] + } + ], + "tool_call": true, + "temperature": true, + "structured_output": true, + "open_weights": false, + "knowledge": "2025-01", + "release_date": "2026-03-03", + "last_updated": "2026-03-03", + "modalities": { + "input": ["text", "image", "video", "audio", "pdf"], + "output": ["text"] + }, + "limit": { + "context": 1048576, + "output": 65536 + }, + "cost": { + "input": 0.25, + "output": 1.5, + "cache_read": 0.025, + "input_audio": 0.5 + }, + "blended_cost": 0.563, + "capabilities": { + "entitled": true, + "chat": "conforms", + "deviations": [], + "native_dialect": "gemini", + "responses": false, + "web_search": false, + "image_generation": false, + "embeddings": false + }, + "use_case": "chat" + }, + { + "id": "gemini-3-1-pro", + "name": "Gemini 3.1 Pro", + "provider": "google", + "family": "gemini-pro", + "released": true, + "attachment": true, + "reasoning": true, + "reasoning_options": [ + { + "type": "effort", + "values": ["low", "medium", "high"] + } + ], + "tool_call": true, + "temperature": true, + "structured_output": true, + "open_weights": false, + "knowledge": "2025-01", + "release_date": "2026-02-19", + "last_updated": "2026-02-19", + "modalities": { + "input": ["text", "image", "video", "audio", "pdf"], + "output": ["text"] + }, + "limit": { + "context": 1048576, + "output": 65536 + }, + "cost": { + "input": 2, + "output": 12, + "cache_read": 0.2, + "tiers": [ + { + "input": 4, + "output": 18, + "cache_read": 0.4, + "tier": { + "type": "context", + "size": 200000 + } + } + ], + "context_over_200k": { + "input": 4, + "output": 18, + "cache_read": 0.4 + } + }, + "blended_cost": 4.5, + "capabilities": { + "entitled": true, + "chat": "conforms", + "deviations": [], + "native_dialect": "gemini", + "responses": false, + "web_search": false, + "image_generation": false, + "embeddings": false + }, + "use_case": "chat" + }, + { + "id": "gemini-3-5-flash", + "name": "Gemini 3.5 Flash", + "provider": "google", + "family": "gemini-flash", + "released": true, + "attachment": true, + "reasoning": true, + "reasoning_options": [ + { + "type": "effort", + "values": ["minimal", "low", "medium", "high"] + } + ], + "tool_call": true, + "temperature": true, + "structured_output": true, + "open_weights": false, + "knowledge": "2025-01", + "release_date": "2026-05-19", + "last_updated": "2026-05-19", + "modalities": { + "input": ["text", "image", "video", "audio", "pdf"], + "output": ["text"] + }, + "limit": { + "context": 1048576, + "output": 65536 + }, + "cost": { + "input": 1.5, + "output": 9, + "cache_read": 0.15, + "input_audio": 1.5 + }, + "blended_cost": 3.375, + "capabilities": { + "entitled": true, + "chat": "conforms", + "deviations": [], + "native_dialect": "gemini", + "responses": false, + "web_search": false, + "image_generation": false, + "embeddings": false + }, + "use_case": "chat" + }, + { + "id": "gemini-3-5-flash-lite", + "name": "Gemini 3.5 Flash Lite", + "provider": "google", + "family": "gemini-flash-lite", + "released": true, + "attachment": true, + "reasoning": true, + "reasoning_options": [ + { + "type": "effort", + "values": ["minimal", "low", "medium", "high"] + } + ], + "tool_call": true, + "temperature": true, + "structured_output": true, + "open_weights": false, + "knowledge": "2026-03", + "release_date": "2026-07-21", + "last_updated": "2026-07-21", + "modalities": { + "input": ["text", "image", "video", "audio", "pdf"], + "output": ["text"] + }, + "limit": { + "context": 1048576, + "output": 65536 + }, + "cost": { + "input": 0.3, + "output": 2.5, + "cache_read": 0.03 + }, + "blended_cost": 0.85, + "capabilities": { + "entitled": true, + "chat": "conforms", + "deviations": [], + "native_dialect": "gemini", + "responses": false, + "web_search": false, + "image_generation": false, + "embeddings": false + }, + "use_case": "chat" + }, + { + "id": "gemini-3-6-flash", + "name": "Gemini 3.6 Flash", + "provider": "google", + "family": "gemini-flash", + "released": true, + "attachment": true, + "reasoning": true, + "reasoning_options": [ + { + "type": "effort", + "values": ["minimal", "low", "medium", "high"] + } + ], + "tool_call": true, + "temperature": false, + "structured_output": true, + "open_weights": false, + "knowledge": "2026-03", + "release_date": "2026-07-21", + "last_updated": "2026-07-21", + "modalities": { + "input": ["text", "image", "video", "audio", "pdf"], + "output": ["text"] + }, + "limit": { + "context": 1048576, + "output": 65536 + }, + "cost": { + "input": 1.5, + "output": 7.5, + "cache_read": 0.15, + "input_audio": 1.5 + }, + "blended_cost": 3, + "capabilities": { + "entitled": true, + "chat": "conforms", + "deviations": [], + "native_dialect": "gemini", + "responses": false, + "web_search": false, + "image_generation": false, + "embeddings": false + }, + "use_case": "chat" + }, + { + "id": "gemini-3-7-flash", + "name": "Gemini 3.7 Flash", + "provider": "google", + "family": "gemini-flash", + "released": true, + "attachment": true, + "reasoning": true, + "reasoning_options": [ + { + "type": "effort", + "values": ["minimal", "low", "medium", "high"] + } + ], + "tool_call": true, + "temperature": false, + "structured_output": true, + "open_weights": false, + "release_date": "2026-08-13", + "last_updated": "2026-08-13", + "modalities": { + "input": ["text", "image", "video", "audio", "pdf"], + "output": ["text"] + }, + "limit": { + "context": 1048576, + "output": 65536 + }, + "cost": { + "input": 1.5, + "output": 7.5, + "cache_read": 0.15 + }, + "blended_cost": 3, + "capabilities": { + "entitled": true, + "chat": "array-content", + "deviations": [ + "`id` is null; the spec requires a string", + "`content` is an array[1] of {text} with keys {text, thoughtSignature, type}; the spec requires a string", + "`usage.total_tokens` is 459, but prompt + completion is 187", + "`usage.reasoning_tokens` is top-level; the spec nests it under `completion_tokens_details`" + ], + "native_dialect": "gemini", + "responses": false, + "web_search": false, + "image_generation": false, + "embeddings": false + }, + "use_case": "chat" + }, + { + "id": "gemini-3-8-flash", + "name": "Gemini 3.8 Flash", + "provider": "google", + "family": "gemini-flash", + "released": true, + "attachment": true, + "reasoning": true, + "reasoning_options": [ + { + "type": "effort", + "values": ["minimal", "low", "medium", "high"] + } + ], + "tool_call": true, + "temperature": false, + "structured_output": true, + "open_weights": false, + "release_date": "2026-09-02", + "last_updated": "2026-09-02", + "modalities": { + "input": ["text", "image", "video", "audio", "pdf"], + "output": ["text"] + }, + "limit": { + "context": 1048576, + "output": 65536 + }, + "cost": { + "input": 1.5, + "output": 7.5, + "cache_read": 0.15 + }, + "blended_cost": 3, + "capabilities": { + "entitled": true, + "chat": "array-content", + "deviations": [ + "`id` is null; the spec requires a string", + "`content` is an array[1] of {text} with keys {text, thoughtSignature, type}; the spec requires a string" + ], + "native_dialect": "gemini", + "responses": false, + "web_search": false, + "image_generation": false, + "embeddings": false + }, + "use_case": "chat" + }, + { + "id": "gemini-3-flash", + "name": "Gemini 3 Flash", + "provider": "google", + "family": "gemini-flash", + "released": true, + "attachment": true, + "reasoning": true, + "reasoning_options": [ + { + "type": "effort", + "values": ["low", "medium", "high"] + } + ], + "tool_call": true, + "temperature": true, + "structured_output": true, + "open_weights": false, + "knowledge": "2025-01", + "release_date": "2025-12-17", + "last_updated": "2025-12-17", + "modalities": { + "input": ["text", "image", "video", "audio", "pdf"], + "output": ["text"] + }, + "limit": { + "context": 1048576, + "output": 65536 + }, + "cost": { + "input": 0.5, + "output": 3, + "cache_read": 0.05, + "input_audio": 1 + }, + "blended_cost": 1.125, + "capabilities": { + "entitled": true, + "chat": "conforms", + "deviations": [], + "native_dialect": "gemini", + "responses": false, + "web_search": false, + "image_generation": false, + "embeddings": false + }, + "use_case": "chat" + }, + { + "id": "gemma-3-12b", + "name": "Gemma 3 12B", + "provider": "google", + "family": "gemma", + "released": true, + "attachment": true, + "reasoning": false, + "tool_call": true, + "temperature": true, + "structured_output": true, + "open_weights": true, + "knowledge": "2024-08-31", + "release_date": "2025-03-13", + "last_updated": "2025-03-13", + "modalities": { + "input": ["text", "image"], + "output": ["text"] + }, + "limit": { + "context": 131072, + "output": 8192 + }, + "cost": { + "input": 0.15, + "output": 0.5 + }, + "blended_cost": 0.238, + "capabilities": { + "entitled": true, + "chat": "conforms", + "deviations": [], + "native_dialect": "none", + "responses": false, + "web_search": false, + "image_generation": false, + "embeddings": false + }, + "use_case": "chat" + }, + { + "id": "glm-5-2", + "name": "GLM 5.2", + "provider": "zhipuai", + "family": "glm", + "released": true, + "attachment": false, + "reasoning": true, + "reasoning_options": [ + { + "type": "toggle" + }, + { + "type": "effort", + "values": ["none", "minimal", "low", "medium", "high", "max"] + } + ], + "tool_call": true, + "temperature": true, + "structured_output": true, + "open_weights": true, + "release_date": "2026-06-13", + "last_updated": "2026-06-13", + "modalities": { + "input": ["text"], + "output": ["text"] + }, + "limit": { + "context": 1000000, + "output": 65536 + }, + "cost": { + "input": 1.4, + "output": 4.4, + "cache_read": 0.26 + }, + "blended_cost": 2.15, + "capabilities": { + "entitled": true, + "chat": "conforms", + "deviations": [], + "native_dialect": "none", + "responses": false, + "web_search": false, + "image_generation": false, + "embeddings": false + }, + "use_case": "chat" + }, + { + "id": "glm-5-3", + "name": "GLM 5.3", + "provider": "zhipuai", + "family": "glm", + "released": true, + "attachment": false, + "reasoning": true, + "reasoning_options": [ + { + "type": "effort", + "values": ["low", "high", "max"] + } + ], + "tool_call": true, + "temperature": true, + "structured_output": true, + "open_weights": true, + "release_date": "2026-08-28", + "last_updated": "2026-08-28", + "modalities": { + "input": ["text"], + "output": ["text"] + }, + "limit": { + "context": 1048576, + "output": 131072 + }, + "cost": { + "input": 1.4, + "output": 4.4, + "cache_read": 0.26 + }, + "blended_cost": 2.15, + "capabilities": { + "entitled": true, + "chat": "conforms", + "deviations": [], + "native_dialect": "none", + "responses": false, + "web_search": false, + "image_generation": false, + "embeddings": false + }, + "use_case": "chat" + }, + { + "id": "glm-5-3-flash", + "name": "GLM 5.3 Flash", + "provider": "zhipuai", + "family": "glm-flash", + "released": true, + "attachment": true, + "reasoning": true, + "reasoning_options": [ + { + "type": "effort", + "values": ["low", "high", "max"] + } + ], + "tool_call": true, + "temperature": true, + "structured_output": true, + "open_weights": true, + "release_date": "2026-08-26", + "last_updated": "2026-08-26", + "modalities": { + "input": ["text", "image"], + "output": ["text"] + }, + "limit": { + "context": 1048576, + "output": 131072 + }, + "cost": { + "input": 0.15, + "output": 0.5, + "cache_read": 0.03 + }, + "blended_cost": 0.238, + "capabilities": { + "entitled": true, + "chat": "conforms", + "deviations": [], + "native_dialect": "none", + "responses": false, + "web_search": false, + "image_generation": false, + "embeddings": false + }, + "use_case": "chat" + }, + { + "id": "gpt-5", + "name": "GPT-5", + "provider": "openai", + "family": "gpt", + "released": true, + "attachment": true, + "reasoning": true, + "reasoning_options": [ + { + "type": "effort", + "values": ["minimal", "low", "medium", "high"] + } + ], + "tool_call": true, + "temperature": false, + "structured_output": true, + "open_weights": false, + "knowledge": "2024-09-30", + "release_date": "2025-08-07", + "last_updated": "2025-08-07", + "modalities": { + "input": ["text", "image"], + "output": ["text", "image"] + }, + "limit": { + "context": 400000, + "input": 272000, + "output": 128000 + }, + "cost": { + "input": 1.25, + "output": 10, + "cache_read": 0.125 + }, + "blended_cost": 3.438, + "capabilities": { + "entitled": true, + "chat": "conforms", + "deviations": [], + "native_dialect": "openai-responses", + "responses": true, + "web_search": true, + "image_generation": true, + "embeddings": false + }, + "use_case": "chat" + }, + { + "id": "gpt-5-1", + "name": "GPT-5.1", + "provider": "openai", + "family": "gpt", + "released": true, + "attachment": true, + "reasoning": true, + "reasoning_options": [ + { + "type": "effort", + "values": ["none", "low", "medium", "high"] + } + ], + "tool_call": true, + "temperature": true, + "structured_output": true, + "open_weights": false, + "knowledge": "2024-09-30", + "release_date": "2025-11-13", + "last_updated": "2025-11-13", + "modalities": { + "input": ["text", "image"], + "output": ["text", "image"] + }, + "limit": { + "context": 400000, + "input": 272000, + "output": 128000 + }, + "cost": { + "input": 1.25, + "output": 10, + "cache_read": 0.125 + }, + "blended_cost": 3.438, + "capabilities": { + "entitled": true, + "chat": "conforms", + "deviations": [], + "native_dialect": "openai-responses", + "responses": true, + "web_search": true, + "image_generation": true, + "embeddings": false + }, + "use_case": "chat" + }, + { + "id": "gpt-5-2", + "name": "GPT-5.2", + "provider": "openai", + "family": "gpt", + "released": true, + "attachment": true, + "reasoning": true, + "reasoning_options": [ + { + "type": "effort", + "values": ["none", "low", "medium", "high", "xhigh"] + } + ], + "tool_call": true, + "temperature": true, + "structured_output": true, + "open_weights": false, + "knowledge": "2025-08-31", + "release_date": "2025-12-11", + "last_updated": "2025-12-11", + "modalities": { + "input": ["text", "image"], + "output": ["text", "image"] + }, + "limit": { + "context": 400000, + "input": 272000, + "output": 128000 + }, + "cost": { + "input": 1.75, + "output": 14, + "cache_read": 0.175 + }, + "blended_cost": 4.813, + "capabilities": { + "entitled": true, + "chat": "conforms", + "deviations": [], + "native_dialect": "openai-responses", + "responses": true, + "web_search": true, + "image_generation": true, + "embeddings": false + }, + "use_case": "chat" + }, + { + "id": "gpt-5-3-codex", + "name": "GPT-5.3 Codex", + "provider": "openai", + "family": "gpt-codex", + "released": true, + "attachment": true, + "reasoning": true, + "reasoning_options": [ + { + "type": "effort", + "values": ["none", "low", "medium", "high", "xhigh"] + } + ], + "tool_call": true, + "temperature": true, + "structured_output": true, + "open_weights": false, + "knowledge": "2025-08-31", + "release_date": "2026-02-05", + "last_updated": "2026-02-05", + "modalities": { + "input": ["text", "image", "pdf"], + "output": ["text", "image"] + }, + "limit": { + "context": 400000, + "input": 272000, + "output": 128000 + }, + "cost": { + "input": 1.75, + "output": 14, + "cache_read": 0.175 + }, + "blended_cost": 4.813, + "capabilities": { + "entitled": true, + "chat": "not-served", + "deviations": [], + "native_dialect": "openai-responses", + "responses": true, + "web_search": true, + "image_generation": true, + "embeddings": false + }, + "use_case": "chat" + }, + { + "id": "gpt-5-4", + "name": "GPT-5.4", + "provider": "openai", + "family": "gpt", + "released": true, + "attachment": true, + "reasoning": true, + "reasoning_options": [ + { + "type": "effort", + "values": ["none", "low", "medium", "high", "xhigh"] + } + ], + "tool_call": true, + "temperature": true, + "structured_output": true, + "open_weights": false, + "knowledge": "2025-08-31", + "release_date": "2026-03-05", + "last_updated": "2026-03-05", + "modalities": { + "input": ["text", "image", "pdf"], + "output": ["text", "image"] + }, + "limit": { + "context": 1050000, + "input": 922000, + "output": 128000 + }, + "cost": { + "input": 2.5, + "output": 15, + "cache_read": 0.25, + "tiers": [ + { + "input": 5, + "output": 22.5, + "cache_read": 0.5, + "tier": { + "type": "context", + "size": 272000 + } + } + ], + "context_over_200k": { + "input": 5, + "output": 22.5, + "cache_read": 0.5 + } + }, + "blended_cost": 5.625, + "capabilities": { + "entitled": true, + "chat": "conforms", + "deviations": [], + "native_dialect": "openai-responses", + "responses": true, + "web_search": true, + "image_generation": true, + "embeddings": false + }, + "use_case": "chat" + }, + { + "id": "gpt-5-4-mini", + "name": "GPT-5.4 Mini", + "provider": "openai", + "family": "gpt-mini", + "released": true, + "attachment": true, + "reasoning": true, + "reasoning_options": [ + { + "type": "effort", + "values": ["none", "low", "medium", "high", "xhigh"] + } + ], + "tool_call": true, + "temperature": true, + "structured_output": true, + "open_weights": false, + "knowledge": "2025-08-31", + "release_date": "2026-03-17", + "last_updated": "2026-03-17", + "modalities": { + "input": ["text", "image"], + "output": ["text", "image"] + }, + "limit": { + "context": 400000, + "input": 272000, + "output": 128000 + }, + "cost": { + "input": 0.75, + "output": 4.5, + "cache_read": 0.075 + }, + "blended_cost": 1.688, + "capabilities": { + "entitled": true, + "chat": "conforms", + "deviations": [], + "native_dialect": "openai-responses", + "responses": true, + "web_search": true, + "image_generation": true, + "embeddings": false + }, + "use_case": "chat" + }, + { + "id": "gpt-5-4-nano", + "name": "GPT-5.4 Nano", + "provider": "openai", + "family": "gpt-nano", + "released": true, + "attachment": true, + "reasoning": true, + "reasoning_options": [ + { + "type": "effort", + "values": ["none", "low", "medium", "high", "xhigh"] + } + ], + "tool_call": true, + "temperature": true, + "structured_output": true, + "open_weights": false, + "knowledge": "2025-08-31", + "release_date": "2026-03-17", + "last_updated": "2026-03-17", + "modalities": { + "input": ["text", "image"], + "output": ["text", "image"] + }, + "limit": { + "context": 400000, + "input": 272000, + "output": 128000 + }, + "cost": { + "input": 0.2, + "output": 1.25, + "cache_read": 0.02 + }, + "blended_cost": 0.463, + "capabilities": { + "entitled": true, + "chat": "conforms", + "deviations": [], + "native_dialect": "openai-responses", + "responses": true, + "web_search": true, + "image_generation": true, + "embeddings": false + }, + "use_case": "chat" + }, + { + "id": "gpt-5-5", + "name": "GPT-5.5", + "provider": "openai", + "family": "gpt", + "released": true, + "attachment": true, + "reasoning": true, + "reasoning_options": [ + { + "type": "effort", + "values": ["none", "low", "medium", "high", "xhigh"] + } + ], + "tool_call": true, + "temperature": false, + "structured_output": true, + "open_weights": false, + "knowledge": "2025-12-01", + "release_date": "2026-04-23", + "last_updated": "2026-04-23", + "modalities": { + "input": ["text", "image", "pdf"], + "output": ["text", "image"] + }, + "limit": { + "context": 1050000, + "input": 922000, + "output": 128000 + }, + "cost": { + "input": 5, + "output": 30, + "cache_read": 0.5, + "tiers": [ + { + "input": 10, + "output": 45, + "cache_read": 1, + "tier": { + "type": "context", + "size": 272000 + } + } + ], + "context_over_200k": { + "input": 10, + "output": 45, + "cache_read": 1 + } + }, + "blended_cost": 11.25, + "capabilities": { + "entitled": true, + "chat": "conforms", + "deviations": [], + "native_dialect": "openai-responses", + "responses": true, + "web_search": true, + "image_generation": true, + "embeddings": false + }, + "use_case": "chat" + }, + { + "id": "gpt-5-5-pro", + "name": "GPT-5.5 Pro", + "provider": "openai", + "family": "gpt-pro", + "released": true, + "attachment": true, + "reasoning": true, + "reasoning_options": [ + { + "type": "effort", + "values": ["medium", "high", "xhigh"] + } + ], + "tool_call": true, + "temperature": false, + "structured_output": true, + "open_weights": false, + "knowledge": "2025-12-01", + "release_date": "2026-04-23", + "last_updated": "2026-04-23", + "modalities": { + "input": ["text", "image", "pdf"], + "output": ["text", "image"] + }, + "limit": { + "context": 1050000, + "input": 922000, + "output": 128000 + }, + "cost": { + "input": 30, + "output": 180, + "tiers": [ + { + "input": 60, + "output": 270, + "tier": { + "type": "context", + "size": 272000 + } + } + ], + "context_over_200k": { + "input": 60, + "output": 270 + } + }, + "blended_cost": 67.5, + "capabilities": { + "entitled": true, + "chat": "not-served", + "deviations": [], + "native_dialect": "openai-responses", + "responses": true, + "web_search": true, + "image_generation": true, + "embeddings": false + }, + "use_case": "chat" + }, + { + "id": "gpt-5-6-luna", + "name": "GPT-5.6 Luna", + "provider": "openai", + "family": "gpt-luna", + "released": true, + "attachment": true, + "reasoning": true, + "reasoning_options": [ + { + "type": "effort", + "values": ["none", "low", "medium", "high", "xhigh", "max"] + } + ], + "tool_call": true, + "temperature": false, + "structured_output": true, + "open_weights": false, + "knowledge": "2026-02-16", + "release_date": "2026-07-09", + "last_updated": "2026-07-09", + "modalities": { + "input": ["text", "image", "pdf"], + "output": ["text", "image"] + }, + "limit": { + "context": 1050000, + "input": 922000, + "output": 128000 + }, + "cost": { + "input": 0.2, + "output": 1.2, + "cache_read": 0.02, + "cache_write": 0.25, + "tiers": [ + { + "input": 0.4, + "output": 1.8, + "cache_read": 0.04, + "cache_write": 0.5, + "tier": { + "type": "context", + "size": 272000 + } + } + ], + "context_over_200k": { + "input": 0.4, + "output": 1.8, + "cache_read": 0.04, + "cache_write": 0.5 + } + }, + "blended_cost": 0.45, + "capabilities": { + "entitled": true, + "chat": "conforms", + "deviations": [], + "native_dialect": "openai-responses", + "responses": true, + "web_search": true, + "image_generation": true, + "embeddings": false + }, + "use_case": "chat" + }, + { + "id": "gpt-5-6-sol", + "name": "GPT-5.6 Sol", + "provider": "openai", + "family": "gpt-sol", + "released": true, + "attachment": true, + "reasoning": true, + "reasoning_options": [ + { + "type": "effort", + "values": ["none", "low", "medium", "high", "xhigh", "max"] + } + ], + "tool_call": true, + "temperature": false, + "structured_output": true, + "open_weights": false, + "knowledge": "2026-02-16", + "release_date": "2026-07-09", + "last_updated": "2026-07-09", + "modalities": { + "input": ["text", "image", "pdf"], + "output": ["text", "image"] + }, + "limit": { + "context": 1050000, + "input": 922000, + "output": 128000 + }, + "cost": { + "input": 5, + "output": 30, + "cache_read": 0.5, + "cache_write": 6.25, + "tiers": [ + { + "input": 10, + "output": 45, + "cache_read": 1, + "cache_write": 12.5, + "tier": { + "type": "context", + "size": 272000 + } + } + ], + "context_over_200k": { + "input": 10, + "output": 45, + "cache_read": 1, + "cache_write": 12.5 + } + }, + "blended_cost": 11.25, + "capabilities": { + "entitled": true, + "chat": "conforms", + "deviations": [], + "native_dialect": "openai-responses", + "responses": true, + "web_search": true, + "image_generation": true, + "embeddings": false + }, + "use_case": "chat" + }, + { + "id": "gpt-5-6-terra", + "name": "GPT-5.6 Terra", + "provider": "openai", + "family": "gpt-terra", + "released": true, + "attachment": true, + "reasoning": true, + "reasoning_options": [ + { + "type": "effort", + "values": ["none", "low", "medium", "high", "xhigh", "max"] + } + ], + "tool_call": true, + "temperature": false, + "structured_output": true, + "open_weights": false, + "knowledge": "2026-02-16", + "release_date": "2026-07-09", + "last_updated": "2026-07-09", + "modalities": { + "input": ["text", "image", "pdf"], + "output": ["text", "image"] + }, + "limit": { + "context": 1050000, + "input": 922000, + "output": 128000 + }, + "cost": { + "input": 2, + "output": 12, + "cache_read": 0.2, + "cache_write": 2.5, + "tiers": [ + { + "input": 4, + "output": 18, + "cache_read": 0.4, + "cache_write": 5, + "tier": { + "type": "context", + "size": 272000 + } + } + ], + "context_over_200k": { + "input": 4, + "output": 18, + "cache_read": 0.4, + "cache_write": 5 + } + }, + "blended_cost": 4.5, + "capabilities": { + "entitled": true, + "chat": "conforms", + "deviations": [], + "native_dialect": "openai-responses", + "responses": true, + "web_search": true, + "image_generation": true, + "embeddings": false + }, + "use_case": "chat" + }, + { + "id": "gpt-5-mini", + "name": "GPT-5 Mini", + "provider": "openai", + "family": "gpt-mini", + "released": true, + "attachment": true, + "reasoning": true, + "reasoning_options": [ + { + "type": "effort", + "values": ["minimal", "low", "medium", "high"] + } + ], + "tool_call": true, + "temperature": false, + "structured_output": true, + "open_weights": false, + "knowledge": "2024-05-30", + "release_date": "2025-08-07", + "last_updated": "2025-08-07", + "modalities": { + "input": ["text", "image"], + "output": ["text", "image"] + }, + "limit": { + "context": 400000, + "input": 272000, + "output": 128000 + }, + "cost": { + "input": 0.25, + "output": 2, + "cache_read": 0.025 + }, + "blended_cost": 0.688, + "capabilities": { + "entitled": true, + "chat": "conforms", + "deviations": [], + "native_dialect": "openai-responses", + "responses": true, + "web_search": true, + "image_generation": true, + "embeddings": false + }, + "use_case": "chat" + }, + { + "id": "gpt-5-nano", + "name": "GPT-5 Nano", + "provider": "openai", + "family": "gpt-nano", + "released": true, + "attachment": true, + "reasoning": true, + "reasoning_options": [ + { + "type": "effort", + "values": ["minimal", "low", "medium", "high"] + } + ], + "tool_call": true, + "temperature": false, + "structured_output": true, + "open_weights": false, + "knowledge": "2024-05-30", + "release_date": "2025-08-07", + "last_updated": "2025-08-07", + "modalities": { + "input": ["text", "image"], + "output": ["text", "image"] + }, + "limit": { + "context": 400000, + "input": 272000, + "output": 128000 + }, + "cost": { + "input": 0.05, + "output": 0.4, + "cache_read": 0.01 + }, + "blended_cost": 0.138, + "capabilities": { + "entitled": true, + "chat": "conforms", + "deviations": [], + "native_dialect": "openai-responses", + "responses": true, + "web_search": true, + "image_generation": true, + "embeddings": false + }, + "use_case": "chat" + }, + { + "id": "gpt-6-1-sol", + "name": "GPT-6.1 Sol", + "provider": "openai", + "family": "gpt-sol", + "released": true, + "attachment": true, + "reasoning": true, + "reasoning_options": [ + { + "type": "effort", + "values": ["low", "medium", "high", "xhigh", "max"] + } + ], + "tool_call": true, + "temperature": false, + "structured_output": true, + "open_weights": false, + "release_date": "2026-09-29", + "last_updated": "2026-09-29", + "modalities": { + "input": ["text", "image", "pdf"], + "output": ["text"] + }, + "limit": { + "context": 1050000, + "input": 922000, + "output": 128000 + }, + "cost": { + "input": 2, + "output": 10, + "cache_read": 0.1, + "cache_write": 2.5, + "tiers": [ + { + "input": 4, + "output": 15, + "cache_read": 0.2, + "cache_write": 5, + "tier": { + "type": "context", + "size": 272000 + } + } + ], + "context_over_200k": { + "input": 4, + "output": 15, + "cache_read": 0.2, + "cache_write": 5 + } + }, + "blended_cost": 4, + "capabilities": { + "entitled": true, + "chat": "conforms", + "deviations": [], + "native_dialect": "openai-responses", + "responses": true, + "web_search": true, + "image_generation": true, + "embeddings": false + }, + "use_case": "chat" + }, + { + "id": "gpt-6-astra", + "name": "GPT-6 Astra", + "provider": "openai", + "family": "gpt-astra", + "released": true, + "attachment": true, + "reasoning": true, + "reasoning_options": [ + { + "type": "effort", + "values": ["low", "medium", "high", "xhigh", "max"] + } + ], + "tool_call": true, + "temperature": false, + "structured_output": true, + "open_weights": false, + "knowledge": "2026-04-30", + "release_date": "2026-09-04", + "last_updated": "2026-09-04", + "modalities": { + "input": ["text", "image", "pdf"], + "output": ["text", "image"] + }, + "limit": { + "context": 1050000, + "input": 922000, + "output": 128000 + }, + "cost": { + "input": 10, + "output": 50, + "cache_read": 1, + "cache_write": 12.5, + "tiers": [ + { + "input": 20, + "output": 75, + "cache_read": 2, + "cache_write": 25, + "tier": { + "type": "context", + "size": 272000 + } + } + ], + "context_over_200k": { + "input": 20, + "output": 75, + "cache_read": 2, + "cache_write": 25 + } + }, + "blended_cost": 20, + "capabilities": { + "entitled": true, + "chat": "conforms", + "deviations": [], + "native_dialect": "openai-responses", + "responses": true, + "web_search": true, + "image_generation": true, + "embeddings": false + }, + "use_case": "chat" + }, + { + "id": "gpt-6-luna", + "name": "GPT-6 Luna", + "provider": "openai", + "family": "gpt-luna", + "released": true, + "attachment": true, + "reasoning": true, + "reasoning_options": [ + { + "type": "effort", + "values": ["low", "medium", "high", "xhigh", "max"] + } + ], + "tool_call": true, + "temperature": false, + "structured_output": true, + "open_weights": false, + "release_date": "2026-09-22", + "last_updated": "2026-09-22", + "modalities": { + "input": ["text", "image", "pdf"], + "output": ["text"] + }, + "limit": { + "context": 1050000, + "input": 922000, + "output": 128000 + }, + "cost": { + "input": 0.1, + "output": 0.5, + "cache_read": 0.01, + "cache_write": 0.125, + "tiers": [ + { + "input": 0.2, + "output": 0.75, + "cache_read": 0.02, + "cache_write": 0.25, + "tier": { + "type": "context", + "size": 272000 + } + } + ], + "context_over_200k": { + "input": 0.2, + "output": 0.75, + "cache_read": 0.02, + "cache_write": 0.25 + } + }, + "blended_cost": 0.2, + "capabilities": { + "entitled": true, + "chat": "conforms", + "deviations": [], + "native_dialect": "openai-responses", + "responses": true, + "web_search": true, + "image_generation": true, + "embeddings": false + }, + "use_case": "chat" + }, + { + "id": "gpt-6-sol", + "name": "GPT-6 Sol", + "provider": "openai", + "family": "gpt-sol", + "released": true, + "attachment": true, + "reasoning": true, + "reasoning_options": [ + { + "type": "effort", + "values": ["low", "medium", "high", "xhigh", "max"] + } + ], + "tool_call": true, + "temperature": false, + "structured_output": true, + "open_weights": false, + "release_date": "2026-09-22", + "last_updated": "2026-09-22", + "modalities": { + "input": ["text", "image", "pdf"], + "output": ["text"] + }, + "limit": { + "context": 1050000, + "input": 922000, + "output": 128000 + }, + "cost": { + "input": 2, + "output": 10, + "cache_read": 0.2, + "cache_write": 2.5, + "tiers": [ + { + "input": 4, + "output": 15, + "cache_read": 0.4, + "cache_write": 5, + "tier": { + "type": "context", + "size": 272000 + } + } + ], + "context_over_200k": { + "input": 4, + "output": 15, + "cache_read": 0.4, + "cache_write": 5 + } + }, + "blended_cost": 4, + "capabilities": { + "entitled": true, + "chat": "conforms", + "deviations": [], + "native_dialect": "openai-responses", + "responses": true, + "web_search": true, + "image_generation": true, + "embeddings": false + }, + "use_case": "chat" + }, + { + "id": "gpt-oss-120b", + "name": "GPT OSS 120B", + "provider": "openai", + "family": "gpt-oss", + "released": true, + "attachment": false, + "reasoning": true, + "reasoning_options": [ + { + "type": "effort", + "values": ["low", "medium", "high"] + } + ], + "tool_call": true, + "temperature": true, + "structured_output": true, + "open_weights": true, + "release_date": "2025-08-05", + "last_updated": "2025-08-05", + "modalities": { + "input": ["text"], + "output": ["text"] + }, + "limit": { + "context": 131072, + "output": 25000 + }, + "cost": { + "input": 0.15, + "output": 0.6 + }, + "blended_cost": 0.262, + "capabilities": { + "entitled": true, + "chat": "conforms", + "deviations": [], + "native_dialect": "none", + "responses": false, + "web_search": false, + "image_generation": false, + "embeddings": false + }, + "use_case": "chat" + }, + { + "id": "gpt-oss-20b", + "name": "GPT OSS 20B", + "provider": "openai", + "family": "gpt-oss", + "released": true, + "attachment": false, + "reasoning": true, + "reasoning_options": [ + { + "type": "effort", + "values": ["low", "medium", "high"] + } + ], + "tool_call": true, + "temperature": true, + "structured_output": true, + "open_weights": true, + "release_date": "2025-08-05", + "last_updated": "2025-08-05", + "modalities": { + "input": ["text"], + "output": ["text"] + }, + "limit": { + "context": 131072, + "output": 25000 + }, + "cost": { + "input": 0.07, + "output": 0.3 + }, + "blended_cost": 0.128, + "capabilities": { + "entitled": true, + "chat": "conforms", + "deviations": [], + "native_dialect": "none", + "responses": false, + "web_search": false, + "image_generation": false, + "embeddings": false + }, + "use_case": "chat" + }, + { + "id": "grok-4-6", + "name": "Grok 4.6", + "provider": "xai", + "family": "grok", + "released": true, + "attachment": true, + "reasoning": true, + "reasoning_options": [ + { + "type": "effort", + "values": ["low", "medium", "high", "xhigh"] + } + ], + "tool_call": true, + "temperature": true, + "structured_output": true, + "open_weights": false, + "knowledge": "2026-02-01", + "release_date": "2026-08-12", + "last_updated": "2026-08-12", + "modalities": { + "input": ["text", "image"], + "output": ["text"] + }, + "limit": { + "context": 500000, + "output": 524288 + }, + "cost": { + "input": 2, + "output": 6, + "cache_read": 0.5 + }, + "blended_cost": 3, + "capabilities": { + "entitled": true, + "chat": "conforms", + "deviations": [], + "native_dialect": "openai-responses", + "responses": true, + "web_search": false, + "image_generation": false, + "embeddings": false + }, + "use_case": "chat" + }, + { + "id": "grok-4-7", + "name": "Grok 4.7", + "provider": "xai", + "family": "grok", + "released": true, + "attachment": false, + "reasoning": true, + "reasoning_options": [ + { + "type": "effort", + "values": ["low", "medium", "high", "xhigh"] + } + ], + "tool_call": true, + "temperature": true, + "structured_output": true, + "open_weights": false, + "release_date": "2026-09-28", + "last_updated": "2026-09-28", + "modalities": { + "input": ["text"], + "output": ["text"] + }, + "limit": { + "context": 500000, + "output": 524288 + }, + "cost": { + "input": 2.5, + "output": 7.5, + "cache_read": 0.5 + }, + "blended_cost": 3.75, + "capabilities": { + "entitled": true, + "chat": "conforms", + "deviations": [], + "native_dialect": "openai-responses", + "responses": true, + "web_search": false, + "image_generation": false, + "embeddings": false + }, + "use_case": "chat" + }, + { + "id": "gte-large-en", + "name": "GTE Large (En)", + "provider": "alibaba", + "family": "text-embedding", + "released": true, + "type": "embedding", + "dimensions": 1024, + "open_weights": true, + "release_date": "2023-07-27", + "last_updated": "2026-09-25", + "cost": { + "input": 0.13 + }, + "capabilities": { + "entitled": true, + "chat": "not-served", + "deviations": [], + "native_dialect": "none", + "responses": false, + "web_search": false, + "image_generation": false, + "embeddings": true + }, + "use_case": "chat", + "unsupported": "gte-large-en does not serve chat completions. It is an embedding model — see the embeddings use case." + }, + { + "id": "inkling", + "name": "Inkling", + "provider": "thinkingmachines", + "family": "ling", + "released": true, + "attachment": true, + "reasoning": true, + "reasoning_options": [ + { + "type": "toggle" + }, + { + "type": "effort", + "values": ["none", "minimal", "low", "medium", "high", "max"] + } + ], + "tool_call": true, + "temperature": true, + "structured_output": false, + "open_weights": true, + "release_date": "2026-07-15", + "last_updated": "2026-07-15", + "modalities": { + "input": ["text", "image", "audio"], + "output": ["text"] + }, + "limit": { + "context": 1048576, + "output": 65536 + }, + "cost": { + "input": 1, + "output": 4.05, + "cache_read": 0.17 + }, + "blended_cost": 1.763, + "capabilities": { + "entitled": true, + "chat": "conforms", + "deviations": [], + "native_dialect": "none", + "responses": false, + "web_search": false, + "image_generation": false, + "embeddings": false + }, + "use_case": "chat" + }, + { + "id": "kimi-k3", + "name": "Kimi K3", + "provider": "moonshotai", + "family": "kimi-k3", + "released": true, + "attachment": true, + "reasoning": true, + "reasoning_options": [ + { + "type": "toggle" + }, + { + "type": "effort", + "values": ["none", "minimal", "low", "medium", "high", "max"] + } + ], + "tool_call": true, + "temperature": true, + "structured_output": true, + "open_weights": true, + "release_date": "2026-07-16", + "last_updated": "2026-07-16", + "modalities": { + "input": ["text", "image", "video"], + "output": ["text"] + }, + "limit": { + "context": 1048576, + "output": 65536 + }, + "cost": { + "input": 3, + "output": 15, + "cache_read": 0.3 + }, + "blended_cost": 6, + "capabilities": { + "entitled": true, + "chat": "conforms", + "deviations": [], + "native_dialect": "none", + "responses": false, + "web_search": false, + "image_generation": false, + "embeddings": false + }, + "use_case": "chat" + }, + { + "id": "llama-4-maverick", + "name": "Llama 4 Maverick", + "provider": "meta", + "family": "llama", + "released": true, + "attachment": true, + "reasoning": false, + "tool_call": true, + "temperature": true, + "structured_output": true, + "open_weights": true, + "knowledge": "2024-08", + "release_date": "2025-04-05", + "last_updated": "2025-04-05", + "modalities": { + "input": ["text", "image"], + "output": ["text"] + }, + "limit": { + "context": 1000000, + "output": 8192 + }, + "cost": { + "input": 0.5, + "output": 1.5 + }, + "blended_cost": 0.75, + "capabilities": { + "entitled": true, + "chat": "conforms", + "deviations": [], + "native_dialect": "none", + "responses": false, + "web_search": false, + "image_generation": false, + "embeddings": false + }, + "use_case": "chat" + }, + { + "id": "meta-llama-3-1-8b-instruct", + "name": "Meta Llama 3.1 8B Instruct", + "provider": "meta", + "family": "llama", + "released": true, + "attachment": false, + "reasoning": false, + "tool_call": true, + "temperature": true, + "structured_output": true, + "open_weights": true, + "knowledge": "2023-12-31", + "release_date": "2024-07-23", + "last_updated": "2024-07-23", + "modalities": { + "input": ["text"], + "output": ["text"] + }, + "limit": { + "context": 131072, + "output": 8192 + }, + "cost": { + "input": 0.15, + "output": 0.45 + }, + "blended_cost": 0.225, + "capabilities": { + "entitled": true, + "chat": "conforms", + "deviations": [], + "native_dialect": "none", + "responses": false, + "web_search": false, + "image_generation": false, + "embeddings": false + }, + "use_case": "chat" + }, + { + "id": "meta-llama-3-3-70b-instruct", + "name": "Meta Llama 3.3 70B Instruct", + "provider": "meta", + "family": "llama", + "released": true, + "attachment": false, + "reasoning": false, + "tool_call": true, + "temperature": true, + "structured_output": true, + "open_weights": true, + "knowledge": "2023-12", + "release_date": "2024-12-06", + "last_updated": "2024-12-06", + "modalities": { + "input": ["text"], + "output": ["text"] + }, + "limit": { + "context": 128000, + "output": 8192 + }, + "cost": { + "input": 0.5, + "output": 1.5 + }, + "blended_cost": 0.75, + "capabilities": { + "entitled": true, + "chat": "conforms", + "deviations": [], + "native_dialect": "none", + "responses": false, + "web_search": false, + "image_generation": false, + "embeddings": false + }, + "use_case": "chat" + }, + { + "id": "qwen3-embedding-0-6b", + "name": "Qwen3 Embedding 0.6B", + "provider": "alibaba", + "family": "text-embedding", + "released": true, + "type": "embedding", + "dimensions": 1024, + "open_weights": true, + "release_date": "2025-06-03", + "last_updated": "2026-09-25", + "cost": { + "input": 0.02 + }, + "capabilities": { + "entitled": true, + "chat": "not-served", + "deviations": [], + "native_dialect": "none", + "responses": false, + "web_search": false, + "image_generation": false, + "embeddings": true + }, + "use_case": "chat", + "unsupported": "qwen3-embedding-0-6b does not serve chat completions. It is an embedding model — see the embeddings use case." + }, + { + "id": "qwen3-next-80b-a3b-instruct", + "name": "Qwen3 Next Instruct", + "provider": "alibaba", + "family": "qwen", + "released": true, + "attachment": false, + "reasoning": false, + "tool_call": true, + "temperature": true, + "structured_output": true, + "open_weights": true, + "knowledge": "2025-04", + "release_date": "2025-09", + "last_updated": "2025-09", + "modalities": { + "input": ["text"], + "output": ["text"] + }, + "limit": { + "context": 131072, + "output": 10000 + }, + "cost": { + "input": 0.15, + "output": 1.2 + }, + "blended_cost": 0.413, + "capabilities": { + "entitled": true, + "chat": "conforms", + "deviations": [], + "native_dialect": "none", + "responses": false, + "web_search": false, + "image_generation": false, + "embeddings": false + }, + "use_case": "chat" + }, + { + "id": "qwen35-122b-a10b", + "name": "Qwen3.5 122B A10B", + "provider": "alibaba", + "family": "qwen", + "released": true, + "attachment": false, + "reasoning": true, + "reasoning_options": [ + { + "type": "effort", + "values": ["none", "low", "medium", "high"] + } + ], + "tool_call": true, + "temperature": true, + "structured_output": true, + "open_weights": true, + "release_date": "2026-02-23", + "last_updated": "2026-02-23", + "modalities": { + "input": ["text"], + "output": ["text"] + }, + "limit": { + "context": 262144, + "output": 25000 + }, + "cost": { + "input": 0.22, + "output": 2.2 + }, + "blended_cost": 0.715, + "capabilities": { + "entitled": true, + "chat": "conforms", + "deviations": [], + "native_dialect": "none", + "responses": false, + "web_search": false, + "image_generation": false, + "embeddings": false + }, + "use_case": "chat" + } +] diff --git a/scripts/neon.models.ts b/scripts/neon.models.ts new file mode 100644 index 0000000000..169db2d37a --- /dev/null +++ b/scripts/neon.models.ts @@ -0,0 +1,47 @@ +/** + * Hand-written wrapper. DO NOT regenerate this file. + * + * The model data lives in `neon.models.json` (regenerated by + * `fetch-neon-models.ts`). This wrapper re-exports it typed so consumers can + * keep `import { models } from './neon.models'`. + * + * The data is loaded via `JSON.parse(readFileSync(...))` rather than an + * `import ... with { type: 'json' }` attribute so the loader doesn't have + * to support JSON import attributes. + */ + +import { readFileSync } from 'node:fs' +import { dirname, resolve } from 'node:path' +import { fileURLToPath } from 'node:url' + +export interface NeonCatalogModel { + id: string + provider?: string + released?: boolean + /** `'embedding'` for an embedding model. Chat models have no `type`. */ + type?: string + reasoning?: boolean + reasoning_options?: Array<{ type: string; values?: Array }> + /** `false` when the model rejects `temperature` (and `top_p`). */ + temperature?: boolean + modalities?: { + input?: Array + output?: Array + } + capabilities?: { + /** How `/v1/chat/completions` serves the model. */ + chat?: 'conforms' | 'array-content' | 'not-served' | (string & {}) + /** Whether `/openai/v1/responses` serves the model. */ + responses?: boolean + native_dialect?: string + embeddings?: boolean + } + [key: string]: unknown +} + +const __dirname = dirname(fileURLToPath(import.meta.url)) +const DATA_PATH = resolve(__dirname, 'neon.models.json') + +export const models: Array = JSON.parse( + readFileSync(DATA_PATH, 'utf-8'), +) diff --git a/skills/tanstack-ai/SKILL.md b/skills/tanstack-ai/SKILL.md index 7e5c28acfb..a443e3c963 100644 --- a/skills/tanstack-ai/SKILL.md +++ b/skills/tanstack-ai/SKILL.md @@ -74,7 +74,7 @@ lockfile shows which package manager to use. | ----------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------- | | Stream a chat, call tools, agent loops, structured outputs, media | `@tanstack/ai` | `@tanstack/ai#ai-core` | | Talk to a model provider | `@tanstack/ai-openai`, `-anthropic`, `-gemini`, `-grok`, `-groq`, `-mistral`, `-cohere`, `-ollama`, `-bedrock`, `-vertex`, `-cloudflare`, `-fal`, `-elevenlabs`, `-byteplus`, `-perplexity`, `-reactor` | `@tanstack/ai#ai-core/adapter-configuration` | -| Many providers behind one key | `@tanstack/ai-openrouter`, `-vercel-gateway`, `-llmgateway`, `-lovable` | `@tanstack/ai#ai-core/adapter-configuration` | +| Many providers behind one key | `@tanstack/ai-openrouter`, `-vercel-gateway`, `-llmgateway`, `-lovable`, `-neon` | `@tanstack/ai#ai-core/adapter-configuration` | | A chat UI with `useChat` | `@tanstack/ai-react`, `-vue`, `-solid`, `-svelte`, `-preact`, `-angular`, `-octane`, `-remix`; vanilla JS uses `@tanstack/ai-client` | `@tanstack/ai#ai-core/chat-experience` | | Inspect messages, tool calls, and streams while developing | `@tanstack/ai-devtools-core` plus `@tanstack/react-ai-devtools`, `solid-`, `preact-`, or `svelte-ai-devtools` | docs | | Keep chat state on the server, resume runs, durable approvals | `@tanstack/ai-persistence` | `@tanstack/ai-persistence#ai-persistence` | diff --git a/testing/e2e/README.md b/testing/e2e/README.md index e81b8286ec..e28cb81663 100644 --- a/testing/e2e/README.md +++ b/testing/e2e/README.md @@ -4,7 +4,7 @@ End-to-end tests for TanStack AI using Playwright and [aimock](https://github.co **Architecture:** Playwright drives a TanStack Start app (`testing/e2e/`) which routes requests through provider adapters pointing at aimock. Fixtures define mock responses. No real API keys needed. All scenarios (including tool execution flows) use aimock fixtures. Tests run in parallel with per-test `X-Test-Id` isolation. -**Providers tested in CI:** openai, anthropic, gemini, vertex, vertex-grok, vertex-mistral, ollama, groq, grok, openrouter, openrouter-responses, vercel-gateway, vercel-gateway-responses, lovable, lovable-responses, bedrock, bedrock-responses, openai-compatible, openai-compatible-legacy, mistral, byteplus, elevenlabs, llmgateway, cloudflare +**Providers tested in CI:** openai, anthropic, gemini, vertex, vertex-grok, vertex-mistral, ollama, groq, grok, openrouter, openrouter-responses, vercel-gateway, vercel-gateway-responses, lovable, lovable-responses, bedrock, bedrock-responses, openai-compatible, openai-compatible-legacy, mistral, byteplus, elevenlabs, llmgateway, cloudflare, neon, neon-anthropic, neon-responses **Local `pnpm test:e2e`** runs openai, anthropic, and gemini. If none of those providers support a feature, the feature still runs on the providers that do. An explicit `E2E_PROVIDERS` list runs only those providers. Use `E2E_PROVIDERS=*` for the full matrix, or `E2E_PROVIDERS=grok` (comma-separated) to pick providers. Playwright does not retry or record video locally; CI retries twice and keeps the video of a failed test. diff --git a/testing/e2e/package.json b/testing/e2e/package.json index 0e0124a173..ca1b4ed91f 100644 --- a/testing/e2e/package.json +++ b/testing/e2e/package.json @@ -42,6 +42,7 @@ "@tanstack/ai-mcp": "workspace:*", "@tanstack/ai-memory": "workspace:*", "@tanstack/ai-mistral": "workspace:*", + "@tanstack/ai-neon": "workspace:*", "@tanstack/ai-ollama": "workspace:*", "@tanstack/ai-openai": "workspace:*", "@tanstack/ai-openrouter": "workspace:*", diff --git a/testing/e2e/src/lib/feature-support.ts b/testing/e2e/src/lib/feature-support.ts index 607fbb2f5b..8bf6f8fa98 100644 --- a/testing/e2e/src/lib/feature-support.ts +++ b/testing/e2e/src/lib/feature-support.ts @@ -28,6 +28,8 @@ export const matrix: Record> = { 'byteplus', 'llmgateway', 'cloudflare', + 'neon', + 'neon-anthropic', ]), 'one-shot-text': new Set([ 'openai', @@ -49,12 +51,16 @@ export const matrix: Record> = { 'byteplus', 'llmgateway', 'cloudflare', + 'neon', + 'neon-anthropic', ]), - // BytePlus and LLM Gateway both stream their reasoning trace as + // BytePlus, LLM Gateway and Neon all stream their reasoning trace as // `delta.reasoning_content`, which is exactly the field aimock's // OpenAI-compatible chunk builder emits for a fixture's `reasoning` channel — // so each adapter's `extractReasoning` override is exercised end-to-end // against the shared fixture. + // `neon-anthropic` is the Anthropic adapter on Neon's Messages route, so it + // gets the same Anthropic thinking stream as `anthropic`. // `openai-compatible-legacy` re-keys aimock's reasoning frames to the // pre-July-2025 `response.reasoning.delta` event name that OpenAI-compatible // endpoints frozen on the older spec still emit (see providers.ts), pinning @@ -69,6 +75,8 @@ export const matrix: Record> = { 'llmgateway', 'cloudflare', 'openai-compatible-legacy', + 'neon', + 'neon-anthropic', ]), 'multi-turn-reasoning': new Set(['anthropic']), 'multi-turn': new Set([ @@ -91,6 +99,8 @@ export const matrix: Record> = { 'byteplus', 'llmgateway', 'cloudflare', + 'neon', + 'neon-anthropic', ]), 'tool-calling': new Set([ 'openai', @@ -115,6 +125,9 @@ export const matrix: Record> = { 'byteplus', 'llmgateway', 'cloudflare', + 'neon', + 'neon-anthropic', + 'neon-responses', ]), // OpenAI only: aimock's journal keeps `tool_choice` for Responses requests, // but stores Anthropic requests in OpenAI shape without it. Anthropic is @@ -139,6 +152,8 @@ export const matrix: Record> = { 'byteplus', 'llmgateway', 'cloudflare', + 'neon', + 'neon-anthropic', ]), // Gemini excluded: approval flow timing issues with Gemini's streaming format // Vertex uses the same Gemini stream, so it is excluded for the same reason. @@ -160,6 +175,8 @@ export const matrix: Record> = { 'byteplus', 'llmgateway', 'cloudflare', + 'neon', + 'neon-anthropic', ]), // Ollama excluded: aimock doesn't support content+toolCalls for /api/chat format 'text-tool-text': new Set([ @@ -181,6 +198,8 @@ export const matrix: Record> = { 'byteplus', 'llmgateway', 'cloudflare', + 'neon', + 'neon-anthropic', ]), 'structured-output': new Set([ 'openai', @@ -202,6 +221,8 @@ export const matrix: Record> = { 'byteplus', 'llmgateway', 'cloudflare', + 'neon', + 'neon-anthropic', ]), // Streaming structured output: only providers with native streaming JSON // schema support are listed here. Other providers fall back to the @@ -222,6 +243,7 @@ export const matrix: Record> = { 'byteplus', 'llmgateway', 'cloudflare', + 'neon', ]), // Multi-turn structured output: every turn produces its own typed // `structured-output` part on the assistant message, and historical @@ -252,6 +274,8 @@ export const matrix: Record> = { 'byteplus', 'llmgateway', 'cloudflare', + 'neon', + 'neon-anthropic', ]), 'agentic-structured': new Set([ 'openai', @@ -273,6 +297,8 @@ export const matrix: Record> = { 'byteplus', 'llmgateway', 'cloudflare', + 'neon', + 'neon-anthropic', ]), // Native-combined-mode adapters only. Each provider's default test model // (or per-feature override in `features.ts`) must opt into combined mode @@ -308,6 +334,8 @@ export const matrix: Record> = { 'byteplus', 'llmgateway', 'cloudflare', + 'neon', + 'neon-anthropic', ]), // OpenAI only: this feature exercises the Responses adapter's PDF // `input_file` conversion (base64 `file_data` + filename). @@ -325,6 +353,8 @@ export const matrix: Record> = { 'byteplus', 'llmgateway', 'cloudflare', + 'neon', + 'neon-anthropic', ]), // byteplus excluded: @tanstack/ai-byteplus ships no summarize adapter — // Ark has no summarization endpoint, and api.summarize.ts builds a @@ -348,6 +378,9 @@ export const matrix: Record> = { 'mistral', 'llmgateway', 'cloudflare', + 'neon', + 'neon-anthropic', + 'neon-responses', ]), 'summarize-stream': new Set([ 'openai', @@ -367,6 +400,9 @@ export const matrix: Record> = { 'mistral', 'llmgateway', 'cloudflare', + 'neon', + 'neon-anthropic', + 'neon-responses', ]), // Embedding (Promise-based `embed()` activity, no streaming). aimock 1.34 // natively mocks OpenAI's /v1/embeddings (JSON fixture in @@ -388,6 +424,7 @@ export const matrix: Record> = { 'mistral', 'vercel-gateway', 'lovable', + 'neon', ]), // Gemini excluded: its image adapter uses the Interactions API, which aimock // doesn't mock. gemini-native-image-wire and gemini-image-ga-models cover it. diff --git a/testing/e2e/src/lib/providers.ts b/testing/e2e/src/lib/providers.ts index 40d2453348..1320e006b0 100644 --- a/testing/e2e/src/lib/providers.ts +++ b/testing/e2e/src/lib/providers.ts @@ -23,6 +23,7 @@ import { createMistralText } from '@tanstack/ai-mistral' import { createBytePlusText } from '@tanstack/ai-byteplus' import { createLLMGatewayText } from '@tanstack/ai-llmgateway' import { createCloudflareText } from '@tanstack/ai-cloudflare' +import { createNeonText } from '@tanstack/ai-neon' import { HTTPClient } from '@openrouter/sdk' import type { AnyTextAdapter } from '@tanstack/ai' import type { BytePlusChatModel } from '@tanstack/ai-byteplus' @@ -115,6 +116,9 @@ const defaultModels: Record = { elevenlabs: '', llmgateway: 'gpt-5.6-terra', cloudflare: '@cf/zai-org/glm-5.3-flash', + neon: 'gemini-3-8-flash', + 'neon-anthropic': 'claude-sonnet-4-5', + 'neon-responses': 'gpt-5-3-codex', } export function createTextAdapter( @@ -409,6 +413,35 @@ export function createTextAdapter( defaultHeaders: testHeaders, }), }), + // Neon's baseURL is the bare branch host; the adapter appends each + // route. aimock serves `/v1/chat/completions` and strips the `/openai/` + // prefix of `/openai/v1/responses` itself. + neon: () => + createChatOptions({ + adapter: createNeonText(model as 'gemini-3-8-flash', DUMMY_KEY, { + baseURL: base, + defaultHeaders: testHeaders, + }), + }), + // Claude goes to Neon's `/anthropic/v1/messages`, which aimock does not + // map, so this fetch moves it to aimock's `/v1/messages`. The unit tests + // pin the real path. + 'neon-anthropic': () => + createChatOptions({ + adapter: createNeonText(model as 'claude-sonnet-4-5', DUMMY_KEY, { + baseURL: base, + defaultHeaders: testHeaders, + fetch: (input, init) => + fetch(String(input).replace('/anthropic/v1/', '/v1/'), init), + }), + }), + 'neon-responses': () => + createChatOptions({ + adapter: createNeonText(model as 'gpt-5-3-codex', DUMMY_KEY, { + baseURL: base, + defaultHeaders: testHeaders, + }), + }), } return factories[provider]() diff --git a/testing/e2e/src/lib/types.ts b/testing/e2e/src/lib/types.ts index f8b111de16..c44f44c583 100644 --- a/testing/e2e/src/lib/types.ts +++ b/testing/e2e/src/lib/types.ts @@ -25,6 +25,9 @@ export type Provider = | 'elevenlabs' | 'llmgateway' | 'cloudflare' + | 'neon' + | 'neon-anthropic' + | 'neon-responses' export type Feature = | 'chat' @@ -87,6 +90,9 @@ export const ALL_PROVIDERS: Provider[] = [ 'elevenlabs', 'llmgateway', 'cloudflare', + 'neon', + 'neon-anthropic', + 'neon-responses', ] export const ALL_FEATURES: Feature[] = [ diff --git a/testing/e2e/src/routeTree.gen.ts b/testing/e2e/src/routeTree.gen.ts index 1cd9853044..a59b1b5cd6 100644 --- a/testing/e2e/src/routeTree.gen.ts +++ b/testing/e2e/src/routeTree.gen.ts @@ -146,6 +146,7 @@ import { Route as ApiMiddlewareTestRouteImport } from './routes/api.middleware-t import { Route as ApiMistralStrictToolNullWireRouteImport } from './routes/api.mistral-strict-tool-null-wire' import { Route as ApiMoonshotUsageDetailsRouteImport } from './routes/api.moonshot-usage-details' import { Route as ApiMultimodalToolResultWireRouteImport } from './routes/api.multimodal-tool-result-wire' +import { Route as ApiNeonWireRouteImport } from './routes/api.neon-wire' import { Route as ApiNonStreamingRunErrorRouteImport } from './routes/api.non-streaming-run-error' import { Route as ApiOpenaiCompletedResponseTextRouteImport } from './routes/api.openai-completed-response-text' import { Route as ApiOpenaiComputerUseWireRouteImport } from './routes/api.openai-computer-use-wire' @@ -921,6 +922,11 @@ const ApiMultimodalToolResultWireRoute = path: '/api/multimodal-tool-result-wire', getParentRoute: () => rootRouteImport, } as any) +const ApiNeonWireRoute = ApiNeonWireRouteImport.update({ + id: '/api/neon-wire', + path: '/api/neon-wire', + getParentRoute: () => rootRouteImport, +} as any) const ApiNonStreamingRunErrorRoute = ApiNonStreamingRunErrorRouteImport.update({ id: '/api/non-streaming-run-error', path: '/api/non-streaming-run-error', @@ -1385,6 +1391,7 @@ export interface FileRoutesByFullPath { '/api/mistral-strict-tool-null-wire': typeof ApiMistralStrictToolNullWireRoute '/api/moonshot-usage-details': typeof ApiMoonshotUsageDetailsRoute '/api/multimodal-tool-result-wire': typeof ApiMultimodalToolResultWireRoute + '/api/neon-wire': typeof ApiNeonWireRoute '/api/non-streaming-run-error': typeof ApiNonStreamingRunErrorRoute '/api/openai-completed-response-text': typeof ApiOpenaiCompletedResponseTextRoute '/api/openai-computer-use-wire': typeof ApiOpenaiComputerUseWireRoute @@ -1585,6 +1592,7 @@ export interface FileRoutesByTo { '/api/mistral-strict-tool-null-wire': typeof ApiMistralStrictToolNullWireRoute '/api/moonshot-usage-details': typeof ApiMoonshotUsageDetailsRoute '/api/multimodal-tool-result-wire': typeof ApiMultimodalToolResultWireRoute + '/api/neon-wire': typeof ApiNeonWireRoute '/api/non-streaming-run-error': typeof ApiNonStreamingRunErrorRoute '/api/openai-completed-response-text': typeof ApiOpenaiCompletedResponseTextRoute '/api/openai-computer-use-wire': typeof ApiOpenaiComputerUseWireRoute @@ -1786,6 +1794,7 @@ export interface FileRoutesById { '/api/mistral-strict-tool-null-wire': typeof ApiMistralStrictToolNullWireRoute '/api/moonshot-usage-details': typeof ApiMoonshotUsageDetailsRoute '/api/multimodal-tool-result-wire': typeof ApiMultimodalToolResultWireRoute + '/api/neon-wire': typeof ApiNeonWireRoute '/api/non-streaming-run-error': typeof ApiNonStreamingRunErrorRoute '/api/openai-completed-response-text': typeof ApiOpenaiCompletedResponseTextRoute '/api/openai-computer-use-wire': typeof ApiOpenaiComputerUseWireRoute @@ -1988,6 +1997,7 @@ export interface FileRouteTypes { | '/api/mistral-strict-tool-null-wire' | '/api/moonshot-usage-details' | '/api/multimodal-tool-result-wire' + | '/api/neon-wire' | '/api/non-streaming-run-error' | '/api/openai-completed-response-text' | '/api/openai-computer-use-wire' @@ -2188,6 +2198,7 @@ export interface FileRouteTypes { | '/api/mistral-strict-tool-null-wire' | '/api/moonshot-usage-details' | '/api/multimodal-tool-result-wire' + | '/api/neon-wire' | '/api/non-streaming-run-error' | '/api/openai-completed-response-text' | '/api/openai-computer-use-wire' @@ -2388,6 +2399,7 @@ export interface FileRouteTypes { | '/api/mistral-strict-tool-null-wire' | '/api/moonshot-usage-details' | '/api/multimodal-tool-result-wire' + | '/api/neon-wire' | '/api/non-streaming-run-error' | '/api/openai-completed-response-text' | '/api/openai-computer-use-wire' @@ -2589,6 +2601,7 @@ export interface RootRouteChildren { ApiMistralStrictToolNullWireRoute: typeof ApiMistralStrictToolNullWireRoute ApiMoonshotUsageDetailsRoute: typeof ApiMoonshotUsageDetailsRoute ApiMultimodalToolResultWireRoute: typeof ApiMultimodalToolResultWireRoute + ApiNeonWireRoute: typeof ApiNeonWireRoute ApiNonStreamingRunErrorRoute: typeof ApiNonStreamingRunErrorRoute ApiOpenaiCompletedResponseTextRoute: typeof ApiOpenaiCompletedResponseTextRoute ApiOpenaiComputerUseWireRoute: typeof ApiOpenaiComputerUseWireRoute @@ -3609,6 +3622,13 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof ApiMultimodalToolResultWireRouteImport parentRoute: typeof rootRouteImport } + '/api/neon-wire': { + id: '/api/neon-wire' + path: '/api/neon-wire' + fullPath: '/api/neon-wire' + preLoaderRoute: typeof ApiNeonWireRouteImport + parentRoute: typeof rootRouteImport + } '/api/non-streaming-run-error': { id: '/api/non-streaming-run-error' path: '/api/non-streaming-run-error' @@ -4239,6 +4259,7 @@ const rootRouteChildren: RootRouteChildren = { ApiMistralStrictToolNullWireRoute: ApiMistralStrictToolNullWireRoute, ApiMoonshotUsageDetailsRoute: ApiMoonshotUsageDetailsRoute, ApiMultimodalToolResultWireRoute: ApiMultimodalToolResultWireRoute, + ApiNeonWireRoute: ApiNeonWireRoute, ApiNonStreamingRunErrorRoute: ApiNonStreamingRunErrorRoute, ApiOpenaiCompletedResponseTextRoute: ApiOpenaiCompletedResponseTextRoute, ApiOpenaiComputerUseWireRoute: ApiOpenaiComputerUseWireRoute, diff --git a/testing/e2e/src/routes/api.embedding.ts b/testing/e2e/src/routes/api.embedding.ts index fd923acb10..f1db58c557 100644 --- a/testing/e2e/src/routes/api.embedding.ts +++ b/testing/e2e/src/routes/api.embedding.ts @@ -6,6 +6,7 @@ import { createMistralEmbedding } from '@tanstack/ai-mistral' import { createOllamaEmbedding } from '@tanstack/ai-ollama' import { createVercelGatewayEmbedding } from '@tanstack/ai-vercel-gateway' import { createLovableEmbedding } from '@tanstack/ai-lovable' +import { createNeonEmbedding } from '@tanstack/ai-neon' import type { Provider } from '@/lib/types' const LLMOCK_BASE = process.env.LLMOCK_URL || 'http://127.0.0.1:4010' @@ -77,6 +78,12 @@ function createEmbeddingAdapter( baseURL: openaiUrl(aimockPort), defaultHeaders: headers, }), + // Neon's baseURL is the bare host; the adapter appends `/v1`. + neon: () => + createNeonEmbedding('qwen3-embedding-0-6b', DUMMY_KEY, { + baseURL: llmockBase(aimockPort), + defaultHeaders: headers, + }), } return factories[provider]?.() } diff --git a/testing/e2e/src/routes/api.neon-wire.ts b/testing/e2e/src/routes/api.neon-wire.ts new file mode 100644 index 0000000000..b53954b959 --- /dev/null +++ b/testing/e2e/src/routes/api.neon-wire.ts @@ -0,0 +1,94 @@ +import { createFileRoute } from '@tanstack/react-router' +import { chat, summarize, toServerSentEventsResponse } from '@tanstack/ai' +import { createNeonSummarize, createNeonText } from '@tanstack/ai-neon' + +// Neon's catalog reports Gemini reasoning_tokens at the root of usage. +// Keep this wire shape independent of aimock's standard OpenAI usage mapper. +export const Route = createFileRoute('/api/neon-wire')({ + server: { + handlers: { + POST: async ({ request }) => { + const params = new URL(request.url).searchParams + if (params.get('activity') === 'summarize') { + const requests: Array<{ url: string; body: unknown }> = [] + const adapter = createNeonSummarize('gpt-6-1-sol', 'test-token', { + api: 'responses', + baseURL: process.env.LLMOCK_URL || 'http://127.0.0.1:4010', + fetch: async (input, init) => { + const outgoing = new Request(input, init) + requests.push({ + url: outgoing.url, + body: await outgoing.clone().json(), + }) + return fetch(outgoing) + }, + }) + const options = { + adapter, + threadId: 'neon-wire', + text: '[summarize] The Fender Stratocaster is a versatile electric guitar', + maxLength: 73, + modelOptions: { max_output_tokens: 91 }, + } + let summary = '' + if (params.get('stream') === 'true') { + for await (const chunk of summarize({ ...options, stream: true })) { + if (chunk.type === 'RUN_ERROR') throw new Error(chunk.message) + if (chunk.type === 'TEXT_MESSAGE_CONTENT') summary += chunk.delta + } + } else { + summary = (await summarize({ ...options, stream: false })).summary + } + return Response.json({ summary, requests }) + } + + const model = 'gemini-3-7-flash' + const envelope = { + id: null, + object: 'chat.completion.chunk', + created: 1, + model, + } + const chunks = [ + { + ...envelope, + choices: [ + { + index: 0, + delta: { content: [{ type: 'text', text: 'Hello' }] }, + finish_reason: 'stop', + }, + ], + }, + { + ...envelope, + choices: [], + usage: { + prompt_tokens: 180, + completion_tokens: 7, + total_tokens: 459, + reasoning_tokens: 272, + }, + }, + ] + const adapter = createNeonText(model, 'test-token', { + baseURL: 'https://neon.test', + fetch: async () => + new Response( + chunks + .map((chunk) => `data: ${JSON.stringify(chunk)}\n\n`) + .join('') + 'data: [DONE]\n\n', + { headers: { 'content-type': 'text/event-stream' } }, + ), + }) + return toServerSentEventsResponse( + chat({ + adapter, + messages: [{ role: 'user', content: 'Hello' }], + threadId: 'neon-wire', + }), + ) + }, + }, + }, +}) diff --git a/testing/e2e/src/routes/api.summarize.ts b/testing/e2e/src/routes/api.summarize.ts index 5228ba504a..7e8040d7bf 100644 --- a/testing/e2e/src/routes/api.summarize.ts +++ b/testing/e2e/src/routes/api.summarize.ts @@ -16,6 +16,7 @@ import { createCloudflareSummarize } from '@tanstack/ai-cloudflare' import { createOpenRouterSummarize } from '@tanstack/ai-openrouter' import { createVercelGatewaySummarize } from '@tanstack/ai-vercel-gateway' import { createLovableSummarize } from '@tanstack/ai-lovable' +import { createNeonSummarize } from '@tanstack/ai-neon' import { HTTPClient } from '@openrouter/sdk' import type { Provider } from '@/lib/types' @@ -148,6 +149,25 @@ function createSummarizeAdapter( baseURL: openaiUrl(aimockPort), defaultHeaders: headers, }), + // Neon's baseURL is the bare host; the adapter appends `/v1`. + neon: () => + createNeonSummarize('gemini-3-8-flash', DUMMY_KEY, { + baseURL: llmockBase(aimockPort), + defaultHeaders: headers, + }), + 'neon-anthropic': () => + createNeonSummarize('claude-opus-5-5', DUMMY_KEY, { + baseURL: llmockBase(aimockPort), + defaultHeaders: headers, + fetch: (input, init) => + fetch(String(input).replace('/anthropic/v1/', '/v1/'), init), + }), + 'neon-responses': () => + createNeonSummarize('gpt-6-1-sol', DUMMY_KEY, { + api: 'responses', + baseURL: llmockBase(aimockPort), + defaultHeaders: headers, + }), } return factories[provider]?.() } diff --git a/testing/e2e/tests/neon-sampling-wire.spec.ts b/testing/e2e/tests/neon-sampling-wire.spec.ts new file mode 100644 index 0000000000..5749dcae88 --- /dev/null +++ b/testing/e2e/tests/neon-sampling-wire.spec.ts @@ -0,0 +1,88 @@ +import { test, expect } from './fixtures' + +for (const provider of ['neon', 'neon-anthropic']) { + for (const stream of [false, true]) { + test(`${provider} -- summarize omits rejected sampling (stream=${stream})`, async ({ + request, + aimockPort, + testId, + }) => { + const response = await request.post('/api/summarize', { + data: { + provider, + text: '[summarize] The Fender Stratocaster is a versatile electric guitar', + maxLength: 73, + stream, + aimockPort, + testId, + }, + }) + expect(response.ok()).toBe(true) + expect(await response.text()).toContain('Stratocaster') + + const journal = await request.get( + `http://127.0.0.1:${aimockPort}/v1/_requests`, + ) + const entries = (await journal.json()) as Array<{ + headers?: Record + body?: Record + }> + const calls = entries.filter( + (entry) => entry.headers?.['x-test-id'] === testId, + ) + expect(calls).toHaveLength(1) + expect(calls[0]?.body?.model).toBe( + provider === 'neon' ? 'gemini-3-8-flash' : 'claude-opus-5-5', + ) + expect(calls[0]?.body?.max_tokens).toBe(73) + expect(calls[0]?.body).not.toHaveProperty('temperature') + expect(calls[0]?.body).not.toHaveProperty('top_p') + }) + } +} + +for (const stream of [false, true]) { + test(`neon-responses -- summarize honors Responses token options (stream=${stream})`, async ({ + request, + }) => { + const response = await request.post( + `/api/neon-wire?activity=summarize&stream=${stream}`, + ) + expect(response.ok()).toBe(true) + const { summary, requests } = await response.json() + expect(summary).toContain('Stratocaster') + expect(requests).toHaveLength(1) + expect(requests[0].url).toContain('/openai/v1/responses') + expect(requests[0].body).toMatchObject({ + model: 'gpt-6-1-sol', + max_output_tokens: 91, + }) + expect(requests[0].body).not.toHaveProperty('max_tokens') + expect(requests[0].body).not.toHaveProperty('temperature') + }) +} + +test('neon -- Gemini reasoning usage reaches the SSE client', async ({ + request, +}) => { + const response = await request.post('/api/neon-wire') + expect(response.ok()).toBe(true) + const events = (await response.text()) + .split('\n') + .filter((line) => line.startsWith('data: {')) + .map((line) => JSON.parse(line.slice(6))) + expect(events.some((event) => event.type === 'RUN_ERROR')).toBe(false) + expect( + events.find((event) => event.type === 'TEXT_MESSAGE_CONTENT'), + ).toMatchObject({ delta: 'Hello' }) + expect(events.find((event) => event.type === 'RUN_FINISHED')).toMatchObject({ + usage: [ + { + inputTokens: 180, + outputTokens: 7, + totalTokens: 459, + reasoningTokens: 272, + }, + ], + }) +}) diff --git a/testing/e2e/tests/test-matrix.ts b/testing/e2e/tests/test-matrix.ts index e3f1d618df..52350d7a0e 100644 --- a/testing/e2e/tests/test-matrix.ts +++ b/testing/e2e/tests/test-matrix.ts @@ -38,6 +38,9 @@ export const providers: Provider[] = [ 'elevenlabs', 'llmgateway', 'cloudflare', + 'neon', + 'neon-anthropic', + 'neon-responses', ] export { isSupported } diff --git a/tools/workspace-plugin/src/generators/react-app/files/package.json b/tools/workspace-plugin/src/generators/react-app/files/package.json index 82a399311e..97293edfc1 100644 --- a/tools/workspace-plugin/src/generators/react-app/files/package.json +++ b/tools/workspace-plugin/src/generators/react-app/files/package.json @@ -41,6 +41,7 @@ "@tanstack/ai-mcp": "workspace:*", "@tanstack/ai-memory": "workspace:*", "@tanstack/ai-mistral": "workspace:*", + "@tanstack/ai-neon": "workspace:*", "@tanstack/ai-ollama": "workspace:*", "@tanstack/ai-openai": "workspace:*", "@tanstack/ai-opencode": "workspace:*",