From ded65742c5b5c96807448fbd868a994c5bffa1cd Mon Sep 17 00:00:00 2001 From: Alex Castillo Date: Wed, 4 Nov 2026 03:08:13 -0500 Subject: [PATCH 1/2] fix(ai-openai): split embed() inputs into requests of at most 2048 Co-Authored-By: Claude Sonnet 5.5 --- .changeset/ai-openai-embed-batch.md | 5 ++ packages/ai-openai/src/adapters/embedding.ts | 61 +++++++++++++------ .../ai-openai/tests/embedding-adapter.test.ts | 39 ++++++++++++ 3 files changed, 87 insertions(+), 18 deletions(-) create mode 100644 .changeset/ai-openai-embed-batch.md diff --git a/.changeset/ai-openai-embed-batch.md b/.changeset/ai-openai-embed-batch.md new file mode 100644 index 0000000000..c97edca756 --- /dev/null +++ b/.changeset/ai-openai-embed-batch.md @@ -0,0 +1,5 @@ +--- +'@tanstack/ai-openai': patch +--- + +`embed()` with the OpenAI adapter now sends inputs in requests of at most 2048, the endpoint's limit. Before, more than 2048 texts failed with a 400 error. diff --git a/packages/ai-openai/src/adapters/embedding.ts b/packages/ai-openai/src/adapters/embedding.ts index 88ec354fc2..817f0813e3 100644 --- a/packages/ai-openai/src/adapters/embedding.ts +++ b/packages/ai-openai/src/adapters/embedding.ts @@ -23,6 +23,12 @@ import type { OpenAIClientConfig } from '../utils/client' */ export interface OpenAIEmbeddingConfig extends OpenAIClientConfig {} +/** + * The embeddings endpoint rejects requests with more than 2048 inputs, so + * larger arrays are sent as several requests. + */ +const OPENAI_MAX_BATCH_SIZE = 2048 + /** * OpenAI Embedding Adapter * @@ -30,7 +36,7 @@ export interface OpenAIEmbeddingConfig extends OpenAIClientConfig {} * Supports text-embedding-3-small and text-embedding-3-large. * * Features: - * - Batch embedding (one request for the whole input array) + * - Batch embedding (inputs are sent in requests of at most 2048) * - Matryoshka dimension reduction via the top-level `dimensions` option */ export class OpenAIEmbeddingAdapter< @@ -61,40 +67,59 @@ export class OpenAIEmbeddingAdapter< // fields (server routes often pass modelOptions through from untyped // client input). encoding_format is pinned to float so vectors are // always number[]. - const request: OpenAI_SDK.EmbeddingCreateParams = { + const baseRequest = { ...modelOptions, model, - input: texts, encoding_format: 'float', - } - if (options.dimensions !== undefined) { - request.dimensions = options.dimensions - } + } as const + const dimensions = + options.dimensions !== undefined + ? { dimensions: options.dimensions } + : {} logger.request( `activity=embed provider=${this.name} model=${model} inputs=${texts.length}`, { provider: this.name, model }, ) - const response = options.abortSignal - ? await this.client.embeddings.create(request, { - signal: options.abortSignal, - }) - : await this.client.embeddings.create(request) + const embeddings: EmbeddingResult['embeddings'] = [] + let promptTokens = 0 + let totalTokens = 0 + + for ( + let start = 0; + start < texts.length; + start += OPENAI_MAX_BATCH_SIZE + ) { + const request: OpenAI_SDK.EmbeddingCreateParams = { + ...baseRequest, + ...dimensions, + input: texts.slice(start, start + OPENAI_MAX_BATCH_SIZE), + } + + const response = options.abortSignal + ? await this.client.embeddings.create(request, { + signal: options.abortSignal, + }) + : await this.client.embeddings.create(request) + + for (const item of response.data) { + embeddings.push({ vector: item.embedding, index: start + item.index }) + } + promptTokens += response.usage.prompt_tokens + totalTokens += response.usage.total_tokens + } const usage: TokenUsage = { - promptTokens: response.usage.prompt_tokens, + promptTokens, completionTokens: 0, - totalTokens: response.usage.total_tokens, + totalTokens, } return { id: generateId(this.name), model, - embeddings: response.data.map((item) => ({ - vector: item.embedding, - index: item.index, - })), + embeddings, usage, } } catch (error: unknown) { diff --git a/packages/ai-openai/tests/embedding-adapter.test.ts b/packages/ai-openai/tests/embedding-adapter.test.ts index 349129086c..41f007cb91 100644 --- a/packages/ai-openai/tests/embedding-adapter.test.ts +++ b/packages/ai-openai/tests/embedding-adapter.test.ts @@ -87,6 +87,45 @@ describe('OpenAI Embedding Adapter', () => { }) }) + it('splits inputs over 2048 into batches and keeps order, indexes and usage', async () => { + const adapter = new TestOpenAIEmbeddingAdapter( + { apiKey: 'test' }, + 'text-embedding-3-small', + ) + const spy = adapter.spyOnEmbeddingsCreate() + // Like the real API: reject any request with more than 2048 inputs. + spy.mockImplementation((params) => { + const input = params.input as Array + if (input.length > 2048) { + return Promise.reject( + new Error('400 max 2048 embedding inputs per request'), + ) as never + } + return Promise.resolve( + mockResponse(input.map((text) => [Number(text)])), + ) as never + }) + + const texts = Array.from({ length: 5000 }, (_, i) => String(i)) + const result = await adapter.createEmbeddings({ + model: 'text-embedding-3-small', + input: texts, + logger: testLogger, + }) + + expect(spy.mock.calls.map(([params]) => params.input.length)).toEqual([ + 2048, 2048, 904, + ]) + expect(result.embeddings).toHaveLength(5000) + expect(result.embeddings[2048]).toEqual({ vector: [2048], index: 2048 }) + expect(result.embeddings[4999]).toEqual({ vector: [4999], index: 4999 }) + expect(result.usage).toEqual({ + promptTokens: 21, + completionTokens: 0, + totalTokens: 21, + }) + }) + it('passes the top-level dimensions option through', async () => { const adapter = new TestOpenAIEmbeddingAdapter( { apiKey: 'test' }, From a1688ca07426152240c8a23b6572879a5394fabe Mon Sep 17 00:00:00 2001 From: Tom Beckenham <34339192+tombeckenham@users.noreply.github.com> Date: Thu, 5 Nov 2026 21:29:25 +1100 Subject: [PATCH 2/2] fix(ai-openai): note 300k token cap, pin empty input, tighten batch test - ponytail comment and changeset name OpenAI's 300k-token-per-request cap - test asserts dimensions/modelOptions on every batch and distinct usage sums - test pins empty input: no request, empty result --- .changeset/ai-openai-embed-batch.md | 2 +- packages/ai-openai/src/adapters/embedding.ts | 7 ++- .../ai-openai/tests/embedding-adapter.test.ts | 47 +++++++++++++++++-- 3 files changed, 48 insertions(+), 8 deletions(-) diff --git a/.changeset/ai-openai-embed-batch.md b/.changeset/ai-openai-embed-batch.md index c97edca756..1ca9efa142 100644 --- a/.changeset/ai-openai-embed-batch.md +++ b/.changeset/ai-openai-embed-batch.md @@ -2,4 +2,4 @@ '@tanstack/ai-openai': patch --- -`embed()` with the OpenAI adapter now sends inputs in requests of at most 2048, the endpoint's limit. Before, more than 2048 texts failed with a 400 error. +`embed()` with the OpenAI adapter now sends inputs in sequential requests of at most 2048 inputs each, the endpoint's limit, and sums the token usage. Before, more than 2048 texts failed with a 400 error. OpenAI also limits one request to 300,000 tokens across all inputs, so a large array of long texts can still fail. Split those calls yourself. diff --git a/packages/ai-openai/src/adapters/embedding.ts b/packages/ai-openai/src/adapters/embedding.ts index 35a455c847..f497ddb079 100644 --- a/packages/ai-openai/src/adapters/embedding.ts +++ b/packages/ai-openai/src/adapters/embedding.ts @@ -26,8 +26,11 @@ export interface OpenAIEmbeddingConfig extends OpenAIClientConfig {} /** * The embeddings endpoint rejects requests with more than 2048 inputs, so - * larger arrays are sent as several requests. + * larger arrays are sent as several sequential requests. */ +// ponytail: splits by input count only. OpenAI also caps a request at 300k +// tokens summed across inputs, so 2048 long texts can still 400. Add a +// token-budget split if that bites (needs a tokeniser or char estimate). const OPENAI_MAX_BATCH_SIZE = 2048 /** @@ -37,7 +40,7 @@ const OPENAI_MAX_BATCH_SIZE = 2048 * Supports text-embedding-3-small and text-embedding-3-large. * * Features: - * - Batch embedding (inputs are sent in requests of at most 2048) + * - Batch embedding (sequential requests of at most 2048 inputs each; usage is summed) * - Matryoshka dimension reduction via the top-level `dimensions` option */ export class OpenAIEmbeddingAdapter< diff --git a/packages/ai-openai/tests/embedding-adapter.test.ts b/packages/ai-openai/tests/embedding-adapter.test.ts index 41f007cb91..24cedf5a28 100644 --- a/packages/ai-openai/tests/embedding-adapter.test.ts +++ b/packages/ai-openai/tests/embedding-adapter.test.ts @@ -101,28 +101,65 @@ describe('OpenAI Embedding Adapter', () => { new Error('400 max 2048 embedding inputs per request'), ) as never } - return Promise.resolve( - mockResponse(input.map((text) => [Number(text)])), - ) as never + // Distinct prompt/total counts per batch prove both fields are summed. + return Promise.resolve({ + ...mockResponse(input.map((text) => [Number(text)])), + usage: { + prompt_tokens: input.length, + total_tokens: input.length + 1, + }, + }) as never }) const texts = Array.from({ length: 5000 }, (_, i) => String(i)) const result = await adapter.createEmbeddings({ model: 'text-embedding-3-small', input: texts, + dimensions: 256, + modelOptions: { user: 'u' }, logger: testLogger, }) expect(spy.mock.calls.map(([params]) => params.input.length)).toEqual([ 2048, 2048, 904, ]) + for (const [params] of spy.mock.calls) { + expect(params).toMatchObject({ + model: 'text-embedding-3-small', + encoding_format: 'float', + dimensions: 256, + user: 'u', + }) + } expect(result.embeddings).toHaveLength(5000) expect(result.embeddings[2048]).toEqual({ vector: [2048], index: 2048 }) expect(result.embeddings[4999]).toEqual({ vector: [4999], index: 4999 }) expect(result.usage).toEqual({ - promptTokens: 21, + promptTokens: 5000, + completionTokens: 0, + totalTokens: 5003, + }) + }) + + it('sends no request for an empty input array', async () => { + const adapter = new TestOpenAIEmbeddingAdapter( + { apiKey: 'test' }, + 'text-embedding-3-small', + ) + const spy = adapter.spyOnEmbeddingsCreate() + + const result = await adapter.createEmbeddings({ + model: 'text-embedding-3-small', + input: [], + logger: testLogger, + }) + + expect(spy).not.toHaveBeenCalled() + expect(result.embeddings).toEqual([]) + expect(result.usage).toEqual({ + promptTokens: 0, completionTokens: 0, - totalTokens: 21, + totalTokens: 0, }) })