From 2bee546961296a1a7cc7363fd47e9c0d774259a0 Mon Sep 17 00:00:00 2001 From: pltoledo Date: Tue, 29 Sep 2026 19:16:47 -0300 Subject: [PATCH 1/2] feat(ai-cloudflare): send defaultHeaders as extraHeaders on binding runs Binding mode dropped every request header, so there was no way to set x-session-affinity for Workers AI prompt caching. The binding config now takes defaultHeaders and forwards them as the extraHeaders run option, next to gateway. The OpenAI SDK's own headers are still not forwarded. Co-Authored-By: Claude Opus 5.5 --- .changeset/cloudflare-binding-headers.md | 5 ++ docs/adapters/cloudflare.md | 34 ++++++++++ docs/config.json | 2 +- packages/ai-cloudflare/src/adapters/text.ts | 2 +- packages/ai-cloudflare/src/utils/config.ts | 5 ++ packages/ai-cloudflare/src/utils/fetch.ts | 16 +++-- packages/ai-cloudflare/src/utils/run.ts | 8 ++- .../tests/cloudflare-adapter.test.ts | 41 ++++++++++++ .../chat/cloudflare-binding-headers-wire.json | 12 ++++ testing/e2e/src/routeTree.gen.ts | 22 +++++++ .../api.cloudflare-binding-headers-wire.ts | 66 +++++++++++++++++++ .../cloudflare-binding-headers-wire.spec.ts | 35 ++++++++++ 12 files changed, 239 insertions(+), 9 deletions(-) create mode 100644 .changeset/cloudflare-binding-headers.md create mode 100644 testing/e2e/fixtures/chat/cloudflare-binding-headers-wire.json create mode 100644 testing/e2e/src/routes/api.cloudflare-binding-headers-wire.ts create mode 100644 testing/e2e/tests/cloudflare-binding-headers-wire.spec.ts diff --git a/.changeset/cloudflare-binding-headers.md b/.changeset/cloudflare-binding-headers.md new file mode 100644 index 0000000000..39efe98bda --- /dev/null +++ b/.changeset/cloudflare-binding-headers.md @@ -0,0 +1,5 @@ +--- +'@tanstack/ai-cloudflare': minor +--- + +Add `defaultHeaders` to the binding config. Every binding run sends them as the Workers AI `extraHeaders` option, next to `gateway`, so you can set `x-session-affinity` for prompt caching when you use `{ binding: env.AI }`. diff --git a/docs/adapters/cloudflare.md b/docs/adapters/cloudflare.md index f06a80fbce..af17eaf911 100644 --- a/docs/adapters/cloudflare.md +++ b/docs/adapters/cloudflare.md @@ -95,6 +95,40 @@ const stream = chat({ The text and summarize REST config also accepts the OpenAI SDK client options (`baseURL`, `defaultHeaders`, `fetch`, `timeout`, `maxRetries`). The other adapters call the native `/ai/run` endpoint and take `fetch` only. +## Reuse the prompt cache + +Every chat turn sends the whole conversation again. Workers AI can serve the repeated start of the prompt from its cache, but only when the turns of one conversation reach the same model instance. Send an `x-session-affinity` header with one value per conversation to keep them together. See [Workers AI prompt caching](https://developers.cloudflare.com/workers-ai/features/prompt-caching/). + +Set the header with `defaultHeaders`: + +```typescript +import { chat, toServerSentEventsResponse } from "@tanstack/ai"; +import { createCloudflareText } from "@tanstack/ai-cloudflare"; +import type { Ai } from "@cloudflare/workers-types"; + +interface Env { + AI: Ai; +} + +export default { + async fetch(request: Request, env: Env) { + const { messages, conversationId } = await request.json(); + + const stream = chat({ + adapter: createCloudflareText("@cf/zai-org/glm-5.3-flash", { + binding: env.AI, + defaultHeaders: { "x-session-affinity": `ses_${conversationId}` }, + }), + messages, + }); + + return toServerSentEventsResponse(stream); + }, +}; +``` + +The binding sends `defaultHeaders` as the `extraHeaders` run option, next to `gateway`. The REST config takes the same `defaultHeaders` field. + ## Route through AI Gateway AI Gateway gives you caching, logs, rate limits, and one bill across providers. Add `gateway` to any config to send requests through it. Use `"default"` for the gateway Cloudflare creates for your account. diff --git a/docs/config.json b/docs/config.json index a5361a2351..cb870c1451 100644 --- a/docs/config.json +++ b/docs/config.json @@ -1251,7 +1251,7 @@ "label": "Cloudflare", "to": "adapters/cloudflare", "addedAt": "2026-09-03", - "updatedAt": "2026-09-18" + "updatedAt": "2026-09-29" }, { "label": "Claude Code", diff --git a/packages/ai-cloudflare/src/adapters/text.ts b/packages/ai-cloudflare/src/adapters/text.ts index 4751f06ac3..e3607a6d5c 100644 --- a/packages/ai-cloudflare/src/adapters/text.ts +++ b/packages/ai-cloudflare/src/adapters/text.ts @@ -45,7 +45,7 @@ function createClient(config: CloudflareTextConfig): OpenAI { return new OpenAI({ // The binding authenticates by itself; the SDK only requires a value. apiKey: 'cloudflare-binding', - fetch: createBindingFetch(config.binding, config.gateway), + fetch: createBindingFetch(config), }) } const { diff --git a/packages/ai-cloudflare/src/utils/config.ts b/packages/ai-cloudflare/src/utils/config.ts index 48fffb36fe..b7efc96be7 100644 --- a/packages/ai-cloudflare/src/utils/config.ts +++ b/packages/ai-cloudflare/src/utils/config.ts @@ -20,6 +20,11 @@ export type FetchLike = NonNullable export interface CloudflareBindingConfig { binding: Ai gateway?: CloudflareGatewayOptions + /** + * Workers AI request headers, such as `x-session-affinity` for prompt + * caching. Sent on every run as the binding's `extraHeaders` option. + */ + defaultHeaders?: Record accountId?: never apiKey?: never } diff --git a/packages/ai-cloudflare/src/utils/fetch.ts b/packages/ai-cloudflare/src/utils/fetch.ts index b1b3e95678..442dae502d 100644 --- a/packages/ai-cloudflare/src/utils/fetch.ts +++ b/packages/ai-cloudflare/src/utils/fetch.ts @@ -1,5 +1,4 @@ -import type { Ai } from '@cloudflare/workers-types' -import type { CloudflareGatewayOptions, FetchLike } from './config' +import type { CloudflareBindingConfig, FetchLike } from './config' /** * Workers AI streams end with a usage-only trailer (`{"response":"","usage": @@ -99,11 +98,15 @@ export async function normalizeResponse(response: Response): Promise { * Makes `env.AI` look like an OpenAI-compatible HTTP endpoint to the OpenAI * SDK: the JSON request body becomes `binding.run(model, inputs)` and the * raw inference `Response` (OpenAI-format JSON or SSE) is handed back. + * + * The SDK's own request headers (auth, user agent, `x-stainless-*`) are + * dropped; only the configured `defaultHeaders` reach Workers AI. */ -export function createBindingFetch( - binding: Ai, - gateway?: CloudflareGatewayOptions, -): FetchLike { +export function createBindingFetch({ + binding, + gateway, + defaultHeaders, +}: CloudflareBindingConfig): FetchLike { // `Ai` is typed against the bundled model catalog; the adapter accepts any // model id, so widen the binding to the open catalog shape for this call. const run = binding.run.bind(binding) as ( @@ -118,6 +121,7 @@ export function createBindingFetch( const response = (await run(model, inputs, { returnRawResponse: true, ...(gateway && { gateway }), + ...(defaultHeaders && { extraHeaders: defaultHeaders }), })) as Response return await normalizeResponse(response) } diff --git a/packages/ai-cloudflare/src/utils/run.ts b/packages/ai-cloudflare/src/utils/run.ts index c0b40c0d1f..efa9102ae6 100644 --- a/packages/ai-cloudflare/src/utils/run.ts +++ b/packages/ai-cloudflare/src/utils/run.ts @@ -41,10 +41,16 @@ export async function runModel( }, } : inputs + const { gateway, defaultHeaders } = config return await run( model, bindingInputs, - config.gateway ? { gateway: config.gateway } : undefined, + gateway || defaultHeaders + ? { + ...(gateway && { gateway }), + ...(defaultHeaders && { extraHeaders: defaultHeaders }), + } + : undefined, ) } diff --git a/packages/ai-cloudflare/tests/cloudflare-adapter.test.ts b/packages/ai-cloudflare/tests/cloudflare-adapter.test.ts index 37336ff53b..1f2b0e3cdf 100644 --- a/packages/ai-cloudflare/tests/cloudflare-adapter.test.ts +++ b/packages/ai-cloudflare/tests/cloudflare-adapter.test.ts @@ -155,6 +155,31 @@ describe('text adapter (binding)', () => { expect(finished.usage).toMatchObject({ promptTokens: 41, totalTokens: 45 }) }) + it('sends defaultHeaders as extraHeaders next to the gateway', async () => { + const { binding, run } = fakeBinding(() => sse(chatStreamEvents)) + const adapter = createCloudflareText('@cf/test', { + binding, + gateway: { id: 'g1' }, + defaultHeaders: { 'x-session-affinity': 'ses_abc' }, + }) + + await collect( + adapter.chatStream({ + model: '@cf/test', + messages: [{ role: 'user', content: 'Hi' }], + logger, + }), + ) + + // Only the configured headers: none of the OpenAI SDK's own headers + // (authorization, user-agent, x-stainless-*) reach the binding. + expect(run.mock.calls[0]![2]).toEqual({ + returnRawResponse: true, + gateway: { id: 'g1' }, + extraHeaders: { 'x-session-affinity': 'ses_abc' }, + }) + }) + it('surfaces reasoning_content as reasoning events', async () => { const { binding } = fakeBinding(() => sse([ @@ -342,6 +367,22 @@ describe('embedding adapter', () => { ]) }) + it('sends defaultHeaders as extraHeaders', async () => { + const { binding, run } = fakeBinding(() => ({ data: [[0.1]] })) + const adapter = createCloudflareEmbedding('@cf/baai/bge-m3', { + binding, + defaultHeaders: { 'x-session-affinity': 'ses_abc' }, + }) + await adapter.createEmbeddings({ + model: '@cf/baai/bge-m3', + input: ['a'], + logger, + }) + expect(run.mock.calls[0]![2]).toEqual({ + extraHeaders: { 'x-session-affinity': 'ses_abc' }, + }) + }) + it('rejects dimensions and a mismatched output shape', async () => { const { binding } = fakeBinding(() => ({ response: 'wrong shape' })) const adapter = createCloudflareEmbedding('@cf/baai/bge-m3', { binding }) diff --git a/testing/e2e/fixtures/chat/cloudflare-binding-headers-wire.json b/testing/e2e/fixtures/chat/cloudflare-binding-headers-wire.json new file mode 100644 index 0000000000..89fd1fe34e --- /dev/null +++ b/testing/e2e/fixtures/chat/cloudflare-binding-headers-wire.json @@ -0,0 +1,12 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "[cloudflare-binding-headers-wire] affinity check" + }, + "response": { + "content": "Affinity checked." + } + } + ] +} diff --git a/testing/e2e/src/routeTree.gen.ts b/testing/e2e/src/routeTree.gen.ts index 013e1601a1..e5efdde0b8 100644 --- a/testing/e2e/src/routeTree.gen.ts +++ b/testing/e2e/src/routeTree.gen.ts @@ -114,6 +114,7 @@ import { Route as ApiDurableTakeoverRouteImport } from './routes/api.durable-tak import { Route as ApiDurableDeliveryRouteImport } from './routes/api.durable-delivery' import { Route as ApiDevtoolsMemoryRouteImport } from './routes/api.devtools-memory' import { Route as ApiCompactionWireRouteImport } from './routes/api.compaction-wire' +import { Route as ApiCloudflareBindingHeadersWireRouteImport } from './routes/api.cloudflare-binding-headers-wire' import { Route as ApiChatRouteImport } from './routes/api.chat' import { Route as ApiByteplusSeedance1080pWireRouteImport } from './routes/api.byteplus-seedance-1080p-wire' import { Route as ApiByokChatRouteImport } from './routes/api.byok-chat' @@ -679,6 +680,12 @@ const ApiCompactionWireRoute = ApiCompactionWireRouteImport.update({ path: '/api/compaction-wire', getParentRoute: () => rootRouteImport, } as any) +const ApiCloudflareBindingHeadersWireRoute = + ApiCloudflareBindingHeadersWireRouteImport.update({ + id: '/api/cloudflare-binding-headers-wire', + path: '/api/cloudflare-binding-headers-wire', + getParentRoute: () => rootRouteImport, + } as any) const ApiChatRoute = ApiChatRouteImport.update({ id: '/api/chat', path: '/api/chat', @@ -820,6 +827,7 @@ export interface FileRoutesByFullPath { '/api/byok-chat': typeof ApiByokChatRoute '/api/byteplus-seedance-1080p-wire': typeof ApiByteplusSeedance1080pWireRoute '/api/chat': typeof ApiChatRoute + '/api/cloudflare-binding-headers-wire': typeof ApiCloudflareBindingHeadersWireRoute '/api/compaction-wire': typeof ApiCompactionWireRoute '/api/devtools-memory': typeof ApiDevtoolsMemoryRoute '/api/durable-delivery': typeof ApiDurableDeliveryRoute @@ -945,6 +953,7 @@ export interface FileRoutesByTo { '/api/byok-chat': typeof ApiByokChatRoute '/api/byteplus-seedance-1080p-wire': typeof ApiByteplusSeedance1080pWireRoute '/api/chat': typeof ApiChatRoute + '/api/cloudflare-binding-headers-wire': typeof ApiCloudflareBindingHeadersWireRoute '/api/compaction-wire': typeof ApiCompactionWireRoute '/api/devtools-memory': typeof ApiDevtoolsMemoryRoute '/api/durable-delivery': typeof ApiDurableDeliveryRoute @@ -1071,6 +1080,7 @@ export interface FileRoutesById { '/api/byok-chat': typeof ApiByokChatRoute '/api/byteplus-seedance-1080p-wire': typeof ApiByteplusSeedance1080pWireRoute '/api/chat': typeof ApiChatRoute + '/api/cloudflare-binding-headers-wire': typeof ApiCloudflareBindingHeadersWireRoute '/api/compaction-wire': typeof ApiCompactionWireRoute '/api/devtools-memory': typeof ApiDevtoolsMemoryRoute '/api/durable-delivery': typeof ApiDurableDeliveryRoute @@ -1198,6 +1208,7 @@ export interface FileRouteTypes { | '/api/byok-chat' | '/api/byteplus-seedance-1080p-wire' | '/api/chat' + | '/api/cloudflare-binding-headers-wire' | '/api/compaction-wire' | '/api/devtools-memory' | '/api/durable-delivery' @@ -1323,6 +1334,7 @@ export interface FileRouteTypes { | '/api/byok-chat' | '/api/byteplus-seedance-1080p-wire' | '/api/chat' + | '/api/cloudflare-binding-headers-wire' | '/api/compaction-wire' | '/api/devtools-memory' | '/api/durable-delivery' @@ -1448,6 +1460,7 @@ export interface FileRouteTypes { | '/api/byok-chat' | '/api/byteplus-seedance-1080p-wire' | '/api/chat' + | '/api/cloudflare-binding-headers-wire' | '/api/compaction-wire' | '/api/devtools-memory' | '/api/durable-delivery' @@ -1574,6 +1587,7 @@ export interface RootRouteChildren { ApiByokChatRoute: typeof ApiByokChatRoute ApiByteplusSeedance1080pWireRoute: typeof ApiByteplusSeedance1080pWireRoute ApiChatRoute: typeof ApiChatRoute + ApiCloudflareBindingHeadersWireRoute: typeof ApiCloudflareBindingHeadersWireRoute ApiCompactionWireRoute: typeof ApiCompactionWireRoute ApiDevtoolsMemoryRoute: typeof ApiDevtoolsMemoryRoute ApiDurableDeliveryRoute: typeof ApiDurableDeliveryRoute @@ -2387,6 +2401,13 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof ApiCompactionWireRouteImport parentRoute: typeof rootRouteImport } + '/api/cloudflare-binding-headers-wire': { + id: '/api/cloudflare-binding-headers-wire' + path: '/api/cloudflare-binding-headers-wire' + fullPath: '/api/cloudflare-binding-headers-wire' + preLoaderRoute: typeof ApiCloudflareBindingHeadersWireRouteImport + parentRoute: typeof rootRouteImport + } '/api/chat': { id: '/api/chat' path: '/api/chat' @@ -2620,6 +2641,7 @@ const rootRouteChildren: RootRouteChildren = { ApiByokChatRoute: ApiByokChatRoute, ApiByteplusSeedance1080pWireRoute: ApiByteplusSeedance1080pWireRoute, ApiChatRoute: ApiChatRoute, + ApiCloudflareBindingHeadersWireRoute: ApiCloudflareBindingHeadersWireRoute, ApiCompactionWireRoute: ApiCompactionWireRoute, ApiDevtoolsMemoryRoute: ApiDevtoolsMemoryRoute, ApiDurableDeliveryRoute: ApiDurableDeliveryRoute, diff --git a/testing/e2e/src/routes/api.cloudflare-binding-headers-wire.ts b/testing/e2e/src/routes/api.cloudflare-binding-headers-wire.ts new file mode 100644 index 0000000000..c467a80bb5 --- /dev/null +++ b/testing/e2e/src/routes/api.cloudflare-binding-headers-wire.ts @@ -0,0 +1,66 @@ +import { createFileRoute } from '@tanstack/react-router' +import { chat, createChatOptions } from '@tanstack/ai' +import { createCloudflareText } from '@tanstack/ai-cloudflare' +import type { CloudflareBindingConfig } from '@tanstack/ai-cloudflare' + +const LLMOCK_DEFAULT_BASE = process.env.LLMOCK_URL || 'http://127.0.0.1:4010' + +/** + * Drives the Cloudflare text adapter in binding mode. The stand-in `env.AI` + * does what the real binding does with `returnRawResponse` and + * `extraHeaders`: it posts the inputs with those headers and returns the raw + * response. The companion spec reads aimock's request journal to check that + * `defaultHeaders` reach Workers AI and the OpenAI SDK's own headers do not. + */ +export const Route = createFileRoute('/api/cloudflare-binding-headers-wire')({ + server: { + handlers: { + POST: async ({ request }) => { + const testId = new URL(request.url).searchParams.get('testId') ?? '' + + const run = async ( + model: string, + inputs: Record, + options: { extraHeaders?: Record }, + ) => + await fetch(`${LLMOCK_DEFAULT_BASE}/v1/chat/completions`, { + method: 'POST', + headers: { + 'content-type': 'application/json', + 'x-test-id': testId, + ...options.extraHeaders, + }, + body: JSON.stringify({ model, ...inputs }), + }) + // ponytail: only `run` is exercised; the real binding type has more. + const binding = { run } as unknown as CloudflareBindingConfig['binding'] + + const adapter = createCloudflareText('@cf/zai-org/glm-5.3-flash', { + binding, + defaultHeaders: { 'x-session-affinity': 'ses_e2e' }, + }) + + try { + for await (const _ of chat({ + ...createChatOptions({ adapter }), + messages: [ + { + role: 'user', + content: '[cloudflare-binding-headers-wire] affinity check', + }, + ], + })) { + // Drain the stream. + } + } catch (error) { + return Response.json({ + ok: false, + error: error instanceof Error ? error.message : String(error), + }) + } + + return Response.json({ ok: true }) + }, + }, + }, +}) diff --git a/testing/e2e/tests/cloudflare-binding-headers-wire.spec.ts b/testing/e2e/tests/cloudflare-binding-headers-wire.spec.ts new file mode 100644 index 0000000000..7d26773cde --- /dev/null +++ b/testing/e2e/tests/cloudflare-binding-headers-wire.spec.ts @@ -0,0 +1,35 @@ +import { test, expect } from './fixtures' + +type JournalEntry = { + headers?: Record +} + +test.describe('cloudflare — binding defaultHeaders on the wire', () => { + test('sends x-session-affinity and drops OpenAI SDK headers', async ({ + request, + aimockPort, + testId, + }) => { + const response = await request.post( + `/api/cloudflare-binding-headers-wire?testId=${encodeURIComponent(testId)}`, + ) + expect(response.ok()).toBe(true) + expect(await response.json()).toEqual({ ok: true }) + + const journalResponse = await request.get( + `http://127.0.0.1:${aimockPort}/v1/_requests`, + ) + const entries = (await journalResponse.json()) as Array + const captured = entries.find( + (entry) => entry.headers?.['x-test-id'] === testId, + ) + expect(captured).toBeDefined() + expect(captured?.headers?.['x-session-affinity']).toBe('ses_e2e') + expect(captured?.headers).not.toHaveProperty('authorization') + expect( + Object.keys(captured?.headers ?? {}).filter((name) => + name.startsWith('x-stainless-'), + ), + ).toEqual([]) + }) +}) From 681340c06b0f727ed228cf1017c01122882182fa Mon Sep 17 00:00:00 2001 From: Alem Tuzlak Date: Thu, 22 Oct 2026 12:26:17 +0200 Subject: [PATCH 2/2] fix(ai-cloudflare): pass threadId to chat(), now required on main --- docs/adapters/cloudflare.md | 1 + testing/e2e/src/routes/api.cloudflare-binding-headers-wire.ts | 1 + 2 files changed, 2 insertions(+) diff --git a/docs/adapters/cloudflare.md b/docs/adapters/cloudflare.md index 6cfd2497b7..abd6117f6a 100644 --- a/docs/adapters/cloudflare.md +++ b/docs/adapters/cloudflare.md @@ -191,6 +191,7 @@ export default { defaultHeaders: { "x-session-affinity": `ses_${conversationId}` }, }), messages, + threadId: conversationId, }); return toServerSentEventsResponse(stream); diff --git a/testing/e2e/src/routes/api.cloudflare-binding-headers-wire.ts b/testing/e2e/src/routes/api.cloudflare-binding-headers-wire.ts index c467a80bb5..cad4ef6ff9 100644 --- a/testing/e2e/src/routes/api.cloudflare-binding-headers-wire.ts +++ b/testing/e2e/src/routes/api.cloudflare-binding-headers-wire.ts @@ -43,6 +43,7 @@ export const Route = createFileRoute('/api/cloudflare-binding-headers-wire')({ try { for await (const _ of chat({ ...createChatOptions({ adapter }), + threadId: 'cloudflare-binding-headers-wire', messages: [ { role: 'user',