Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 43 additions & 6 deletions packages/ai/src/route/transport/http.ts
Original file line number Diff line number Diff line change
@@ -1,11 +1,20 @@
import { Effect } from "effect"
import { Duration, Effect, Stream } from "effect"
import { Headers, HttpClientRequest } from "effect/unstable/http"
import { Auth } from "../auth.js"
import { render as renderEndpoint } from "../endpoint.js"
import { Framing } from "../framing.js"
import type { HttpMiddleware, Transport, TransportPrepareInput } from "./index.js"
import * as ProviderShared from "../../protocols/shared.js"
import { mergeJsonRecords, type LLMRequest } from "../../schema/index.js"
import {
AIError,
DEFAULT_HTTP_TIMEOUT_MS,
mergeJsonRecords,
TransportError,
type HttpContext,
type HttpTimeout,
type LLMRequest,
type TransportOperation,
} from "../../schema/index.js"
import { RequestExecutor } from "../executor.js"

export type JsonRequestInput<Body> = TransportPrepareInput<Body>
Expand Down Expand Up @@ -87,17 +96,45 @@ export const httpJson = <Body, Frame>(input: HttpJsonInput<Body, Frame>): HttpJs
middleware: prepareInput.middleware,
}
}),
execute: (prepared, _request, runtime) =>
execute: (prepared, request, runtime) =>
Effect.gen(function* () {
const response = yield* runtime.http.execute(prepared.request, prepared.middleware)
const timeout = (operation: TransportOperation, message: string, http?: HttpContext) =>
new AIError({
reason: new TransportError({
message,
transport: "http",
operation,
code: "Timeout",
url: prepared.request.url,
http,
}),
})
const response = yield* runtime.http.execute(prepared.request, prepared.middleware).pipe(
Effect.timeoutOrElse({
duration: timeoutDuration(request.http?.headerTimeout),
orElse: () => Effect.fail(timeout("request", "Timed out waiting for response headers")),
}),
)
const http = RequestExecutor.responseHttp(response)
return {
frames: prepared.framing.frame(RequestExecutor.responseStream(response)),
http: RequestExecutor.responseHttp(response),
frames: prepared.framing.frame(
RequestExecutor.responseStream(response).pipe(
Stream.timeoutOrElse({
duration: timeoutDuration(request.http?.chunkTimeout),
orElse: () => Stream.fail(timeout("read", "Timed out waiting for response data", http)),
}),
),
),
http,
body: prepared.framing.body,
}
}),
})

// `false` disables a timer; an unset value uses the shared default.
const timeoutDuration = (value: HttpTimeout | undefined) =>
value === false ? Duration.infinity : Duration.millis(value ?? DEFAULT_HTTP_TIMEOUT_MS)

export const sseJson = {
id: "http-json/sse",
with: <Body>() => httpJson<Body, string>({ framing: Framing.sse }),
Expand Down
17 changes: 15 additions & 2 deletions packages/ai/src/schema/options.ts
Original file line number Diff line number Diff line change
Expand Up @@ -43,10 +43,21 @@ export const mergeProviderOptions = (
...items: ReadonlyArray<ProviderOptions | undefined>
): ProviderOptions | undefined => mergeJsonRecords(...items)

/** Milliseconds for an HTTP timeout, or `false` to disable it. */
export const HttpTimeout = Schema.Union([Schema.Number.check(Schema.isGreaterThan(0)), Schema.Literal(false)])
export type HttpTimeout = Schema.Schema.Type<typeof HttpTimeout>

/** Default for `headerTimeout` and `chunkTimeout` when a request leaves them unset. */
export const DEFAULT_HTTP_TIMEOUT_MS = 300_000

export class HttpOptions extends Schema.Class<HttpOptions>("AI.HttpOptions")({
body: Schema.optional(JsonSchema),
headers: Schema.optional(Schema.Record(Schema.String, Schema.String)),
query: Schema.optional(Schema.Record(Schema.String, Schema.String)),
/** Time allowed for response headers to arrive. Defaults to five minutes. */
headerTimeout: Schema.optional(HttpTimeout),
/** Time allowed between streamed response chunks once headers have arrived. Defaults to five minutes. */
chunkTimeout: Schema.optional(HttpTimeout),
}) {}

export namespace HttpOptions {
Expand All @@ -60,8 +71,10 @@ export const mergeHttpOptions = (...items: ReadonlyArray<HttpOptions | undefined
const body = mergeJsonRecords(...items.map((item) => item?.body))
const headers = mergeStringRecords(...items.map((item) => item?.headers))
const query = mergeStringRecords(...items.map((item) => item?.query))
if (!body && !headers && !query) return undefined
return new HttpOptions({ body, headers, query })
const headerTimeout = items.findLast((item) => item?.headerTimeout !== undefined)?.headerTimeout
const chunkTimeout = items.findLast((item) => item?.chunkTimeout !== undefined)?.chunkTimeout
if (!body && !headers && !query && headerTimeout === undefined && chunkTimeout === undefined) return undefined
return new HttpOptions({ body, headers, query, headerTimeout, chunkTimeout })
}

export class GenerationOptions extends Schema.Class<GenerationOptions>("LLM.GenerationOptions")({
Expand Down
144 changes: 144 additions & 0 deletions packages/ai/test/http-timeout.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,144 @@
import { describe, expect } from "bun:test"
import { Deferred, Effect, Fiber } from "effect"
import * as TestClock from "effect/testing/TestClock"
import { HttpOptions, LLM, mergeHttpOptions } from "../src/index.js"
import { LLMClient } from "../src/route.js"
import { configure } from "../src/providers/openai.js"
import { dynamicResponse } from "./lib/http.js"
import { deltaChunk, finishChunk } from "./lib/openai-chunks.js"
import { sseEvents } from "./lib/sse.js"
import { it } from "./lib/effect.js"

const model = configure({ baseURL: "https://api.openai.test/v1/", apiKey: "test" }).chat("gpt-4.1-mini")
const body = sseEvents(deltaChunk({ role: "assistant", content: "Hi" }), finishChunk("stop"))
const SSE = { headers: { "content-type": "text/event-stream" } }

// Never produces headers; the header timer is the only way out.
const silentServer = dynamicResponse(() => Effect.never)

// Sends one chunk, then stalls until `resume` releases the rest of the body.
const stalledServer = Effect.gen(function* () {
const stalled = yield* Deferred.make<void>()
let resume = () => {}
const released = new Promise<void>((resolve) => {
resume = resolve
})
const encoder = new TextEncoder()
const layer = dynamicResponse((input) =>
Effect.sync(() =>
input.respond(
new ReadableStream<Uint8Array>({
start(controller) {
controller.enqueue(encoder.encode(`data: ${JSON.stringify(deltaChunk({ content: "Hi" }))}\n\n`))
},
async pull(controller) {
Deferred.doneUnsafe(stalled, Effect.void)
await released
controller.enqueue(encoder.encode(`data: ${JSON.stringify(finishChunk("stop"))}\n\ndata: [DONE]\n\n`))
controller.close()
},
}),
SSE,
),
),
)
return { layer, stalled, resume: () => resume() }
})

const slowHeadersServer = dynamicResponse((input) =>
Effect.sleep("10 minutes").pipe(Effect.as(input.respond(body, SSE))),
)

describe("HTTP transport timeouts", () => {
it.effect("fails when response headers take longer than five minutes", () =>
Effect.gen(function* () {
const fiber = yield* LLMClient.generate(LLM.request({ model, prompt: "Hello" })).pipe(
Effect.provide(silentServer),
Effect.flip,
Effect.forkChild({ startImmediately: true }),
)
yield* TestClock.adjust("5 minutes")
const error = yield* Fiber.join(fiber)

expect(error.reason).toMatchObject({
_tag: "Transport",
transport: "http",
operation: "request",
code: "Timeout",
})
expect(error.reason.message).toContain("response headers")
}),
)

it.effect("fails when the response body stalls for five minutes", () =>
Effect.gen(function* () {
const server = yield* stalledServer
const fiber = yield* LLMClient.generate(LLM.request({ model, prompt: "Hello" })).pipe(
Effect.provide(server.layer),
Effect.flip,
Effect.forkChild({ startImmediately: true }),
)
yield* Deferred.await(server.stalled)
yield* Effect.yieldNow
yield* TestClock.adjust("5 minutes")
const error = yield* Fiber.join(fiber)

expect(error.reason).toMatchObject({ _tag: "Transport", transport: "http", operation: "read", code: "Timeout" })
expect(error.reason.http).toMatchObject({ status: 200 })
}),
)

it.effect("applies a configured header timeout", () =>
Effect.gen(function* () {
const fiber = yield* LLMClient.generate(
LLM.request({ model, prompt: "Hello", http: { headerTimeout: 1_000 } }),
).pipe(Effect.provide(silentServer), Effect.flip, Effect.forkChild({ startImmediately: true }))
yield* TestClock.adjust("1 second")
const error = yield* Fiber.join(fiber)

expect(error.reason).toMatchObject({ _tag: "Transport", operation: "request", code: "Timeout" })
}),
)

it.effect("disables the header timeout with false", () =>
Effect.gen(function* () {
const fiber = yield* LLMClient.generate(
LLM.request({ model, prompt: "Hello", http: { headerTimeout: false } }),
).pipe(Effect.provide(slowHeadersServer), Effect.forkChild({ startImmediately: true }))
yield* TestClock.adjust("10 minutes")
const response = yield* Fiber.join(fiber)

expect(response.text).toBe("Hi")
}),
)

it.effect("disables the chunk timeout with false", () =>
Effect.gen(function* () {
const server = yield* stalledServer
const fiber = yield* LLMClient.generate(
LLM.request({ model, prompt: "Hello", http: { chunkTimeout: false } }),
).pipe(Effect.provide(server.layer), Effect.forkChild({ startImmediately: true }))
yield* Deferred.await(server.stalled)
yield* Effect.yieldNow
yield* TestClock.adjust("10 minutes")
server.resume()
const response = yield* Fiber.join(fiber)

expect(response.text).toBe("Hi")
}),
)

it.effect("merges timeouts with later values winning", () =>
Effect.sync(() => {
const merged = mergeHttpOptions(
new HttpOptions({ headerTimeout: 1_000, chunkTimeout: 2_000 }),
new HttpOptions({ headers: { a: "b" } }),
new HttpOptions({ chunkTimeout: false }),
)

expect(merged).toMatchObject({ headers: { a: "b" }, headerTimeout: 1_000, chunkTimeout: false })
expect(mergeHttpOptions(new HttpOptions({}), undefined)).toBeUndefined()
expect(mergeHttpOptions(new HttpOptions({ headerTimeout: false }))?.headerTimeout).toBe(false)
}),
)
})
45 changes: 28 additions & 17 deletions packages/core/src/aisdk.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import type {
SharedV3ProviderOptions,
} from "@ai-sdk/provider"
import {
DEFAULT_HTTP_TIMEOUT_MS,
FinishReason,
LLMEvent,
AIError,
Expand Down Expand Up @@ -126,21 +127,28 @@ function prepareOptions(model: Info, pkg: string) {
}

const customFetch = options.fetch
const chunkTimeout = options.chunkTimeout
const timeouts = Provider.timeouts(options)
const chunkTimeout = timeouts.chunkTimeout ?? DEFAULT_HTTP_TIMEOUT_MS
const headerTimeout = timeouts.headerTimeout ?? DEFAULT_HTTP_TIMEOUT_MS
delete options.chunkTimeout
delete options.headerTimeout
options.fetch = async (input: Parameters<typeof fetch>[0], init?: RequestInit) => {
const opts = { ...(init ?? {}) }
const signals = [
opts.signal,
typeof chunkTimeout === "number" && chunkTimeout > 0 ? new AbortController() : undefined,
options.timeout !== undefined && options.timeout !== null && options.timeout !== false
? AbortSignal.timeout(options.timeout)
: undefined,
].filter((item): item is AbortSignal | AbortController => item !== undefined && item !== null)
const chunkAbortCtl = signals.find((item): item is AbortController => item instanceof AbortController)
const abortSignals = signals.map((item) => (item instanceof AbortController ? item.signal : item))
if (abortSignals.length === 1) opts.signal = abortSignals[0]
if (abortSignals.length > 1) opts.signal = AbortSignal.any(abortSignals)
const ctl = new AbortController()
// Only covers the wait for response headers; wrapSSE takes over once the body streams.
const headerTimer =
headerTimeout === false
? undefined
: setTimeout(() => ctl.abort(new Error(HEADER_TIMEOUT_MESSAGE)), headerTimeout)
opts.signal = AbortSignal.any(
[
opts.signal,
ctl.signal,
options.timeout !== undefined && options.timeout !== null && options.timeout !== false
? AbortSignal.timeout(options.timeout)
: undefined,
].filter((item): item is AbortSignal => item !== undefined && item !== null),
)

if (typeof opts.body === "string" && model.body !== undefined) {
const decoded = Option.getOrUndefined(decodeJson(opts.body))
Expand All @@ -152,14 +160,16 @@ function prepareOptions(model: Info, pkg: string) {
const res = await (typeof customFetch === "function" ? customFetch : fetch)(input, {
...opts,
timeout: false,
})
if (!chunkAbortCtl || typeof chunkTimeout !== "number") return res
return wrapSSE(res, chunkTimeout, chunkAbortCtl)
}).finally(() => clearTimeout(headerTimer))
if (chunkTimeout === false) return res
return wrapSSE(res, chunkTimeout, ctl)
}

return options
}

const HEADER_TIMEOUT_MESSAGE = "Response headers timed out"

export class InitError extends Schema.TaggedError<InitError>()("AISDK.InitError", {
providerID: Provider.ID,
cause: Schema.Defect(),
Expand Down Expand Up @@ -388,7 +398,7 @@ function requestSettings(settings: Readonly<Record<string, unknown>> | undefined
if (settings === undefined) return undefined
const result = Object.fromEntries(
Object.entries(settings).filter(
([key]) => !["apiKey", "authToken", "baseURL", "chunkTimeout", "fetch", "timeout"].includes(key),
([key]) => !["apiKey", "authToken", "baseURL", "chunkTimeout", "fetch", "headerTimeout", "timeout"].includes(key),
),
)
return Object.keys(result).length === 0 ? undefined : result
Expand Down Expand Up @@ -838,7 +848,7 @@ function llmError(error: unknown, operation: "request" | "read") {

// Runtime-generated network failure shapes. The codes mirror the AI SDK's own
// Bun network error list in handleFetchError; the messages are undici's fetch
// TypeError and stream termination strings plus our SSE chunk timeout error.
// TypeError and stream termination strings plus our header and chunk timeout errors.
// Unrecognized shapes still retry via the UnknownProvider default; this match
// only adds transport semantics (continuation eligibility, display).
const NETWORK_ERROR_CODES = new Set([
Expand All @@ -856,6 +866,7 @@ const NETWORK_ERROR_MESSAGES = new Set([
"terminated",
"other side closed",
"sse read timed out",
HEADER_TIMEOUT_MESSAGE.toLowerCase(),
])

const NativeErrorShape = Schema.Struct({
Expand Down
8 changes: 6 additions & 2 deletions packages/core/src/model-resolver.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
export * as ModelResolver from "./model-resolver.js"

import { makeLocationNode } from "@opencode-ai/util/effect/app-node"
import { LanguageModel } from "@opencode-ai/ai"
import { HttpOptions, LanguageModel, mergeHttpOptions } from "@opencode-ai/ai"
import { Auth } from "@opencode-ai/ai/route"
import { Context, Effect, Layer, Schema, Struct } from "effect"
import { AISDK } from "./aisdk.js"
Expand Down Expand Up @@ -128,7 +128,10 @@ const resolveCatalogModel = Effect.fn("ModelResolver.resolveCatalogModel")(funct
const resolved = prepareRuntimeModel(model, credential)
const packageName = Provider.packageName(resolved.package)
const configuration = credential?.type === "key" ? credential.configuration : undefined
const configured = { ...resolved.settings, ...credential?.metadata, ...configuration }
const merged = { ...resolved.settings, ...credential?.metadata, ...configuration }
// Timeouts are transport policy: they become route HTTP defaults rather than provider package settings.
const timeouts = Provider.timeouts(merged)
const configured = Struct.omit(merged, ["headerTimeout", "chunkTimeout"])
const mapping = Provider.isAISDK(resolved.package)
? AISDKNative.map({
packageName,
Expand Down Expand Up @@ -173,6 +176,7 @@ const resolveCatalogModel = Effect.fn("ModelResolver.resolveCatalogModel")(funct
compatibility: resolved.compatibility
? Object.assign({}, runtime.compatibility, resolved.compatibility)
: runtime.compatibility,
defaults: { ...runtime.defaults, http: mergeHttpOptions(runtime.defaults?.http, new HttpOptions(timeouts)) },
})
},
catch: () => unsupported(resolved),
Expand Down
Loading
Loading