diff --git a/docs/providers.md b/docs/providers.md index 143e564db..46aa78bad 100644 --- a/docs/providers.md +++ b/docs/providers.md @@ -1,4 +1,4 @@ - + # Provider @@ -130,6 +130,12 @@ router 自动加载。 `incomplete_details.reason`,被截断的工具调用由 Core 标成未执行。输出字符数超过 `max_output_tokens × 12` 时终止请求并报告输出字符数超限。 +带图的请求被上游以 413 拒收，或以 400 拒收且模块的 `contextOverflow` 不认时，去掉这次请求的图片重发一次。重发被接受: +这些图之后不再发给这个端点(按句柄记在内存里，实例重建或进程重启后清空),每条失去图片的记录在正文后补一行说明。重发也被拒：错误带上两次尝试交给 Core, +去掉图片后仍是 413 的按输入超限交接(见 [sessions.md](sessions.md))。每个端点都有这层回落(`ImageFallback`):内建模块的 client 自己套， +扩展 provider 由 `ProviderRegistry.bind` 套。扩展的 client 自己套了的，在 `create()` 返回的实例上写 `handlesRefusedImages: true`, +`bind` 就不再套;只给一部分记录附图的 client 要自己套，用 `attached` 交回附了图的记录。 + ## 计价 每次请求尝试按价目中声明的计量项计费,包括输入、输出、缓存命中与推理用量等,币种默认 USD,写入 diff --git a/docs/sessions.md b/docs/sessions.md index f869e9984..7fe6d1cf9 100644 --- a/docs/sessions.md +++ b/docs/sessions.md @@ -36,7 +36,7 @@ Persona 声明,Core 只把 `role` 当分类标签。 ## 上下文与交接 Core 的输入上限为 `hardTokens = max(0, 生效窗口 − (spec.maxTokens ?? 0))`。生效窗口取服务探测值与配置的 -`contextWindow` 中的较小者；两者都缺失时不设置此上限。超过上限或服务拒绝超长输入时， +`contextWindow` 中的较小者；两者都缺失时不设置此上限。超过上限或服务拒绝超长输入时(413 请求体过大也算), Core 调用 `Persona.onHandoff(snapshot, { hardTokens })`。Persona 决定保留的上下文和交接笔记， Core 重建 system 前缀、按容量截断保留内容并重置 session。 上一批已按计数强制交接、其间没有一次计数检查回到上限内,计数又越过同一上限时(窗口不大于最大输出,或交接后的 diff --git a/src/providers/README.md b/src/providers/README.md index d272f0ef2..43fda7f9a 100644 --- a/src/providers/README.md +++ b/src/providers/README.md @@ -1,4 +1,4 @@ - + # src/providers @@ -36,7 +36,8 @@ effort 收任意非空串)、`serviceTiers`、`create(name, entry, host)`。可 声明仍参与服务端校验。启停与模型操作这类动作走面板 invoke。 `create()` 返回 `ProviderInstance`:`client`(实现 `respond`)、`listModels?`、`control?`、 -`compatibilityKey?`、`start?` / `stop?`、`contextWindow?(model)`。`listModels()` 返回 `ListedModel[]`: +`compatibilityKey?`、`start?` / `stop?`、`contextWindow?(model)`、`handlesRefusedImages?`(client 自己已经做了带图请求被拒时的回落, +见下文 `image-fallback.ts`)。`listModels()` 返回 `ListedModel[]`: `id` 必有,`displayName`、`contextWindow`、`maxOutputTokens`、`inputImages` 只在上游给出时才有。 `ProviderHost` 给实例:`stateDir`(`<部署根>/providers/<端点名>/`,归实例独占)、`repoRoot`、 @@ -54,7 +55,8 @@ effort 收任意非空串)、`serviceTiers`、`create(name, entry, host)`。可 `spec` 的条目 JSON 加端点 `.env` 的内容指纹为缓存键(改模型或价格不重建实例,密钥文件变更后 下一次解析重建),填 `stateDir` 与 `secret`。 `bind(name)` 在实例外包一层:注入 `quote`(报价快照)与 `origin`(实例、模块、模型、 -`compatibilityDomain` = sha256(kind + baseUrl + `compatibilityKey()`))。Responses 历史推理仅在实例、 +`compatibilityDomain` = sha256(kind + baseUrl + `compatibilityKey()`));实例没有声明 `handlesRefusedImages` 时, +client 外再套一层 `ImageFallback`,每个实例一份,随实例重建。Responses 历史推理仅在实例、 模块、兼容域与模型均匹配时回传。`invalidate()` 清除缓存,WebUI 保存端点后调用;`.env` 变更经缓存键在下一次解析生效。 ## 配置形状 @@ -91,6 +93,13 @@ function_call / function_call_output 出线时去掉可选的 `id`,`call_id` 不 `finish_reason` 的 `length` / `content_filter` 落成 `incomplete_details.reason`; `response-meters.ts` 把两种 usage 归一成 `TokenMeters`,缺项保持 null。 +`image-fallback.ts` 的 `ImageFallback` 是带图请求被拒时的回落:请求带图、上游回 413,或回 400 +且模块的 `contextOverflow` 不认，就去掉这次请求的图片重发一次。重发被接受，按句柄记下这些图(只在内存里),之后经这个 client +的请求都不带，每条失去图片的记录在正文后补一行 `imageLeftOut`;重发也被拒，错误带上两次尝试，什么都不记。 +`attached` 交回 client 实际附图的那些记录，不给就按全部记录算。`isContextOverflow` 把任何 413 算作超限，所以去掉图片后仍 413 时由 Core 交接。 +两个内建模块的 client 自己套着它，实例声明 `handlesRefusedImages`;其余实例由 `ProviderRegistry.bind` 套，图像开关取端点的 `multimodal`, +`contextOverflow` 取模块的，不给 `attached`。只给一部分记录附图的 client 自己套并声明 `handlesRefusedImages`。 + 线协议类型在 `src/protocol/open-responses/`(规范 2026-04-24 生成),`ResponseAccumulator` 逐事件强校验:序号递增、终态自洽、已关闭的 Item 不得再变。 diff --git a/src/providers/base.ts b/src/providers/base.ts index 2649d218a..34b362ac1 100644 --- a/src/providers/base.ts +++ b/src/providers/base.ts @@ -57,6 +57,11 @@ export interface ListedModel { export interface ProviderInstance { client: BaseProvider; + /** + * The client already sends a request refused for its images once more without them: it is an + * `ImageFallback`, or does the same itself. Absent: `ProviderRegistry.bind` puts one around the client. + */ + handlesRefusedImages?: boolean; /** Models the endpoint advertises (`GET /models`). */ listModels?(): Promise; /** Module-owned control object; interpreted by that module's WebUI contribution. */ diff --git a/src/providers/llamacpp/index.ts b/src/providers/llamacpp/index.ts index 4668b9186..f70df9d12 100644 --- a/src/providers/llamacpp/index.ts +++ b/src/providers/llamacpp/index.ts @@ -2,6 +2,7 @@ import type { ProviderModule } from '../base.ts'; import type { ReasoningTier } from '../../core/types.ts'; import type { Language } from '../../core/language.ts'; import { isContextOverflow } from '../transport/errors.ts'; +import { ImageFallback } from '../transport/image-fallback.ts'; import { modelsRoot, runtimesRoot } from '../../paths.ts'; import { RouterCatalog } from './catalog.ts'; import { runtimeConfig } from './config.ts'; @@ -76,6 +77,7 @@ export default { roots: { runtimes: runtimesRoot(), models: modelsRoot() }, }); const runtime = host.resource?.('runtime', createRuntime) ?? createRuntime(); + const media = { enabled: () => entry.multimodal === true, read: host.readBlob }; return { control: { runtime, catalog }, listModels: () => catalog.listModels(), @@ -86,13 +88,14 @@ export default { catalog.stop(); await runtime.stop(); }, - client: new LlamaCppProvider({ + client: new ImageFallback(new LlamaCppProvider({ baseUrl: entry.baseUrl, apiKey, log: host.log, - media: { enabled: () => entry.multimodal === true, read: host.readBlob }, + media, keepThinking: host.keepThinking, - }), + }), { media, contextOverflow: isContextOverflow, log: host.log }), + handlesRefusedImages: true, }; }, } satisfies ProviderModule; diff --git a/src/providers/openai-responses-compat/index.ts b/src/providers/openai-responses-compat/index.ts index 4533067f8..08e7361df 100644 --- a/src/providers/openai-responses-compat/index.ts +++ b/src/providers/openai-responses-compat/index.ts @@ -2,6 +2,7 @@ import type { ProviderModule } from '../base.ts'; import type { LLMProviderEntry } from '../../core/types.ts'; import type { ResponseClient } from '../../core/generation.ts'; import { isContextOverflow } from '../transport/errors.ts'; +import { ImageFallback } from '../transport/image-fallback.ts'; import { REASONING_REPLAYS, type ReasoningReplay } from '../transport/responses-input.ts'; import { ModelCatalog, ResponsesProvider, type ResponsesProviderOptions } from './native.ts'; import { text } from './strings.ts'; @@ -94,6 +95,7 @@ export default { ...(apiKey ? { Authorization: `Bearer ${apiKey}` } : {}), }); const catalog = new ModelCatalog(() => ({ baseUrl: entry.baseUrl, headers: headers() })); + const media = { enabled: () => entry.multimodal === true, read: host.readBlob }; const clientOptions = (reasoningReplay: ReasoningReplay | undefined): ResponsesProviderOptions => ({ baseUrl: entry.baseUrl, apiKey, @@ -101,7 +103,7 @@ export default { extraHeaders: options.extraHeaders, extraBody: options.extraBody, log: host.log, - media: { enabled: () => entry.multimodal === true, read: host.readBlob }, + media, keepThinking: host.keepThinking, reasoningReplay, syntheticReasoningText: options.syntheticReasoningText, @@ -112,7 +114,8 @@ export default { contextWindow: (model) => catalog.contextWindow(model), compatibilityKey: () => [options.endpointPath ?? '/responses', name], control: { probeClient: (replay) => new ResponsesProvider({ ...clientOptions(replay), sendSyntheticReasoning: true }) } satisfies CompatControl, - client: new ResponsesProvider(clientOptions(options.reasoningReplay)), + client: new ImageFallback(new ResponsesProvider(clientOptions(options.reasoningReplay)), { media, contextOverflow: isContextOverflow, log: host.log }), + handlesRefusedImages: true, }; }, } satisfies ProviderModule; diff --git a/src/providers/registry.ts b/src/providers/registry.ts index 1131d757f..0ae20b3d5 100644 --- a/src/providers/registry.ts +++ b/src/providers/registry.ts @@ -2,7 +2,8 @@ import { existsSync, readdirSync, readFileSync } from 'node:fs'; import { join } from 'node:path'; import { URL, fileURLToPath, pathToFileURL } from 'node:url'; import type { LLMProviderEntry } from '../core/types.ts'; -import type { ProviderHostBase, ProviderInstance, ProviderModule } from './base.ts'; +import type { BaseProvider, ProviderHostBase, ProviderInstance, ProviderModule } from './base.ts'; +import { ImageFallback } from './transport/image-fallback.ts'; import { quotePrices } from './pricebook.ts'; import { createHash } from 'node:crypto'; import { secretReader } from '../core/secrets.ts'; @@ -66,6 +67,8 @@ export function providerModule(kind: string): ProviderModule { export class ProviderRegistry { private readonly instances = new Map(); private readonly resources = new Map(); + /** One per instance: `bind` runs on every request, and the fallback holds the handles the service refused. */ + private readonly imageFallbacks = new WeakMap(); constructor( private readonly entries: () => Record, private readonly host: ProviderHostBase, @@ -130,6 +133,7 @@ export class ProviderRegistry { const module = this.module(raw.kind); const entry = module.normalize?.(structuredClone(raw)) ?? structuredClone(raw); const instance = this.resolve(name); + const target = this.imageFallback(instance, module, entry); const domain = () => createHash('sha256') .update(JSON.stringify([entry.kind, entry.baseUrl, instance.compatibilityKey?.() ?? null])) @@ -137,7 +141,7 @@ export class ProviderRegistry { const client: ResponseClient = { bind: () => client, respond: (request, options) => - instance.client.respond(request, { + target.respond(request, { ...options, quote: (at) => quotePrices(entry, request, at, module.prices?.(entry, request, at) ?? []), origin: { @@ -151,6 +155,21 @@ export class ProviderRegistry { return client; } + /** The instance's client behind an `ImageFallback`; the client itself when the instance declares `handlesRefusedImages`. */ + private imageFallback(instance: ProviderInstance, module: ProviderModule, entry: LLMProviderEntry): BaseProvider { + if (instance.handlesRefusedImages) return instance.client; + let fallback = this.imageFallbacks.get(instance); + if (!fallback) { + fallback = new ImageFallback(instance.client, { + media: { enabled: () => entry.multimodal === true, read: this.host.readBlob }, + contextOverflow: (error) => module.contextOverflow?.(error) ?? false, + log: this.host.log, + }); + this.imageFallbacks.set(instance, fallback); + } + return fallback; + } + /** Drop the cached instance so the next resolve rebuilds it; `host.resource` objects survive. */ invalidate(name: string): void { this.instances.delete(name); diff --git a/src/providers/transport/errors.ts b/src/providers/transport/errors.ts index d7068a69d..b3a6b6a79 100644 --- a/src/providers/transport/errors.ts +++ b/src/providers/transport/errors.ts @@ -29,13 +29,14 @@ export function upstreamText(body: string): string { } /** - * Whether an upstream rejection says the input exceeded the model's context. Status 0 is an - * in-stream failure event; 400/413 are the request-rejection codes. The body patterns cover - * the OpenAI `context_length_exceeded` code, DeepSeek's "maximum context length" message - * and llama-server's "exceeds the available context size". + * Whether an upstream rejection says the input is too large for the model. A 413 always is: the + * request body exceeds the service's size limit, whatever the body says. Status 0 (an in-stream + * failure event) and 400 count when the body matches: the OpenAI `context_length_exceeded` code, + * DeepSeek's "maximum context length" message and llama-server's "exceeds the available context size". */ export function isContextOverflow(error: { status: number; body: string }): boolean { - if (error.status !== 0 && error.status !== 400 && error.status !== 413) return false; + if (error.status === 413) return true; + if (error.status !== 0 && error.status !== 400) return false; return /context_length_exceeded|context length|context size|context window|maximum context|prompt is too long|input is too long|too many tokens/i.test(error.body); } diff --git a/src/providers/transport/image-fallback.ts b/src/providers/transport/image-fallback.ts new file mode 100644 index 000000000..c0ca5fdbf --- /dev/null +++ b/src/providers/transport/image-fallback.ts @@ -0,0 +1,104 @@ +/** + * Fallback for a request whose images the model service refuses. A request that carries images and + * is refused with 413, or with a 400 that is not a context overflow, is sent once more without + * them. When that one is accepted, the images were the cause: their handles are remembered, every + * later request through this client leaves them out, and each record that lost one gets the + * `imageLeftOut` line after its text. The handles live in memory only, so after a restart the first + * request carrying one of them is refused once more. + * + * A 413 is retried without images even though `isContextOverflow` counts it: attachment bytes reach a + * service's body size limit long before they move the token count. A 413 that survives the retry + * reaches Core as a context overflow. + */ +import { BaseProvider } from '../base.ts'; +import { GenerationError, type GenerateOptions, type Generation } from '../../core/generation.ts'; +import type { ContextRecord, Item } from '../../protocol/open-responses/context.ts'; +import type { Request } from '../../protocol/open-responses/index.ts'; +import type { Logger } from '../../core/types.ts'; +import { imageBlobs, type CompatMediaOptions } from './history.ts'; + +/** Statuses of a request the service will not take as written. */ +const REFUSED: ReadonlySet = new Set([400, 413]); + +export interface ImageFallbackOptions { + media: CompatMediaOptions; + /** The module's `contextOverflow`. A refusal it matches is not retried, except a 413. */ + contextOverflow(error: GenerationError): boolean; + /** The records whose images the client attaches to a request; every record when absent. */ + attached?(context: readonly ContextRecord[]): readonly ContextRecord[]; + log: Logger; +} + +/** Follows the `[blob …]` line of an image a request goes out without. */ +export const imageLeftOut = (handle: string): string => + `[${handle} left out: the model service rejected the request that carried this image]`; + +export class ImageFallback extends BaseProvider { + /** Images whose request was accepted only without them. */ + private readonly refused = new Set(); + + constructor(private readonly inner: BaseProvider, private readonly options: ImageFallbackOptions) { + super(); + } + + async respond(request: Request, options: GenerateOptions = {}): Promise { + const context = options.context && withoutImages(options.context, this.refused); + try { + return await this.inner.respond(request, context ? { ...options, context } : options); + } catch (error) { + if (!context || options.diagnostic || options.signal?.aborted || !(error instanceof GenerationError) || !this.retried(error)) throw error; + const sent = this.sentImages(context); + if (!sent.size) throw error; + let generation: Generation; + try { + generation = await this.inner.respond(request, { ...options, context: withoutImages(context, sent) }); + } catch (retry) { + if (!(retry instanceof GenerationError)) throw retry; + throw new GenerationError(retry.message, [...error.attempts, ...retry.attempts], retry.partial, retry.origin, retry.status, retry.body, { cause: retry.cause }); + } + for (const handle of sent) this.refused.add(handle); + this.options.log.warn('Model service rejected a request with images; sent again without them', { status: error.status, images: [...sent] }); + return { ...generation, attempts: [...error.attempts, ...generation.attempts] }; + } + } + + private retried(error: GenerationError): boolean { + return REFUSED.has(error.status) && (error.status === 413 || !this.options.contextOverflow(error)); + } + + /** The images the client attaches to a request built from `context`. */ + private sentImages(context: readonly ContextRecord[]): Set { + const { media, attached } = this.options; + if (!media.enabled()) return new Set(); + return new Set((attached?.(context) ?? context).flatMap((entry) => imageBlobs(entry.context.blobs) + .filter((ref) => media.read(ref.handle)) + .map((ref) => ref.handle))); + } +} + +/** The context without the images in `handles`; each record that lost one gets an `imageLeftOut` line per image. */ +export function withoutImages(context: readonly ContextRecord[], handles: ReadonlySet): readonly ContextRecord[] { + if (!handles.size) return context; + return context.map((entry) => { + const blobs = entry.context.blobs ?? []; + const out = imageBlobs(blobs).filter((ref) => handles.has(ref.handle)); + if (!out.length) return entry; + return { + ...entry, + item: withLine(entry.item, out.map((ref) => imageLeftOut(ref.handle)).join('\n')), + context: { ...entry.context, blobs: blobs.filter((ref) => !out.includes(ref)) }, + }; + }); +} + +/** + * The item with `line` on its own line after its text; content parts other than text stay in place. + * An appended part starts with a newline because Chat joins text parts without a separator. + */ +function withLine(item: Item, line: string): Item { + if (item.type === 'message') + return { ...item, content: typeof item.content === 'string' ? `${item.content}\n${line}` : [...item.content, { type: 'input_text', text: `\n${line}` }] } as Item; + if (item.type === 'function_call_output') + return { ...item, output: typeof item.output === 'string' ? `${item.output}\n${line}` : [...item.output, { type: 'input_text', text: `\n${line}` }] } as Item; + return item; +} diff --git a/tests/core/llm.test.ts b/tests/core/llm.test.ts index fba549838..90247c4ac 100644 --- a/tests/core/llm.test.ts +++ b/tests/core/llm.test.ts @@ -6,6 +6,7 @@ import { afterEach, describe, expect, it, vi } from 'vitest'; import module, { compatOptions } from '../../src/providers/openai-responses-compat/index.ts'; import { ResponsesProvider, buildResponsesBody } from '../../src/providers/openai-responses-compat/native.ts'; import { validateEntry, validateSpec } from '../../src/providers/configuration.ts'; +import { ImageFallback } from '../../src/providers/transport/image-fallback.ts'; import { responseRequest } from '../../src/protocol/open-responses/context-helpers.ts'; import { message, record } from '../../src/protocol/open-responses/context.ts'; import { GenerationError } from '../../src/core/generation.ts'; @@ -167,7 +168,7 @@ describe('模块声明与条目校验', () => { stateDir: 'unused', secret: (name) => (name === 'K' ? 'sk' : ''), readBlob: () => null, keepThinking: () => true, log: { info() {}, warn() {}, error() {}, debug() {}, child() { return this; } } as never, }); - expect(instance.client).toBeInstanceOf(ResponsesProvider); + expect(instance.client).toBeInstanceOf(ImageFallback); expect(typeof instance.listModels).toBe('function'); expect(instance.contextWindow!('anything')).toBeUndefined(); expect(instance.compatibilityKey!()).toEqual(['/x', 'cloud']); diff --git a/tests/providers/registry.test.ts b/tests/providers/registry.test.ts index a989da5ae..6cfe71b3a 100644 --- a/tests/providers/registry.test.ts +++ b/tests/providers/registry.test.ts @@ -1,4 +1,7 @@ -/** 实例缓存键感知端点 `.env` 的内容:文件变更后下一次解析重建实例。 */ +/** + * 实例缓存键感知端点 `.env` 的内容:文件变更后下一次解析重建实例。 + * `bind` 给没有声明 `handlesRefusedImages` 的实例套一层 `ImageFallback`,每个实例一份。 + */ import { afterEach, expect, it } from 'vitest'; import { mkdirSync, writeFileSync } from 'node:fs'; import { join } from 'node:path'; @@ -6,6 +9,10 @@ import { makeCfg, makeTmpDir } from '../core/helpers.ts'; import { nullLogger } from '../../src/core/util.ts'; import { ProviderRegistry } from '../../src/providers/registry.ts'; import { BaseProvider, type ProviderModule } from '../../src/providers/base.ts'; +import { GenerationError, type GenerateOptions, type Generation } from '../../src/core/generation.ts'; +import type { BlobRef } from '../../src/core/types.ts'; +import { createResponse, type Request } from '../../src/protocol/open-responses/index.ts'; +import { message } from '../../src/protocol/open-responses/context.ts'; class StubClient extends BaseProvider { async respond(): Promise { throw new Error('unused'); } @@ -48,3 +55,43 @@ it('keeps the cached instance while the .env content is unchanged', () => { expect(registry.resolve('Alpha')).toBe(first); expect(seenKeys).toHaveLength(1); }); + +const picture: BlobRef = { handle: 'log:0123456789abcdef0123.png', mime: 'image/png', name: 'big.png', fallbackText: 'Image' }; + +/** A client whose service refuses every request that carries an image; `carried` is one entry per request. */ +class RefusesImages extends BaseProvider { + readonly carried: boolean[] = []; + async respond(request: Request, options: GenerateOptions = {}): Promise { + const carries = (options.context ?? []).some((entry) => entry.context.blobs?.length); + this.carried.push(carries); + if (carries) throw new GenerationError('refused', [], null, options.origin!, 400, 'unsupported image'); + return { response: createResponse('r1', request), origin: options.origin!, attempts: [] }; + } +} + +function imageFixture(handlesRefusedImages?: true) { + const temp = makeTmpDir(); cleanups.push(temp.cleanup); + const client = new RefusesImages(); + const module: ProviderModule = { + id: 'stub', title: 'Stub', reasoningTiers: [], serviceTiers: [], + create: () => ({ client, ...(handlesRefusedImages ? { handlesRefusedImages } : {}) }), + }; + const cfg = makeCfg({ providers: { Alpha: { kind: 'stub', baseUrl: 'https://model.test', multimodal: true } }, activeProvider: 'Alpha' }); + const registry = new ProviderRegistry(() => cfg.providers, { stateRoot: temp.dir, readBlob: () => Buffer.from('image'), keepThinking: () => true, log: nullLogger() }, [module]); + const ask = () => registry.bind('Alpha').respond({ model: 'm', input: [] }, { context: [message('user', 'look', { blobs: [picture] })] }); + return { client, ask }; +} + +it('sends a request refused for its images once more without them, and the next bind leaves them out: one request', async () => { + const { client, ask } = imageFixture(); + await ask(); + expect(client.carried).toEqual([true, false]); + await ask(); + expect(client.carried).toEqual([true, false, false]); +}); + +it('adds no fallback around an instance that declares handlesRefusedImages: one request, and the refusal reaches the caller', async () => { + const { client, ask } = imageFixture(true); + await expect(ask()).rejects.toMatchObject({ status: 400 }); + expect(client.carried).toEqual([true]); +}); diff --git a/tests/providers/transport/errors.test.ts b/tests/providers/transport/errors.test.ts index 3bec09a7f..0373ed10a 100644 --- a/tests/providers/transport/errors.test.ts +++ b/tests/providers/transport/errors.test.ts @@ -1,8 +1,19 @@ import { afterEach, describe, expect, it, vi } from 'vitest'; -import { parseRetryAfter } from '../../../src/providers/transport/errors.ts'; +import { isContextOverflow, parseRetryAfter } from '../../../src/providers/transport/errors.ts'; afterEach(() => vi.useRealTimers()); +describe('isContextOverflow', () => { + it('counts every 413, whatever its body says', () => { + expect(isContextOverflow({ status: 413, body: '413 Request Entity Too Large' })).toBe(true); + expect(isContextOverflow({ status: 413, body: '{"code":41301,"message":"request body over the size limit"}' })).toBe(true); + }); + it('counts a 400 only when its body says the context is exceeded', () => { + expect(isContextOverflow({ status: 400, body: '{"error":{"code":"context_length_exceeded"}}' })).toBe(true); + expect(isContextOverflow({ status: 400, body: '{"error":{"message":"unsupported image"}}' })).toBe(false); + }); +}); + describe('parseRetryAfter', () => { it('数字按秒,HTTP 日期按距今,缺失与不成形为 null', () => { vi.useFakeTimers(); diff --git a/tests/providers/transport/image-fallback.test.ts b/tests/providers/transport/image-fallback.test.ts new file mode 100644 index 000000000..326cf0c44 --- /dev/null +++ b/tests/providers/transport/image-fallback.test.ts @@ -0,0 +1,115 @@ +/** + * Requests the model service refuses for their images, through both built-in modules against a + * local fake service that refuses a request by its body. + */ +import { afterEach, describe, expect, it } from 'vitest'; +import { createServer, type Server } from 'node:http'; +import { mkdtempSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { GenerationError } from '../../../src/core/generation.ts'; +import { nullLogger } from '../../../src/core/util.ts'; +import { blobLine } from '../../../src/core/blobs.ts'; +import type { BlobRef, LLMProviderEntry } from '../../../src/core/types.ts'; +import type { ProviderModule } from '../../../src/providers/base.ts'; +import type { Request } from '../../../src/protocol/open-responses/index.ts'; +import { message } from '../../../src/protocol/open-responses/context.ts'; +import { isContextOverflow } from '../../../src/providers/transport/errors.ts'; +import { imageLeftOut } from '../../../src/providers/transport/image-fallback.ts'; +import COMPAT from '../../../src/providers/openai-responses-compat/index.ts'; +import LLAMACPP from '../../../src/providers/llamacpp/index.ts'; + +const IMAGE = Buffer.from('an image the service will not take'); +const IMAGE_DATA = IMAGE.toString('base64'); +const picture: BlobRef = { handle: 'log:0123456789abcdef0123.png', mime: 'image/png', name: 'big.png', fallbackText: 'Image' }; +const context = [message('user', `look\n${blobLine(picture)}`, { blobs: [picture] })]; +const OPENRESTY_413 = '413 Request Entity Too Large
openresty
'; + +const REPLIES: Record = { + '/responses': { id: 'r1', object: 'response', model: 'm', status: 'completed', created_at: 1, usage: { input_tokens: 1, output_tokens: 1, total_tokens: 2 }, + output: [{ type: 'message', id: 'm1', role: 'assistant', status: 'completed', content: [{ type: 'output_text', text: 'ok', annotations: [] }] }] }, + '/chat/completions': { id: 'c1', model: 'm', choices: [{ index: 0, message: { role: 'assistant', content: 'ok' }, finish_reason: 'stop' }], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 } }, +}; + +let server: Server | null = null; +afterEach(() => { server?.close(); server = null; }); + +/** A fake service that answers `status` with `error` to the requests `refuse` picks. */ +async function service(refuse: (body: string) => boolean, status = 400, error = JSON.stringify({ error: { message: 'unsupported image' } })) { + const bodies: string[] = []; + server = createServer(async (req, res) => { + let body = ''; + for await (const chunk of req) body += chunk; + bodies.push(body); + if (refuse(body)) return res.writeHead(status, { 'content-type': error.startsWith('<') ? 'text/html' : 'application/json' }).end(error); + res.writeHead(200, { 'content-type': 'application/json' }).end(JSON.stringify(REPLIES[(req.url ?? '').split('?')[0]!] ?? {})); + }); + await new Promise((resolve) => server!.listen(0, '127.0.0.1', () => resolve())); + return { url: `http://127.0.0.1:${(server!.address() as { port: number }).port}`, bodies }; +} + +const carriesImage = (body: string) => body.includes(IMAGE_DATA); + +function client(module: ProviderModule, url: string, patch: Partial = {}) { + const entry: LLMProviderEntry = { kind: module.id, baseUrl: url, multimodal: true, spec: { model: 'm', thinking: false }, ...patch }; + return module.create('x', entry, { + stateDir: mkdtempSync(join(tmpdir(), 'image-fallback-')), secret: () => '', keepThinking: () => true, log: nullLogger(), + readBlob: (handle: string) => (handle === picture.handle ? IMAGE : null), + }).client; +} + +const ask = (target: ReturnType) => target.respond({ model: 'm', input: [] } as unknown as Request, { context }); + +describe('a request the model service refuses for its images', () => { + it('is sent once more without them on both modules, and later requests leave them out: one request each', async () => { + const { url, bodies } = await service(carriesImage); + for (const module of [COMPAT, LLAMACPP]) { + bodies.length = 0; + const target = client(module, url); + const first = await ask(target); + expect(first.attempts.map((a) => a.status), module.id).toEqual([400, 200]); + expect(bodies.map(carriesImage), module.id).toEqual([true, false]); + expect(bodies[1], module.id).toContain(imageLeftOut(picture.handle)); + await ask(target); + expect(bodies.map(carriesImage), module.id).toEqual([true, false, false]); + expect(bodies[2], module.id).toBe(bodies[1]); + } + }); + + it('puts the left-out line on its own line on Chat, which joins text parts without a separator', async () => { + const { url, bodies } = await service(carriesImage); + await ask(client(LLAMACPP, url)); + expect(JSON.parse(bodies[1]!).messages.at(-1).content).toBe(`look\n${blobLine(picture)}\n${imageLeftOut(picture.handle)}`); + }); + + it('is sent again without them on a 413, which otherwise counts as a context overflow', async () => { + const { url, bodies } = await service(carriesImage, 413, OPENRESTY_413); + const first = await ask(client(COMPAT, url)); + expect(first.attempts.map((a) => a.status)).toEqual([413, 200]); + expect(bodies.map(carriesImage)).toEqual([true, false]); + }); + + it('keeps both attempts on the error when the request without them is refused too, and sends them next time', async () => { + const { url, bodies } = await service(() => true, 413, OPENRESTY_413); + const target = client(COMPAT, url); + const error = await ask(target).catch((e: unknown) => e); + expect(error).toBeInstanceOf(GenerationError); + expect((error as GenerationError).attempts.map((a) => a.status)).toEqual([413, 413]); + expect(isContextOverflow(error as GenerationError)).toBe(true); + await ask(target).catch(() => {}); + expect(bodies.map(carriesImage)).toEqual([true, false, true, false]); + }); + + it('is not sent again when the refusal is a token overflow: one request', async () => { + const { url, bodies } = await service(carriesImage, 400, JSON.stringify({ error: { code: 'context_length_exceeded', message: 'too long' } })); + await expect(ask(client(COMPAT, url))).rejects.toMatchObject({ status: 400 }); + expect(bodies).toHaveLength(1); + }); + + it('is not sent again when the endpoint sends no images: one request', async () => { + const { url, bodies } = await service(() => true); + await expect(ask(client(COMPAT, url, { multimodal: false }))).rejects.toMatchObject({ status: 400 }); + expect(bodies).toHaveLength(1); + }); +});