From 8d962059580a90a7d3ac01fec7964b5403b01681 Mon Sep 17 00:00:00 2001 From: Mark Buckaway Date: Tue, 1 Sep 2026 23:07:12 -0400 Subject: [PATCH 1/2] feat: recover from image rejections and add Files API image upload When a provider rejects an image (e.g. DeepSeek's 'This model does not support image' 400), the session is no longer frozen: the rejected image is stripped from all later requests, across turns and session reloads. Remote http(s) media is now capability-gated, the non-standard image_url id is dropped, and a config-declared image_file_api capability sends images via the provider's Files API in addition to inline base64 and URLs. --- .../deepseek-image-rejection-recovery.md | 5 + .changeset/image-file-api-upload.md | 5 + .../agent-core-v2/docs/state-manifest.d.ts | 12 +- .../agent-core-v2/docs/wire-manifest.d.ts | 16 ++- .../src/agent/contextMemory/vacuousContent.ts | 1 + .../agent/contextProjector/mediaProjection.ts | 22 ++- .../src/agent/llmRequester/llmRequestOps.ts | 15 ++ .../agent/llmRequester/llmRequesterService.ts | 43 +++--- .../src/agent/loop/loopService.ts | 1 + .../agent-core-v2/src/agent/mcp/output.ts | 2 +- .../src/agent/media/mediaResolverService.ts | 76 ++++++++-- .../src/kosong/contract/capability.ts | 3 + .../src/kosong/contract/errors.ts | 12 ++ .../src/kosong/contract/message.ts | 14 +- .../src/kosong/contract/provider.ts | 12 +- .../src/kosong/contract/tokens.ts | 2 + .../src/kosong/model/catalogService.ts | 1 + .../src/kosong/model/modelRequester.ts | 8 +- .../src/kosong/model/modelRequesterImpl.ts | 20 ++- .../src/kosong/protocol/protocolTrait.ts | 9 +- .../provider/bases/openai/openai-common.ts | 10 +- .../provider/bases/openai/openai-legacy.ts | 15 ++ .../provider/bases/openai/openaiHooks.ts | 6 +- .../provider/providers/kimi/kimi-files.ts | 78 +++++++++- .../provider/providers/kimi/kimi.contrib.ts | 2 + .../projector-tool-exchanges.test.ts | 38 +++++ .../llmRequesterMediaStrip.test.ts | 25 ++++ .../llmRequester/llmRequesterService.test.ts | 43 +++++- .../test/agent/media/mediaResolver.test.ts | 134 ++++++++++++++++++ .../test/app/llmProtocol/errors.test.ts | 44 ++++++ packages/agent-core-v2/test/index.test.ts | 1 + .../test/kosong/model/catalog.test.ts | 45 ++++++ .../test/kosong/provider/errors.test.ts | 28 +++- .../test/kosong/provider/kimi.test.ts | 35 +++++ .../test/state/builtinReplayableKeys.ts | 2 + packages/agent-core/src/agent/index.ts | 14 ++ packages/agent-core/src/agent/tool/index.ts | 58 ++++++++ packages/agent-core/src/agent/turn/index.ts | 46 +++++- packages/agent-core/src/config/env-model.ts | 6 +- packages/agent-core/src/mcp/output.ts | 2 +- .../src/services/message/message.ts | 5 + packages/agent-core/src/session/index.ts | 17 ++- .../src/session/provider-manager.ts | 1 + .../src/tools/builtin/file/read-media.ts | 51 ++++--- .../src/tools/support/video-delivery.ts | 38 ++++- packages/agent-core/src/utils/tokens.ts | 2 + packages/agent-core/test/agent/basic.test.ts | 20 +++ .../agent-core/test/agent/harness/agent.ts | 4 + .../agent-core/test/agent/kosong-llm.test.ts | 19 +++ packages/agent-core/test/agent/turn.test.ts | 67 +++++++++ .../agent-core/test/config/env-model.test.ts | 7 + packages/agent-core/test/session/init.test.ts | 27 ++++ .../test/session/provider-manager.test.ts | 33 +++++ .../agent-core/test/tools/read-media.test.ts | 73 ++++++++++ packages/kosong/src/capability.ts | 3 + packages/kosong/src/errors.ts | 6 + packages/kosong/src/index.ts | 1 + packages/kosong/src/message.ts | 16 ++- packages/kosong/src/provider.ts | 14 +- packages/kosong/src/providers/kimi-files.ts | 94 +++++++++++- packages/kosong/src/providers/kimi.ts | 5 + .../kosong/src/providers/openai-common.ts | 10 +- packages/kosong/test/capability.test.ts | 17 +++ packages/kosong/test/errors.test.ts | 44 ++++++ packages/kosong/test/kimi-files.test.ts | 39 +++++ packages/kosong/test/message.test.ts | 5 + .../kosong/test/openai-common-errors.test.ts | 11 +- packages/kosong/test/type-safety.test.ts | 4 + 68 files changed, 1445 insertions(+), 99 deletions(-) create mode 100644 .changeset/deepseek-image-rejection-recovery.md create mode 100644 .changeset/image-file-api-upload.md create mode 100644 packages/agent-core-v2/test/agent/llmRequester/llmRequesterMediaStrip.test.ts diff --git a/.changeset/deepseek-image-rejection-recovery.md b/.changeset/deepseek-image-rejection-recovery.md new file mode 100644 index 00000000000..115d0adb1b7 --- /dev/null +++ b/.changeset/deepseek-image-rejection-recovery.md @@ -0,0 +1,5 @@ +--- +"@moonshot-ai/kimi-code": patch +--- + +Fix image upload rejections freezing the session: when a provider rejects an image, the image is no longer re-sent, so the conversation continues without restarting. diff --git a/.changeset/image-file-api-upload.md b/.changeset/image-file-api-upload.md new file mode 100644 index 00000000000..bdf62d1a540 --- /dev/null +++ b/.changeset/image-file-api-upload.md @@ -0,0 +1,5 @@ +--- +"@moonshot-ai/kimi-code": minor +--- + +Send images to vision models through the provider's Files API. Enable it by adding `image_file_api` to a model's `capabilities` in config.toml. diff --git a/packages/agent-core-v2/docs/state-manifest.d.ts b/packages/agent-core-v2/docs/state-manifest.d.ts index 9c44bb78b8e..855528a25f0 100644 --- a/packages/agent-core-v2/docs/state-manifest.d.ts +++ b/packages/agent-core-v2/docs/state-manifest.d.ts @@ -1069,6 +1069,9 @@ export interface AgentStateSnapshot { url: string; id?: string; }; + } | /* FilePart — packages/agent-core-v2/src/kosong/contract/message.ts */ { + type: 'file'; + fileId: string; })[]; readonly toolCalls: /* ToolCall — packages/agent-core-v2/src/kosong/contract/message.ts */ { type: 'function'; @@ -1192,9 +1195,8 @@ export interface AgentStateSnapshot { 'llmRequester.emittedThinkingEffortWarnings': Set; 'llmRequester.lastConfigLogSignature': string | undefined; 'llmRequester.mediaDegradedTurns': Set; - 'llmRequester.mediaStrippedTurns': Map; + // replayable · durable — folds: MediaStripped + 'llmRequester.mediaStrippedTurns': readonly string[]; 'llmRequester.turnConfigs': Map; // src/agent/media/mediaToolsRegistrar.ts 'media.registeredKey': string | undefined; diff --git a/packages/agent-core-v2/docs/wire-manifest.d.ts b/packages/agent-core-v2/docs/wire-manifest.d.ts index 5afc4d0e6bd..e7fc8d813af 100644 --- a/packages/agent-core-v2/docs/wire-manifest.d.ts +++ b/packages/agent-core-v2/docs/wire-manifest.d.ts @@ -24,7 +24,7 @@ // cross-reducers), blobs (the folding states whose blob codec offloads inline // media to blob storage), owner (the source file declaring the class). -// Index (57 record types) +// Index (58 record types) // config.update profile src/agent/profile/profileOps.ts // context.append_loop_event contextMemory, turn src/agent/contextMemory/contextEvents.ts // context.append_message contextMemory, plan, task.notificationDelivery src/agent/contextMemory/contextEvents.ts @@ -44,6 +44,7 @@ // interaction.request (none) src/features/interaction/interactionOps.ts // interaction.resolved (none) src/features/interaction/interactionOps.ts // interruptionReminder.recorded interruptionReminder src/agent/interruptionReminder/interruptionReminderOps.ts +// llm.media_stripped llmRequester.mediaStrippedTurns src/agent/llmRequester/llmRequestOps.ts // llm.request llm.requestTrace src/agent/llmRequester/llmRequestOps.ts // llm.tools_snapshot llm.requestTrace src/agent/llmRequester/llmRequestOps.ts // mcp.tools_discovered mcp.discovery src/agent/mcp/mcpDiscoveryOps.ts @@ -128,7 +129,7 @@ interface ContextAppendMessagePayload { message: { role: 'system' | 'user' | 'assistant' | 'tool'; name?: string; - content: ('text' | 'think' | 'image_url' | 'audio_url' | 'video_url')[]; + content: ('text' | 'think' | 'image_url' | 'audio_url' | 'video_url' | 'file')[]; toolCalls: { type: 'function'; id: string; @@ -339,6 +340,16 @@ interface InterruptionReminderRecordedPayload { turnId: number; } +/** + * states: llmRequester.mediaStrippedTurns + * owner: src/agent/llmRequester/llmRequestOps.ts + */ +interface LlmMediaStrippedPayload { + _name: 'llm.media_stripped'; + agentId: string; + keys: string[]; +} + /** * states: llm.requestTrace * owner: src/agent/llmRequester/llmRequestOps.ts @@ -867,6 +878,7 @@ interface WirePayloadMap { "interaction.request": InteractionRequestPayload; "interaction.resolved": InteractionResolvedPayload; "interruptionReminder.recorded": InterruptionReminderRecordedPayload; + "llm.media_stripped": LlmMediaStrippedPayload; "llm.request": LlmRequestPayload; "llm.tools_snapshot": LlmToolsSnapshotPayload; "mcp.tools_discovered": McpToolsDiscoveredPayload; diff --git a/packages/agent-core-v2/src/agent/contextMemory/vacuousContent.ts b/packages/agent-core-v2/src/agent/contextMemory/vacuousContent.ts index e0e1f9e46ab..b95b41c3548 100644 --- a/packages/agent-core-v2/src/agent/contextMemory/vacuousContent.ts +++ b/packages/agent-core-v2/src/agent/contextMemory/vacuousContent.ts @@ -9,6 +9,7 @@ export function isVacuousContentPart(part: ContentPart): boolean { case 'image_url': case 'audio_url': case 'video_url': + case 'file': return false; default: { const exhaustive: never = part; diff --git a/packages/agent-core-v2/src/agent/contextProjector/mediaProjection.ts b/packages/agent-core-v2/src/agent/contextProjector/mediaProjection.ts index 741465ce9b1..812c1e20369 100644 --- a/packages/agent-core-v2/src/agent/contextProjector/mediaProjection.ts +++ b/packages/agent-core-v2/src/agent/contextProjector/mediaProjection.ts @@ -36,10 +36,6 @@ interface MediaContainer { readonly id?: string; } -interface MediaStripSnapshotData { - readonly keys: ReadonlySet; -} - type MediaContainerKeyCache = Partial>; const MEDIA_CONTAINER_KEY_CACHE = new WeakMap(); @@ -77,8 +73,13 @@ function mediaStripKey(part: DegradableMediaPart): string { return key; } -function mediaStripSnapshotKeys(snapshot: MediaStripSnapshot): ReadonlySet { - return (snapshot as unknown as MediaStripSnapshotData).keys; +export function mediaStripSnapshotKeys(snapshot: MediaStripSnapshot): ReadonlySet { + const keys = (snapshot as unknown as { keys?: ReadonlySet | undefined }).keys; + return keys ?? new Set(); +} + +export function mediaStripSnapshotFromKeys(keys: Iterable): MediaStripSnapshot { + return Object.freeze({ keys: new Set(keys) }) as unknown as MediaStripSnapshot; } export function captureMediaStripSnapshot( @@ -93,6 +94,15 @@ export function captureMediaStripSnapshot( return Object.freeze({ keys }) as unknown as MediaStripSnapshot; } +export function mergeMediaStripSnapshots( + a: MediaStripSnapshot, + b: MediaStripSnapshot, +): MediaStripSnapshot { + const keys = new Set(mediaStripSnapshotKeys(a)); + for (const key of mediaStripSnapshotKeys(b)) keys.add(key); + return Object.freeze({ keys }) as unknown as MediaStripSnapshot; +} + export function stripMediaPartsBySnapshot( messages: readonly Message[], snapshot: MediaStripSnapshot, diff --git a/packages/agent-core-v2/src/agent/llmRequester/llmRequestOps.ts b/packages/agent-core-v2/src/agent/llmRequester/llmRequestOps.ts index 0f60cc00944..b8f54f03f6b 100644 --- a/packages/agent-core-v2/src/agent/llmRequester/llmRequestOps.ts +++ b/packages/agent-core-v2/src/agent/llmRequester/llmRequestOps.ts @@ -96,6 +96,21 @@ export interface LlmRequest { readonly droppedCount?: number; } +const mediaStrippedSchema = z.object({ + agentId: z.string(), + keys: z.array(z.string()).readonly(), +}); + +export class MediaStripped extends AgentEvent2> { + static override readonly type = 'llm.media_stripped'; + static override readonly durable = true; + static override readonly schema = mediaStrippedSchema; +} +export interface MediaStripped { + readonly agentId: string; + readonly keys: readonly string[]; +} + export const llmRequestTraceKey = defineState( 'llm.requestTrace', (): LlmRequestTraceState => ({ seenToolsHashes: [] }), diff --git a/packages/agent-core-v2/src/agent/llmRequester/llmRequesterService.ts b/packages/agent-core-v2/src/agent/llmRequester/llmRequesterService.ts index 4f72b8814f8..13527c287f5 100644 --- a/packages/agent-core-v2/src/agent/llmRequester/llmRequesterService.ts +++ b/packages/agent-core-v2/src/agent/llmRequester/llmRequesterService.ts @@ -1,4 +1,5 @@ import { createHash } from 'node:crypto'; +import { z } from 'zod'; import { LifecycleScope } from '#/app/scopes'; import { ScopeActivation, registerScopedService } from '#/_base/di/scope'; import { defineState } from '#/state/state'; @@ -8,6 +9,10 @@ import { type MediaStripSnapshot, type ProjectionPolicy, } from '#/agent/contextProjector/contextProjector'; +import { + mediaStripSnapshotFromKeys, + mediaStripSnapshotKeys, +} from '#/agent/contextProjector/mediaProjection'; import { ISessionTokenCountingService } from '#/session/tokenCounting/sessionTokenCounting'; import { IAgentProfileService, type ProfileModelContext } from '#/agent/profile/profile'; import { IAgentStateService } from '#/agent/state/agentState'; @@ -69,6 +74,7 @@ import { LlmRequest, llmRequestTraceKey, LlmToolsSnapshot, + MediaStripped, type LlmRequestPayload, type LlmRequestToolSchema, } from './llmRequestOps'; @@ -136,10 +142,18 @@ export const llmRequesterMediaDegradedTurnsKey = defineState>( 'llmRequester.mediaDegradedTurns', () => new Set(), ); -export const llmRequesterMediaStrippedTurnsKey = defineState>( +export const llmRequesterMediaStrippedTurnsKey = defineState( 'llmRequester.mediaStrippedTurns', - () => new Map(), -); + (): readonly string[] => [], +) + .replayable({ schema: z.array(z.string()).readonly() }) + .on(MediaStripped, (s, e) => { + for (const key of e.keys) { + if (!s.includes(key)) { + s.push(key); + } + } + }); export const llmRequesterEmittedThinkingEffortWarningsKey = defineState>( 'llmRequester.emittedThinkingEffortWarnings', () => new Set(), @@ -193,10 +207,6 @@ export class AgentLLMRequesterService implements IAgentLLMRequesterService { return this.states.get(llmRequesterMediaDegradedTurnsKey); } - private get mediaStrippedTurns(): Map { - return this.states.get(llmRequesterMediaStrippedTurnsKey); - } - private get emittedThinkingEffortWarnings(): Set { return this.states.get(llmRequesterEmittedThinkingEffortWarningsKey); } @@ -594,21 +604,22 @@ export class AgentLLMRequesterService implements IAgentLLMRequesterService { } private mediaStripSnapshotForTurn( - source: AgentLLMRequestSource | undefined, + _source: AgentLLMRequestSource | undefined, ): MediaStripSnapshot | undefined { - if (source?.type !== 'turn') return undefined; - return this.mediaStrippedTurns.get(source.turnId); + const keys = this.states.get(llmRequesterMediaStrippedTurnsKey); + return keys.length === 0 ? undefined : mediaStripSnapshotFromKeys(keys); } private markMediaStrippedRecoveryTurn( snapshot: MediaStripSnapshot, - source: AgentLLMRequestSource | undefined, + _source: AgentLLMRequestSource | undefined, ): void { - if (source?.type !== 'turn') return; - for (const id of this.mediaStrippedTurns.keys()) { - if (id < source.turnId) this.mediaStrippedTurns.delete(id); - } - this.mediaStrippedTurns.set(source.turnId, snapshot); + void this.dispatcher.dispatch( + new MediaStripped({ + agentId: this.scopeContext.agentId, + keys: [...mediaStripSnapshotKeys(snapshot)], + }), + ); } private markRecoveryTurn(set: Set, source: AgentLLMRequestSource | undefined): void { diff --git a/packages/agent-core-v2/src/agent/loop/loopService.ts b/packages/agent-core-v2/src/agent/loop/loopService.ts index d094d2a5527..9602a261191 100644 --- a/packages/agent-core-v2/src/agent/loop/loopService.ts +++ b/packages/agent-core-v2/src/agent/loop/loopService.ts @@ -1151,6 +1151,7 @@ export class AgentLoopService extends Disposable implements IAgentLoopService { case 'image_url': case 'audio_url': case 'video_url': + case 'file': return; case 'function': { onResponseEvent(); diff --git a/packages/agent-core-v2/src/agent/mcp/output.ts b/packages/agent-core-v2/src/agent/mcp/output.ts index dcf44180da2..c6addb49807 100644 --- a/packages/agent-core-v2/src/agent/mcp/output.ts +++ b/packages/agent-core-v2/src/agent/mcp/output.ts @@ -217,7 +217,7 @@ function applyBinaryPartCap(parts: readonly ContentPart[]): { const notices: string[] = []; for (const part of parts) { - if (part.type === 'text' || part.type === 'think') { + if (part.type === 'text' || part.type === 'think' || part.type === 'file') { out.push(part); continue; } diff --git a/packages/agent-core-v2/src/agent/media/mediaResolverService.ts b/packages/agent-core-v2/src/agent/media/mediaResolverService.ts index 55f640f9a1d..d548dd1dba8 100644 --- a/packages/agent-core-v2/src/agent/media/mediaResolverService.ts +++ b/packages/agent-core-v2/src/agent/media/mediaResolverService.ts @@ -1,6 +1,7 @@ import { createHash } from 'node:crypto'; import { ScopeActivation, registerScopedService } from '#/_base/di/scope'; +import { MEDIA_STRIPPED_PLACEHOLDERS } from '#/agent/contextProjector/mediaProjection'; import { defineState } from '#/state/state'; import { IAgentStateService } from '#/agent/state/agentState'; import { IFileService } from '#/app/file/fileService'; @@ -70,29 +71,37 @@ export class AgentMediaResolverService implements IAgentMediaResolverService { requester: ModelRequester, signal?: AbortSignal, ): Promise { - if (!messages.some(hasDaemonFileMediaPart)) return messages; + if (!messages.some(hasMediaPart)) return messages; let changed = false; const out: Message[] = []; for (const message of messages) { - if (!hasDaemonFileMediaPart(message)) { + if (!hasMediaPart(message)) { out.push(message); continue; } const content: ContentPart[] = []; let sawVideoRef = false; + let messageChanged = false; for (const part of message.content) { const daemonPart = daemonFileRefFromPart(part); - if (daemonPart === undefined) { - content.push(part); + if (daemonPart !== undefined) { + sawVideoRef ||= daemonPart.kind === 'video'; + const resolved = + daemonPart.kind === 'video' + ? await this.resolveVideoPart(daemonPart.ref, requester, signal) + : await this.resolveImagePart(daemonPart.ref, requester, signal); + content.push(resolved); + messageChanged = true; continue; } - sawVideoRef ||= daemonPart.kind === 'video'; - const resolved = - daemonPart.kind === 'video' - ? await this.resolveVideoPart(daemonPart.ref, requester, signal) - : await this.resolveImagePart(daemonPart.ref, requester, signal); - content.push(resolved); + const gated = gateUnsupportedMediaPart(part, requester); + content.push(gated); + if (gated !== part) messageChanged = true; + } + if (!messageChanged) { + out.push(message); + continue; } out.push({ ...message, @@ -139,10 +148,15 @@ export class AgentMediaResolverService implements IAgentMediaResolverService { if (fileType.kind !== 'image') return degradedImage(path); if (!isModelAcceptedImageMime(fileType.mimeType)) return degradedImage(path); + const mimeType = normalizeImageMime(fileType.mimeType); + if (requester.model.capabilities.image_file_api === true) { + const uploaded = await this.tryUploadImage(source, mimeType, requester, signal); + if (uploaded !== undefined) return uploaded; + } const part: ContentPart = { type: 'image_url', imageUrl: { - url: `data:${normalizeImageMime(fileType.mimeType)};base64,${source.bytes.toString('base64')}`, + url: `data:${mimeType};base64,${source.bytes.toString('base64')}`, }, }; if (source.bytes.length <= IMAGE_MEMO_MAX_BYTES) { @@ -151,6 +165,27 @@ export class AgentMediaResolverService implements IAgentMediaResolverService { return part; } + private async tryUploadImage( + source: { readonly bytes: Buffer; readonly filename: string }, + mimeType: string, + requester: ModelRequester, + signal: AbortSignal | undefined, + ): Promise { + const uploadImage = requester.uploadImage; + if (uploadImage === undefined) return undefined; + try { + const uploaded = await uploadImage( + { data: source.bytes, mimeType, filename: source.filename }, + signal === undefined ? undefined : { signal }, + ); + return { type: 'file', fileId: uploaded.fileId }; + } catch (error) { + if (signal?.aborted) throw error; + if (isVideoUploadAuthError(error)) throw error; + return undefined; + } + } + private memoedImage(cacheKey: string): ContentPart | undefined { const entry = this.imageMemo.get(cacheKey); if (entry === undefined) return undefined; @@ -292,8 +327,23 @@ export class AgentMediaResolverService implements IAgentMediaResolverService { } } -function hasDaemonFileMediaPart(message: Message): boolean { - return message.content.some((part) => daemonFileRefFromPart(part) !== undefined); +function hasMediaPart(message: Message): boolean { + return message.content.some( + (part) => part.type === 'image_url' || part.type === 'video_url' || part.type === 'audio_url', + ); +} + +function gateUnsupportedMediaPart(part: ContentPart, requester: ModelRequester): ContentPart { + if (part.type === 'image_url' && !requester.model.capabilities.image_in) { + return { type: 'text', text: MEDIA_STRIPPED_PLACEHOLDERS.image_url }; + } + if (part.type === 'video_url' && !requester.model.capabilities.video_in) { + return { type: 'text', text: MEDIA_STRIPPED_PLACEHOLDERS.video_url }; + } + if (part.type === 'audio_url' && !requester.model.capabilities.audio_in) { + return { type: 'text', text: MEDIA_STRIPPED_PLACEHOLDERS.audio_url }; + } + return part; } function degradedImage(path: string | undefined): ContentPart { diff --git a/packages/agent-core-v2/src/kosong/contract/capability.ts b/packages/agent-core-v2/src/kosong/contract/capability.ts index a8825fb60b9..c938c4e96a7 100644 --- a/packages/agent-core-v2/src/kosong/contract/capability.ts +++ b/packages/agent-core-v2/src/kosong/contract/capability.ts @@ -7,6 +7,7 @@ export interface ModelCapability { readonly max_context_tokens: number; readonly max_input_tokens?: number; readonly dynamically_loaded_tools?: boolean; + readonly image_file_api?: boolean; } const UNKNOWN_CAPABILITY_MARKER = Symbol.for('moonshot-ai.kosong.UNKNOWN_CAPABILITY'); @@ -21,6 +22,7 @@ export const UNKNOWN_CAPABILITY: ModelCapability = Object.freeze( tool_use: false, max_context_tokens: 0, dynamically_loaded_tools: false, + image_file_api: false, }, UNKNOWN_CAPABILITY_MARKER, { value: true }, @@ -38,6 +40,7 @@ export function isUnknownCapability(capability: ModelCapability): boolean { !capability.audio_in && !capability.thinking && !capability.tool_use && + !capability.image_file_api && capability.dynamically_loaded_tools !== true && capability.max_context_tokens === 0 ); diff --git a/packages/agent-core-v2/src/kosong/contract/errors.ts b/packages/agent-core-v2/src/kosong/contract/errors.ts index 6f293da3aca..dd32ecbc028 100644 --- a/packages/agent-core-v2/src/kosong/contract/errors.ts +++ b/packages/agent-core-v2/src/kosong/contract/errors.ts @@ -60,6 +60,13 @@ export class VideoUploadUnsupportedError extends ChatProviderError { } } +export class ImageUploadUnsupportedError extends ChatProviderError { + constructor(message: string) { + super(message); + this.name = 'ImageUploadUnsupportedError'; + } +} + export class APITimeoutError extends ChatProviderError { constructor(message: string) { super(message, PROVIDER_CONNECTION_ERROR_CODE); @@ -212,6 +219,11 @@ const IMAGE_FORMAT_STATUS_MESSAGE_PATTERNS = [ /unable to process (?:the |input )?image/, /failed to decode (?:the )?image/, /invalid image(?: data| type| format)?/, + /this model does not support image/, + /unknown variant\s+\x60?image_url\x60?/, + /images are supported in user messages only/, + /image in (?:assistant|system|tool) message is not supported/, + /you have uploaded an unsupported image/, ] as const; const MEDIA_TYPE_FIELD_PATTERN = /(?:media|mime)_?type/; diff --git a/packages/agent-core-v2/src/kosong/contract/message.ts b/packages/agent-core-v2/src/kosong/contract/message.ts index 2743c9e7d69..aeee9140207 100644 --- a/packages/agent-core-v2/src/kosong/contract/message.ts +++ b/packages/agent-core-v2/src/kosong/contract/message.ts @@ -28,7 +28,12 @@ export interface VideoURLPart { videoUrl: { url: string; id?: string | undefined }; } -export type ContentPart = TextPart | ThinkPart | ImageURLPart | AudioURLPart | VideoURLPart; +export interface FilePart { + type: 'file'; + fileId: string; +} + +export type ContentPart = TextPart | ThinkPart | ImageURLPart | AudioURLPart | VideoURLPart | FilePart; export interface ToolCall { type: 'function'; @@ -60,7 +65,12 @@ export interface Message { export function isContentPart(part: StreamedMessagePart): part is ContentPart { const t = part.type; return ( - t === 'text' || t === 'think' || t === 'image_url' || t === 'audio_url' || t === 'video_url' + t === 'text' || + t === 'think' || + t === 'image_url' || + t === 'audio_url' || + t === 'video_url' || + t === 'file' ); } diff --git a/packages/agent-core-v2/src/kosong/contract/provider.ts b/packages/agent-core-v2/src/kosong/contract/provider.ts index 7deb5433253..10ceed50c01 100644 --- a/packages/agent-core-v2/src/kosong/contract/provider.ts +++ b/packages/agent-core-v2/src/kosong/contract/provider.ts @@ -1,4 +1,4 @@ -import type { Message, StreamedMessagePart, VideoURLPart } from './message'; +import type { FilePart, Message, StreamedMessagePart, VideoURLPart } from './message'; import type { Tool } from './tool'; import type { TokenUsage } from './usage'; @@ -70,6 +70,12 @@ export interface VideoUploadInput { readonly filename?: string | undefined; } +export interface ImageUploadInput { + readonly data: Uint8Array; + readonly mimeType: string; + readonly filename?: string | undefined; +} + export interface GenerateOptions { signal?: AbortSignal; auth?: ProviderRequestAuth; @@ -99,4 +105,8 @@ export interface ChatProvider { options?: GenerateOptions, ): Promise; uploadVideo?(input: string | VideoUploadInput, options?: GenerateOptions): Promise; + uploadImage?( + input: string | ImageUploadInput, + options?: GenerateOptions, + ): Promise; } diff --git a/packages/agent-core-v2/src/kosong/contract/tokens.ts b/packages/agent-core-v2/src/kosong/contract/tokens.ts index 046a91ee76e..4e5316536a4 100644 --- a/packages/agent-core-v2/src/kosong/contract/tokens.ts +++ b/packages/agent-core-v2/src/kosong/contract/tokens.ts @@ -72,6 +72,8 @@ export function estimateTokensForContentPart(part: ContentPart): number { case 'audio_url': case 'video_url': return MEDIA_TOKEN_ESTIMATE; + case 'file': + return estimateTokens(part.fileId); default: { const exhaustive: never = part; void exhaustive; diff --git a/packages/agent-core-v2/src/kosong/model/catalogService.ts b/packages/agent-core-v2/src/kosong/model/catalogService.ts index 903ebceea3d..b3cf7c5a486 100644 --- a/packages/agent-core-v2/src/kosong/model/catalogService.ts +++ b/packages/agent-core-v2/src/kosong/model/catalogService.ts @@ -497,6 +497,7 @@ function resolveModelCapabilities( dynamically_loaded_tools: declared.has('dynamically_loaded_tools') || detected.dynamically_loaded_tools === true, + image_file_api: declared.has('image_file_api') || detected.image_file_api === true, }; } diff --git a/packages/agent-core-v2/src/kosong/model/modelRequester.ts b/packages/agent-core-v2/src/kosong/model/modelRequester.ts index 737b266c424..de27275de1d 100644 --- a/packages/agent-core-v2/src/kosong/model/modelRequester.ts +++ b/packages/agent-core-v2/src/kosong/model/modelRequester.ts @@ -1,6 +1,7 @@ -import type { Message, StreamedMessagePart, VideoURLPart } from '#/kosong/contract/message'; +import type { FilePart, Message, StreamedMessagePart, VideoURLPart } from '#/kosong/contract/message'; import type { FinishReason, + ImageUploadInput, ResponseFormat, SamplingOptions, ThinkingEffort, @@ -65,6 +66,11 @@ export interface ModelRequester { input: string | VideoUploadInput, options?: { readonly signal?: AbortSignal }, ): Promise; + + uploadImage?( + input: string | ImageUploadInput, + options?: { readonly signal?: AbortSignal }, + ): Promise; } export function effectiveMaxCompletionTokens(params?: ModelRequestParams): number | undefined { diff --git a/packages/agent-core-v2/src/kosong/model/modelRequesterImpl.ts b/packages/agent-core-v2/src/kosong/model/modelRequesterImpl.ts index 39c3209bf29..f9e5c761334 100644 --- a/packages/agent-core-v2/src/kosong/model/modelRequesterImpl.ts +++ b/packages/agent-core-v2/src/kosong/model/modelRequesterImpl.ts @@ -1,8 +1,9 @@ import { AsyncEventQueue } from '#/_base/asyncEventQueue'; -import type { VideoURLPart } from '#/kosong/contract/message'; +import type { FilePart, VideoURLPart } from '#/kosong/contract/message'; import { APIContextOverflowError, APIStatusError, + ImageUploadUnsupportedError, isAbortError, VideoUploadUnsupportedError, } from '#/kosong/contract/errors'; @@ -10,6 +11,7 @@ import { generate, type GenerateResult } from '#/kosong/contract/generate'; import type { ChatProvider, GenerateOptions, + ImageUploadInput, ProviderRequestAuth, StreamDecodeStats, VideoUploadInput, @@ -77,6 +79,22 @@ export class ModelRequesterImpl implements ModelRequester { ); } + async uploadImage( + input: string | ImageUploadInput, + options?: { readonly signal?: AbortSignal }, + ): Promise { + const provider = this.resolveChatProvider(); + if (provider.uploadImage === undefined) { + throw new ImageUploadUnsupportedError( + `Model "${this.model.id}" (protocol=${this.model.protocol}) does not support image upload`, + ); + } + const uploadImage = provider.uploadImage.bind(provider); + return this.runWithAuthRefresh((auth) => + uploadImage(input, { signal: options?.signal, auth }), + ); + } + private async runRequest( input: ModelRequestInput, signal: AbortSignal | undefined, diff --git a/packages/agent-core-v2/src/kosong/protocol/protocolTrait.ts b/packages/agent-core-v2/src/kosong/protocol/protocolTrait.ts index 3670b4a4c4a..12660711ab9 100644 --- a/packages/agent-core-v2/src/kosong/protocol/protocolTrait.ts +++ b/packages/agent-core-v2/src/kosong/protocol/protocolTrait.ts @@ -1,8 +1,9 @@ import type { ModelCapability } from '#/kosong/contract/capability'; import type { ChatProviderError } from '#/kosong/contract/errors'; -import type { Message, VideoURLPart } from '#/kosong/contract/message'; +import type { FilePart, Message, VideoURLPart } from '#/kosong/contract/message'; import type { GenerateOptions, + ImageUploadInput, ThinkingEffort, ToolCallIdPolicy, VideoUploadInput, @@ -86,6 +87,12 @@ export interface ProtocolTrait { options: GenerateOptions | undefined, ctx: TraitContext, ): Promise; + + uploadImage?( + input: string | ImageUploadInput, + options: GenerateOptions | undefined, + ctx: TraitContext, + ): Promise; } export interface ResolvedTrait { diff --git a/packages/agent-core-v2/src/kosong/provider/bases/openai/openai-common.ts b/packages/agent-core-v2/src/kosong/provider/bases/openai/openai-common.ts index ea99d7fd7f5..f1f4cef6c77 100644 --- a/packages/agent-core-v2/src/kosong/provider/bases/openai/openai-common.ts +++ b/packages/agent-core-v2/src/kosong/provider/bases/openai/openai-common.ts @@ -26,9 +26,10 @@ import type { TokenUsage } from '#/kosong/contract/usage'; export interface OpenAIContentPart { type: string; text?: string | undefined; - image_url?: { url: string; id?: string | null } | undefined; + image_url?: { url: string } | undefined; audio_url?: { url: string; id?: string | null } | undefined; video_url?: { url: string; id?: string | null } | undefined; + file_id?: string | undefined; } export function convertContentPart(part: ContentPart): OpenAIContentPart | null { @@ -40,10 +41,7 @@ export function convertContentPart(part: ContentPart): OpenAIContentPart | null case 'image_url': return { type: 'image_url', - image_url: - part.imageUrl.id === undefined - ? { url: part.imageUrl.url } - : { url: part.imageUrl.url, id: part.imageUrl.id }, + image_url: { url: part.imageUrl.url }, }; case 'audio_url': return { @@ -61,6 +59,8 @@ export function convertContentPart(part: ContentPart): OpenAIContentPart | null ? { url: part.videoUrl.url } : { url: part.videoUrl.url, id: part.videoUrl.id }, }; + case 'file': + return { type: 'file', file_id: part.fileId }; default: throw new BugIndicatingError(`Unknown content part type: ${(part as ContentPart).type}`); } diff --git a/packages/agent-core-v2/src/kosong/provider/bases/openai/openai-legacy.ts b/packages/agent-core-v2/src/kosong/provider/bases/openai/openai-legacy.ts index f2b36111cab..52f607fe687 100644 --- a/packages/agent-core-v2/src/kosong/provider/bases/openai/openai-legacy.ts +++ b/packages/agent-core-v2/src/kosong/provider/bases/openai/openai-legacy.ts @@ -3,6 +3,7 @@ import OpenAI from 'openai'; import { parseTraceId, type ChatProviderError } from '#/kosong/contract/errors'; import type { ContentPart, + FilePart, Message, StreamedMessagePart, ToolCall, @@ -13,6 +14,7 @@ import type { ChatProvider, FinishReason, GenerateOptions, + ImageUploadInput, ProviderRequestAuth, ResponseFormat, StreamedMessage, @@ -87,6 +89,10 @@ export interface OpenAIChatCompletionsHooks { input: string | VideoUploadInput, options?: GenerateOptions, ) => Promise; + uploadImage?: ( + input: string | ImageUploadInput, + options?: GenerateOptions, + ) => Promise; } export interface OpenAILegacyOptions { @@ -507,6 +513,11 @@ export class OpenAILegacyChatProvider implements ChatProvider { options?: GenerateOptions, ) => Promise; + readonly uploadImage?: ( + input: string | ImageUploadInput, + options?: GenerateOptions, + ) => Promise; + constructor(options: OpenAILegacyOptions) { const apiKey = options.apiKey ?? process.env['OPENAI_API_KEY']; this._apiKey = apiKey === undefined || apiKey.length === 0 ? undefined : apiKey; @@ -537,6 +548,10 @@ export class OpenAILegacyChatProvider implements ChatProvider { if (uploadVideo !== undefined) { this.uploadVideo = (input, generateOptions) => uploadVideo(input, generateOptions); } + const uploadImage = this._hooks?.uploadImage; + if (uploadImage !== undefined) { + this.uploadImage = (input, generateOptions) => uploadImage(input, generateOptions); + } } get modelName(): string { diff --git a/packages/agent-core-v2/src/kosong/provider/bases/openai/openaiHooks.ts b/packages/agent-core-v2/src/kosong/provider/bases/openai/openaiHooks.ts index a1a512491ff..986aeac3eb9 100644 --- a/packages/agent-core-v2/src/kosong/provider/bases/openai/openaiHooks.ts +++ b/packages/agent-core-v2/src/kosong/provider/bases/openai/openaiHooks.ts @@ -1,4 +1,4 @@ -import type { GenerateOptions, VideoUploadInput } from '#/kosong/contract/provider'; +import type { GenerateOptions, ImageUploadInput, VideoUploadInput } from '#/kosong/contract/provider'; import type { Tool } from '#/kosong/contract/tool'; import type { ProtocolEndpoint, ResolvedTrait } from '#/kosong/protocol/protocolTrait'; @@ -80,6 +80,10 @@ export function composeOpenAIChatHooks( hooks.uploadVideo = (input: string | VideoUploadInput, options?: GenerateOptions) => trait.uploadVideo!(input, options, context); } + if (trait.uploadImage !== undefined) { + hooks.uploadImage = (input: string | ImageUploadInput, options?: GenerateOptions) => + trait.uploadImage!(input, options, context); + } } return Object.keys(hooks).length > 0 ? hooks : undefined; diff --git a/packages/agent-core-v2/src/kosong/provider/providers/kimi/kimi-files.ts b/packages/agent-core-v2/src/kosong/provider/providers/kimi/kimi-files.ts index 138068ee91c..7933500ef91 100644 --- a/packages/agent-core-v2/src/kosong/provider/providers/kimi/kimi-files.ts +++ b/packages/agent-core-v2/src/kosong/provider/providers/kimi/kimi-files.ts @@ -6,8 +6,12 @@ import type OpenAI from 'openai'; import OpenAIClient from 'openai'; import { ChatProviderError } from '#/kosong/contract/errors'; -import type { VideoURLPart } from '#/kosong/contract/message'; -import type { ProviderRequestAuth, VideoUploadInput } from '#/kosong/contract/provider'; +import type { FilePart, VideoURLPart } from '#/kosong/contract/message'; +import type { + ImageUploadInput, + ProviderRequestAuth, + VideoUploadInput, +} from '#/kosong/contract/provider'; import { convertOpenAIError } from '../../bases/openai/openai-common'; import { @@ -104,6 +108,53 @@ export class KimiFiles { }; } + async uploadImage( + input: string | ImageUploadInput, + options?: KimiUploadOptions, + ): Promise { + let file: unknown; + + if (typeof input === 'string') { + if (!fs.existsSync(input)) { + throw new ChatProviderError(`Image file not found: ${input}`); + } + const filename = path.basename(input); + const mimeType = guessImageMimeTypeFromExt(filename); + if (mimeType === undefined || !mimeType.startsWith('image/')) { + throw new ChatProviderError( + `KimiFiles.uploadImage: file extension does not indicate an image type: ${filename}`, + ); + } + const data = await fs.promises.readFile(input); + const blob = new Blob([new Uint8Array(data)], { type: mimeType }); + file = new File([blob], filename, { type: mimeType }); + } else { + if (!input.mimeType.startsWith('image/')) { + throw new ChatProviderError(`Expected an image mime type, got ${input.mimeType}`); + } + const filename = input.filename ?? guessImageFilename(input.mimeType); + const bytes = input.data instanceof Uint8Array ? input.data : new Uint8Array(input.data); + const blob = new Blob([bytes], { type: input.mimeType }); + file = new File([blob], filename, { type: input.mimeType }); + } + + let uploaded: { id: string }; + try { + const client = this._createClient(options?.auth); + uploaded = (await client.files.create( + { + file: file as never, + purpose: 'user_data' as never, + }, + options?.signal ? { signal: options.signal } : undefined, + )) as unknown as { id: string }; + } catch (error: unknown) { + throw convertOpenAIError(error, classifyKimiQuotaError); + } + + return { type: 'file', fileId: uploaded.id }; + } + private _createClient(auth: ProviderRequestAuth | undefined): OpenAI { return resolveAuthBackedClient( { cachedClient: this._client, clientFactory: this._clientFactory }, @@ -146,3 +197,26 @@ function guessMimeTypeFromExt(filename: string): string | undefined { const ext = filename.slice(dot + 1).toLowerCase(); return EXT_TO_MIME[ext]; } + +const IMAGE_MIME_TO_EXT: Record = { + 'image/png': 'png', + 'image/jpeg': 'jpg', + 'image/webp': 'webp', + 'image/gif': 'gif', +}; + +const IMAGE_EXT_TO_MIME: Record = Object.fromEntries( + Object.entries(IMAGE_MIME_TO_EXT).map(([mime, ext]) => [ext, mime]), +); + +function guessImageFilename(mimeType: string): string { + const ext = IMAGE_MIME_TO_EXT[mimeType.toLowerCase()] ?? 'bin'; + return `upload.${ext}`; +} + +function guessImageMimeTypeFromExt(filename: string): string | undefined { + const dot = filename.lastIndexOf('.'); + if (dot < 0) return undefined; + const ext = filename.slice(dot + 1).toLowerCase(); + return IMAGE_EXT_TO_MIME[ext]; +} diff --git a/packages/agent-core-v2/src/kosong/provider/providers/kimi/kimi.contrib.ts b/packages/agent-core-v2/src/kosong/provider/providers/kimi/kimi.contrib.ts index 7c609c92a4a..df92761e0ce 100644 --- a/packages/agent-core-v2/src/kosong/provider/providers/kimi/kimi.contrib.ts +++ b/packages/agent-core-v2/src/kosong/provider/providers/kimi/kimi.contrib.ts @@ -196,6 +196,8 @@ export const kimiOpenAITrait: ProtocolTrait = { }, uploadVideo: (input, options, ctx) => resolveFiles(ctx).uploadVideo(input, options), + + uploadImage: (input, options, ctx) => resolveFiles(ctx).uploadImage(input, options), }; export const kimiAnthropicTrait: ProtocolTrait = { diff --git a/packages/agent-core-v2/test/agent/contextProjector/projector-tool-exchanges.test.ts b/packages/agent-core-v2/test/agent/contextProjector/projector-tool-exchanges.test.ts index a10bdf11c7d..82d976528f8 100644 --- a/packages/agent-core-v2/test/agent/contextProjector/projector-tool-exchanges.test.ts +++ b/packages/agent-core-v2/test/agent/contextProjector/projector-tool-exchanges.test.ts @@ -6,6 +6,7 @@ import { TestInstantiationService } from '#/_base/di/test'; import { ILogService, type ILogger } from '#/_base/log/log'; import type { ContextMessage } from '#/agent/contextMemory/types'; import { IAgentContextProjectorService } from '#/agent/contextProjector/contextProjector'; +import { mergeMediaStripSnapshots } from '#/agent/contextProjector/mediaProjection'; import { AgentContextProjectorService } from '#/agent/contextProjector/contextProjectorService'; import { IAgentScopeContext, makeAgentScopeContext } from '#/agent/scopeContext/scopeContext'; import { IAgentStateService } from '#/agent/state/agentState'; @@ -834,5 +835,42 @@ describe('projector tool-exchange normalization', () => { .find((part) => part.type === 'image_url'); expect(image).toMatchObject({ imageUrl: { url, id: 'new-id' } }); }); + + it('unions the key sets of two snapshots so both images are stripped', () => { + const a = projector.captureMediaStripSnapshot([imageMessage('data:image/png;base64,AAA', 'a-id')]); + const b = projector.captureMediaStripSnapshot([imageMessage('data:image/png;base64,BBB', 'b-id')]); + const merged = mergeMediaStripSnapshots(a, b); + + const projected = projectStripped( + [imageMessage('data:image/png;base64,AAA', 'a-id'), imageMessage('data:image/png;base64,BBB', 'b-id')], + merged, + ); + + expect( + projected + .flatMap((message) => message.content) + .some((part) => part.type === 'image_url'), + ).toBe(false); + }); + + it('merged snapshot strips an image present in only one of the sources', () => { + const a = projector.captureMediaStripSnapshot([imageMessage('data:image/png;base64,AAA', 'a-id')]); + const b = projector.captureMediaStripSnapshot([imageMessage('data:image/png;base64,BBB', 'b-id')]); + const merged = mergeMediaStripSnapshots(a, b); + + const projectedA = projectStripped([imageMessage('data:image/png;base64,AAA', 'a-id')], merged); + const projectedB = projectStripped([imageMessage('data:image/png;base64,BBB', 'b-id')], merged); + + expect( + projectedA + .flatMap((message) => message.content) + .some((part) => part.type === 'image_url'), + ).toBe(false); + expect( + projectedB + .flatMap((message) => message.content) + .some((part) => part.type === 'image_url'), + ).toBe(false); + }); }); }); diff --git a/packages/agent-core-v2/test/agent/llmRequester/llmRequesterMediaStrip.test.ts b/packages/agent-core-v2/test/agent/llmRequester/llmRequesterMediaStrip.test.ts new file mode 100644 index 00000000000..c7344a58634 --- /dev/null +++ b/packages/agent-core-v2/test/agent/llmRequester/llmRequesterMediaStrip.test.ts @@ -0,0 +1,25 @@ +import { describe, expect, it } from 'vitest'; + +import { IAgentScopeContext } from '#/agent/scopeContext/scopeContext'; +import { llmRequesterMediaStrippedTurnsKey } from '#/agent/llmRequester/llmRequesterService'; +import { MediaStripped } from '#/agent/llmRequester/llmRequestOps'; +import { testAgent } from '../../harness'; + +describe('llmRequester media-strip durability', () => { + it('reconstructs the stripped-media key set from a persisted MediaStripped event', async () => { + const ctx = testAgent({ autoConfigure: false }); + const agentId = ctx.get(IAgentScopeContext).agentId; + try { + await ctx.dispatcher.dispatch( + new MediaStripped({ agentId, keys: ['media-key-a', 'media-key-b'] }), + ); + await ctx.restorePersisted(); + expect(ctx.agentState.get(llmRequesterMediaStrippedTurnsKey)).toEqual([ + 'media-key-a', + 'media-key-b', + ]); + } finally { + await ctx.dispose(); + } + }); +}); diff --git a/packages/agent-core-v2/test/agent/llmRequester/llmRequesterService.test.ts b/packages/agent-core-v2/test/agent/llmRequester/llmRequesterService.test.ts index 967a843cbb6..f260522fac6 100644 --- a/packages/agent-core-v2/test/agent/llmRequester/llmRequesterService.test.ts +++ b/packages/agent-core-v2/test/agent/llmRequester/llmRequesterService.test.ts @@ -9,9 +9,9 @@ import { IAgentContextMemoryService } from '#/agent/contextMemory/contextMemory' import type { ContextMessage } from '#/agent/contextMemory/types'; import { IAgentContextProjectorService, - type MediaStripSnapshot, type ProjectionPolicy, } from '#/agent/contextProjector/contextProjector'; +import { mediaStripSnapshotFromKeys } from '#/agent/contextProjector/mediaProjection'; import { AgentContextProjectorService } from '#/agent/contextProjector/contextProjectorService'; import { AgentLLMRequesterService, KIMI_CODE_INFINITE_RETRY_ENV } from '#/agent/llmRequester/llmRequesterService'; import { IAgentLLMRequesterService } from '#/agent/llmRequester/llmRequester'; @@ -219,7 +219,7 @@ function createService( shapeTools: (entries) => entries, shapeHistory: (messages) => messages, }; - const testSnapshot = Object.freeze({}) as MediaStripSnapshot; + const testSnapshot = mediaStripSnapshotFromKeys(['stub-rejected-media']); const events: Event2[] = []; const eventBus: IEventBus = { _serviceBrand: undefined, @@ -507,6 +507,45 @@ describe('AgentLLMRequesterService media-stripped resend', () => { expect(projection.calls).toEqual(['normal', 'stripped', 'stripped']); }); + it('keeps an image rejected in an earlier turn stripped on a later turn (sticky snapshot)', async () => { + const calls = { value: 0 }; + const capturedInputs: ModelRequestInput[] = []; + const requester = createRequester(calls); + const poisonedImage: Message = { + role: 'user', + content: [{ type: 'image_url', imageUrl: { url: 'data:image/avif;base64,QUJD', id: 'poisoned-id' } }], + toolCalls: [], + }; + requester.request = async function* (input) { + calls.value += 1; + capturedInputs.push(input); + const hasImage = input.messages.some((message) => + message.content.some((part) => part.type === 'image_url'), + ); + if (hasImage) throw IMAGE_FORMAT_400; + yield { + type: 'finish', + message: { role: 'assistant', content: [{ type: 'text', text: 'ok' }], toolCalls: [] }, + providerFinishReason: 'completed', + rawFinishReason: 'stop', + id: 'resp-1', + } satisfies ModelRequestEvent; + }; + const { service } = createService(requester, undefined, { + contextMessages: [poisonedImage], + }); + + await service.request({ messages: [poisonedImage], source: { type: 'turn', turnId: 1, step: 1 } }); + await service.request({ messages: [poisonedImage], source: { type: 'turn', turnId: 2, step: 1 } }); + + expect(calls.value).toBe(3); + const lastVisible = capturedInputs + .at(-1) + ?.messages.flatMap((message) => message.content) + .filter((part) => part.type === 'image_url'); + expect(lastVisible).toHaveLength(0); + }); + it('does not resend for an unrelated 400', async () => { const calls = { value: 0 }; const projection = recordProjectionCalls(); diff --git a/packages/agent-core-v2/test/agent/media/mediaResolver.test.ts b/packages/agent-core-v2/test/agent/media/mediaResolver.test.ts index 64bcf454d31..731479f3ee8 100644 --- a/packages/agent-core-v2/test/agent/media/mediaResolver.test.ts +++ b/packages/agent-core-v2/test/agent/media/mediaResolver.test.ts @@ -13,6 +13,7 @@ import { } from '#/_base/di/scope'; import { LifecycleScope } from '#/app/scopes'; import { createScopedTestHost, createServices, stubPair } from '#/_base/di/test'; +import { MEDIA_STRIPPED_PLACEHOLDERS } from '#/agent/contextProjector/mediaProjection'; import { buildKimiFileUrl } from '#/agent/media/kimiFileUrl'; import { IAgentMediaResolverService } from '#/agent/media/mediaResolver'; import { AgentMediaResolverService } from '#/agent/media/mediaResolverService'; @@ -156,9 +157,12 @@ async function plantCanonical(fileId: string, ext: string, bytes: Buffer): Promi function requester(opts: { videoIn?: boolean; imageIn?: boolean; + audioIn?: boolean; + imageFileApi?: boolean; protocol?: Protocol; providerType?: string; uploadVideo?: ModelRequester['uploadVideo']; + uploadImage?: ModelRequester['uploadImage']; }): ModelRequester { return { model: { @@ -170,6 +174,8 @@ function requester(opts: { capabilities: { video_in: opts.videoIn ?? true, image_in: opts.imageIn ?? true, + audio_in: opts.audioIn ?? true, + image_file_api: opts.imageFileApi ?? false, } as unknown as ModelCapability, maxContextSize: 1000, alwaysThinking: false, @@ -181,6 +187,7 @@ function requester(opts: { throw new Error('unused'); }, uploadVideo: opts.uploadVideo, + uploadImage: opts.uploadImage, }; } @@ -550,6 +557,51 @@ describe('AgentMediaResolverService image strategy', () => { expect(out[0]!.content).toEqual([{ type: 'text', text: expected }]); }); + it('uploads a daemon-ref image as a file part when image_file_api is declared', async () => { + const upload = vi.fn(async (): Promise<{ type: 'file'; fileId: string }> => { + return { type: 'file', fileId: 'file-api-123' }; + }); + const res = resolver(new Map([[FILE_ID, { name: 'pic.png', bytes: PNG_BYTES }]])); + const message = imageMessage(buildKimiFileUrl(FILE_ID)); + + const out = await res.resolve( + [message], + requester({ imageFileApi: true, uploadImage: upload }), + ); + + expect(firstPart(out)).toEqual({ type: 'file', fileId: 'file-api-123' }); + expect(upload).toHaveBeenCalledWith( + { data: PNG_BYTES, mimeType: 'image/png', filename: 'pic.png' }, + undefined, + ); + }); + + it('inlines a daemon-ref image as base64 when image_file_api is not declared', async () => { + const upload = vi.fn(); + const res = resolver(new Map([[FILE_ID, { name: 'pic.png', bytes: PNG_BYTES }]])); + const message = imageMessage(buildKimiFileUrl(FILE_ID)); + + const out = await res.resolve([message], requester({ imageFileApi: false, uploadImage: upload })); + + expect(firstPart(out)).toEqual({ type: 'image_url', imageUrl: { url: PNG_DATA_URL } }); + expect(upload).not.toHaveBeenCalled(); + }); + + it('falls back to an inline base64 image when the upload fails', async () => { + const upload = vi.fn(async () => { + throw new Error('files endpoint unavailable'); + }); + const res = resolver(new Map([[FILE_ID, { name: 'pic.png', bytes: PNG_BYTES }]])); + const message = imageMessage(buildKimiFileUrl(FILE_ID)); + + const out = await res.resolve( + [message], + requester({ imageFileApi: true, uploadImage: upload }), + ); + + expect(firstPart(out)).toEqual({ type: 'image_url', imageUrl: { url: PNG_DATA_URL } }); + }); + it('memoizes an inlined image across resolves without re-reading the bytes', async () => { const files = new Map([[FILE_ID, { name: 'pic.png', bytes: PNG_BYTES }]]); const counting = countingFileService(files); @@ -666,6 +718,88 @@ describe('AgentMediaResolverService image strategy', () => { ); }); +describe('AgentMediaResolverService remote media capability gating', () => { + it('degrades a remote https image_url when the model lacks image_in', async () => { + const res = resolver(new Map()); + const message = imageMessage('https://example.com/pic.png'); + + const out = await res.resolve([message], requester({ imageIn: false })); + + expect(out[0]!.content).toEqual([ + { type: 'text', text: MEDIA_STRIPPED_PLACEHOLDERS.image_url }, + ]); + }); + + it('keeps a remote https image_url when the model supports image_in', async () => { + const res = resolver(new Map()); + const message = imageMessage('https://example.com/pic.png'); + + const out = await res.resolve([message], requester({})); + + expect(out[0]!.content).toEqual([ + { type: 'image_url', imageUrl: { url: 'https://example.com/pic.png' } }, + ]); + }); + + it('degrades a remote https video_url when the model lacks video_in', async () => { + const res = resolver(new Map()); + const message = videoMessage('https://example.com/clip.mp4'); + + const out = await res.resolve([message], requester({ videoIn: false })); + + expect(out[0]!.content).toEqual([ + { type: 'text', text: MEDIA_STRIPPED_PLACEHOLDERS.video_url }, + ]); + }); + + it('degrades a remote https audio_url when the model lacks audio_in', async () => { + const res = resolver(new Map()); + const message: Message = { + role: 'user', + content: [{ type: 'audio_url', audioUrl: { url: 'https://example.com/clip.mp3' } }], + toolCalls: [], + }; + + const out = await res.resolve([message], requester({ audioIn: false })); + + expect(out[0]!.content).toEqual([ + { type: 'text', text: MEDIA_STRIPPED_PLACEHOLDERS.audio_url }, + ]); + }); + + it('keeps non-media content and degrades only the unsupported remote part', async () => { + const res = resolver(new Map()); + const textPart: ContentPart = { type: 'text', text: 'hello' }; + const message: Message = { + role: 'user', + content: [textPart, { type: 'image_url', imageUrl: { url: 'https://example.com/pic.png' } }], + toolCalls: [], + }; + + const out = await res.resolve([message], requester({ imageIn: false })); + + expect(out[0]!.content).toEqual([ + textPart, + { type: 'text', text: MEDIA_STRIPPED_PLACEHOLDERS.image_url }, + ]); + }); + + it('degrades remote media alongside an inlined daemon image in the same message', async () => { + const res = resolver(new Map([[FILE_ID, { name: 'pic.png', bytes: PNG_BYTES }]])); + const message = imageMessage(buildKimiFileUrl(FILE_ID), { + type: 'image_url', + imageUrl: { url: 'https://example.com/pic.png' }, + }); + + const out = await res.resolve([message], requester({ imageIn: false })); + + expect(out[0]!.content).toEqual([ + { type: 'text', text: MEDIA_STRIPPED_PLACEHOLDERS.image_url }, + { type: 'text', text: IMAGE_UNAVAILABLE_TEXT }, + ]); + }); +}); + describe('AgentMediaResolverService session-canonical display path', () => { it.each([ { diff --git a/packages/agent-core-v2/test/app/llmProtocol/errors.test.ts b/packages/agent-core-v2/test/app/llmProtocol/errors.test.ts index a76bb83df72..2e96216f833 100644 --- a/packages/agent-core-v2/test/app/llmProtocol/errors.test.ts +++ b/packages/agent-core-v2/test/app/llmProtocol/errors.test.ts @@ -231,6 +231,32 @@ describe('isImageFormatError', () => { ).toBe(true); }); + it('matches DeepSeek image-rejection 400 messages', () => { + expect( + isImageFormatError(new APIStatusError(400, 'This model does not support image input')), + ).toBe(true); + expect( + isImageFormatError( + new APIStatusError(400, 'unknown variant `image_url`, expected one of `input_image`'), + ), + ).toBe(true); + expect( + isImageFormatError(new APIStatusError(400, 'Images are supported in user messages only')), + ).toBe(true); + expect( + isImageFormatError(new APIStatusError(400, 'Image in assistant message is not supported')), + ).toBe(true); + expect( + isImageFormatError(new APIStatusError(400, 'Image in system message is not supported')), + ).toBe(true); + expect( + isImageFormatError(new APIStatusError(400, 'Image in tool message is not supported')), + ).toBe(true); + expect( + isImageFormatError(new APIStatusError(400, 'You have uploaded an unsupported image')), + ).toBe(true); + }); + it('matches client-side image whitelist throws', () => { expect( isImageFormatError(new ChatProviderError('Unsupported media type for base64 image: image/avif')), @@ -291,6 +317,24 @@ describe('isImageFormatError', () => { expect(isImageFormatError(new APIStatusError(400, 'invalid media type'))).toBe(false); }); + it('does not classify a non-image media rejection or a non-400 status as an image error', () => { + expect( + isImageFormatError( + new APIStatusError( + 400, + "messages.0.content.1.audio.source.base64.media_type: Input should be 'audio/mp3'", + ), + ), + ).toBe(false); + expect( + isImageFormatError(new APIStatusError(400, 'This model does not support video input')), + ).toBe(false); + expect(isImageFormatError(new APIStatusError(429, 'Rate limit exceeded'))).toBe(false); + expect( + isImageFormatError(new APIStatusError(429, 'This model does not support image')), + ).toBe(false); + }); + it('is excluded from the transient-retry fallback so dedicated recovery fires first', () => { expect(isRetryableGenerateError(new ChatProviderError('transient blip'))).toBe(true); expect( diff --git a/packages/agent-core-v2/test/index.test.ts b/packages/agent-core-v2/test/index.test.ts index c02666aa935..5f1d8c332e9 100644 --- a/packages/agent-core-v2/test/index.test.ts +++ b/packages/agent-core-v2/test/index.test.ts @@ -88,6 +88,7 @@ const V2_RECORD_TYPES: ReadonlySet = new Set([ 'interaction.resolved', 'plan.revision', 'interruptionReminder.recorded', + 'llm.media_stripped', 'plugin.session_start', 'runtime.set_binding', 'turn.ended', diff --git a/packages/agent-core-v2/test/kosong/model/catalog.test.ts b/packages/agent-core-v2/test/kosong/model/catalog.test.ts index 42c9969c70c..fc8cd1ebee4 100644 --- a/packages/agent-core-v2/test/kosong/model/catalog.test.ts +++ b/packages/agent-core-v2/test/kosong/model/catalog.test.ts @@ -1356,3 +1356,48 @@ describe('ModelCatalog setDefaultModel', () => { } }); }); + +describe('image_file_api capability resolution', () => { + it('maps a declared image_file_api capability onto the resolved model', () => { + const { host, catalog } = createHost({ + providers: { + kimi: { type: 'kimi', apiKey: 'sk-test', baseUrl: 'https://api.example/v1' }, + }, + models: { + img: { + provider: 'kimi', + model: 'kimi-k2', + maxContextSize: 131072, + displayName: 'Kimi K2', + capabilities: ['image_file_api'], + }, + }, + }); + try { + expect(catalog.get('img')?.capabilities.image_file_api).toBe(true); + } finally { + host.dispose(); + } + }); + + it('defaults image_file_api to false when not declared', () => { + const { host, catalog } = createHost({ + providers: { + kimi: { type: 'kimi', apiKey: 'sk-test', baseUrl: 'https://api.example/v1' }, + }, + models: { + turbo: { + provider: 'kimi', + model: 'kimi-turbo', + maxContextSize: 32768, + displayName: 'Kimi Turbo', + }, + }, + }); + try { + expect(catalog.get('turbo')?.capabilities.image_file_api).not.toBe(true); + } finally { + host.dispose(); + } + }); +}); diff --git a/packages/agent-core-v2/test/kosong/provider/errors.test.ts b/packages/agent-core-v2/test/kosong/provider/errors.test.ts index 60e35dfda36..3c99f380524 100644 --- a/packages/agent-core-v2/test/kosong/provider/errors.test.ts +++ b/packages/agent-core-v2/test/kosong/provider/errors.test.ts @@ -15,7 +15,7 @@ import type { ProtocolAdapterConfig } from '#/kosong/protocol/protocol'; import { traitConvertError, type TraitContext } from '#/kosong/protocol/protocolTrait'; import { convertAnthropicError } from '#/kosong/provider/bases/anthropic/anthropic'; import { convertGoogleGenAIError } from '#/kosong/provider/bases/google-genai/google-genai'; -import { convertOpenAIError } from '#/kosong/provider/bases/openai/openai-common'; +import { convertContentPart, convertOpenAIError } from '#/kosong/provider/bases/openai/openai-common'; import { OpenAIResponsesStreamedMessage } from '#/kosong/provider/bases/openai/openai-responses'; import { composeOpenAIChatHooks } from '#/kosong/provider/bases/openai/openaiHooks'; import { kimiAnthropicTrait, kimiOpenAITrait } from '#/kosong/provider/providers/kimi/kimi.contrib'; @@ -420,3 +420,29 @@ describe('convertGoogleGenAIError RetryInfo recovery', () => { expect((error as APIStatusError).retryAfterMs).toBe(5000); }); }); + +describe('convertContentPart image_url wire shape', () => { + it('emits only url for an image part with an id, dropping id from the OpenAI image_url contract', () => { + expect( + convertContentPart({ + type: 'image_url', + imageUrl: { url: 'https://ex/img.png', id: 'img-1' }, + }), + ).toEqual({ type: 'image_url', image_url: { url: 'https://ex/img.png' } }); + }); + + it('emits only url for an image part without an id', () => { + expect( + convertContentPart({ type: 'image_url', imageUrl: { url: 'https://ex/img.png' } }), + ).toEqual({ type: 'image_url', image_url: { url: 'https://ex/img.png' } }); + }); +}); + +describe('convertContentPart file wire shape', () => { + it('emits a DeepSeek file content part from an uploaded file id', () => { + expect(convertContentPart({ type: 'file', fileId: 'file-api-abc' })).toEqual({ + type: 'file', + file_id: 'file-api-abc', + }); + }); +}); diff --git a/packages/agent-core-v2/test/kosong/provider/kimi.test.ts b/packages/agent-core-v2/test/kosong/provider/kimi.test.ts index 92e5fdb7b93..56c650e2e62 100644 --- a/packages/agent-core-v2/test/kosong/provider/kimi.test.ts +++ b/packages/agent-core-v2/test/kosong/provider/kimi.test.ts @@ -273,6 +273,7 @@ describe('trait objects are plain declarations', () => { 'extractUsage', 'preserveThinking', 'strictThinkingValidation', + 'uploadImage', 'uploadVideo', 'withMaxCompletionTokens', 'withThinking', @@ -315,3 +316,37 @@ describe('KimiFiles upload error conversion', () => { expect(isRetryableGenerateError(caught)).toBe(false); }); }); + +describe('KimiFiles.uploadImage', () => { + it('uploads an image with purpose user_data and returns a file part', async () => { + const create = vi.fn().mockResolvedValue({ id: 'file-api-123' }); + const files = new KimiFiles({ + baseUrl: 'https://api.example/v1', + clientFactory: () => ({ files: { create } }) as never, + }); + + const part = await files.uploadImage( + { data: Buffer.from([1, 2, 3]), mimeType: 'image/png', filename: 'pic.png' }, + { auth: { apiKey: 'request-token' } }, + ); + + expect(part).toEqual({ type: 'file', fileId: 'file-api-123' }); + expect(create).toHaveBeenCalledWith( + { file: expect.any(File) as never, purpose: 'user_data' as never }, + undefined, + ); + }); + + it('rejects a non-image mime type before hitting the files API', async () => { + const create = vi.fn(); + const files = new KimiFiles({ + baseUrl: 'https://api.example/v1', + clientFactory: () => ({ files: { create } }) as never, + }); + + await expect( + files.uploadImage({ data: Buffer.from([1, 2, 3]), mimeType: 'video/mp4' }), + ).rejects.toThrow(/image mime/i); + expect(create).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/agent-core-v2/test/state/builtinReplayableKeys.ts b/packages/agent-core-v2/test/state/builtinReplayableKeys.ts index ec6e1b26044..0ed14e5445a 100644 --- a/packages/agent-core-v2/test/state/builtinReplayableKeys.ts +++ b/packages/agent-core-v2/test/state/builtinReplayableKeys.ts @@ -5,6 +5,7 @@ import { staleGuardKey } from '#/features/staleGuard/staleGuardOps'; import { fullCompactionKey } from '#/agent/fullCompaction/compactionOps'; import { interruptionReminderKey } from '#/agent/interruptionReminder/interruptionReminderOps'; import { llmRequestTraceKey } from '#/agent/llmRequester/llmRequestOps'; +import { llmRequesterMediaStrippedTurnsKey } from '#/agent/llmRequester/llmRequesterService'; import { turnKey } from '#/agent/loop/turnOps'; import { mcpDiscoveryKey } from '#/agent/mcp/mcpDiscoveryOps'; import { @@ -30,6 +31,7 @@ export const BUILTIN_REPLAYABLE_STATE_KEYS: readonly ReplayableStateKey[] = fullCompactionKey, interruptionReminderKey, llmRequestTraceKey, + llmRequesterMediaStrippedTurnsKey, turnKey, mcpDiscoveryKey, permissionModeKey, diff --git a/packages/agent-core/src/agent/index.ts b/packages/agent-core/src/agent/index.ts index 14d1620fd8d..9367bc799e0 100644 --- a/packages/agent-core/src/agent/index.ts +++ b/packages/agent-core/src/agent/index.ts @@ -112,6 +112,16 @@ export interface AgentOptions { readonly replay?: ReplayBuilderOptions; readonly additionalDirs?: readonly string[]; readonly systemPromptContextProvider?: (() => Promise) | undefined; + /** + * Content-identity digests of media parts the provider rejected, persisted in + * the session state so a resumed session keeps stripping them. + */ + readonly initialStrippedMediaKeys?: readonly string[]; + /** + * Invoked whenever the turn flow's stripped-media accumulator grows, so the + * session can persist the set back into its state. + */ + readonly onStrippedMediaKeysChange?: ((keys: readonly string[]) => void) | undefined; } export class Agent { @@ -191,6 +201,8 @@ export class Agent { readonly knownEfforts: string | undefined; }> = []; private readonly systemPromptContextProvider?: (() => Promise) | undefined; + readonly initialStrippedMediaKeys?: readonly string[]; + readonly onStrippedMediaKeysChange?: ((keys: readonly string[]) => void) | undefined; constructor(options: AgentOptions) { this.type = options.type ?? 'main'; @@ -198,6 +210,8 @@ export class Agent { this.kimiConfig = options.config; this.homedir = options.homedir; this.mediaOriginalsDir = options.mediaOriginalsDir; + this.initialStrippedMediaKeys = options.initialStrippedMediaKeys; + this.onStrippedMediaKeysChange = options.onStrippedMediaKeysChange; this.rpc = options.rpc; this.toolServices = options.toolServices; this.pluginSessionStarts = options.pluginSessionStarts ?? []; diff --git a/packages/agent-core/src/agent/tool/index.ts b/packages/agent-core/src/agent/tool/index.ts index 1e0b6b6d3fb..46dc12b0e4c 100644 --- a/packages/agent-core/src/agent/tool/index.ts +++ b/packages/agent-core/src/agent/tool/index.ts @@ -780,6 +780,7 @@ export class ToolManager { background, } = this.agent; const videoUploader = this.createVideoUploader(provider); + const imageUploader = this.createImageUploader(provider); const workspace = extendWorkspaceWithSkillRoots( { workspaceDir: cwd, @@ -814,6 +815,7 @@ export class ToolManager { videoUploader, this.agent.telemetry, this.agent.imageLimits, + imageUploader, ), new b.EnterPlanModeTool(this.agent), new b.ExitPlanModeTool(this.agent), @@ -897,6 +899,62 @@ export class ToolManager { return this.createVideoUploader(this.agent.config.provider); } + /** + * Uploader bound to the agent's current provider, for images that the model + * reads through ReadMediaFile and the provider can reference by file id. + * `undefined` when no model is bound or the provider has no image upload + * channel. + */ + imageUploader(): b.ImageUploader | undefined { + if (!this.agent.config.hasProvider) return undefined; + return this.createImageUploader(this.agent.config.provider); + } + + private createImageUploader(provider: ChatProvider): b.ImageUploader | undefined { + const uploadImage = provider.uploadImage?.bind(provider); + if (uploadImage === undefined) return undefined; + + const modelAlias = this.agent.config.modelAlias!; + const withAuth = this.agent.modelProvider?.resolveAuth?.(modelAlias, { + log: this.agent.log, + }); + const baseProps = this.videoUploadTelemetryProps(modelAlias); + const upload = + withAuth === undefined + ? (input: b.ImageUploadInput, signal?: AbortSignal) => uploadImage(input, { signal }) + : (input: b.ImageUploadInput, signal?: AbortSignal) => + withAuth((auth) => uploadImage(input, { auth, signal })); + + return async (input, options) => { + const startedAt = Date.now(); + const base = { + ...baseProps, + mime_type: input.mimeType, + size_bytes: input.data.length, + }; + const track = (props: Record): void => { + try { + this.agent.telemetry.track('image_upload', props); + } catch { + // Telemetry must never affect the upload outcome. + } + }; + try { + const part = await upload(input, options?.signal); + track({ ...base, outcome: 'success', duration_ms: Date.now() - startedAt }); + return part; + } catch (error) { + track({ + ...base, + outcome: 'error', + duration_ms: Date.now() - startedAt, + error_type: error instanceof Error ? error.name : 'Unknown', + }); + throw error; + } + }; + } + private createVideoUploader(provider: ChatProvider): b.VideoUploader | undefined { const uploadVideo = provider.uploadVideo?.bind(provider); if (uploadVideo === undefined) return undefined; diff --git a/packages/agent-core/src/agent/turn/index.ts b/packages/agent-core/src/agent/turn/index.ts index 107a5d3460a..adeed94fefc 100644 --- a/packages/agent-core/src/agent/turn/index.ts +++ b/packages/agent-core/src/agent/turn/index.ts @@ -43,7 +43,6 @@ import { USER_PROMPT_ORIGIN, type PromptOrigin } from '../context'; import { captureMediaStripSnapshot, stripMediaPartsBySnapshot, - type MediaStripSnapshot, } from '../context/projector'; import { renderUserPromptHookBlockResult, renderUserPromptHookResult } from '../../session/hooks'; import { canonicalTelemetryArgs, isPlainRecord } from './canonical-args'; @@ -154,8 +153,35 @@ export class TurnFlow { private activeRequestTrace: LLMRequestTrace | undefined; private latestTraceId: string | undefined; private currentStep = 0; + private readonly mediaStripAccumulator: Set = new Set(); - constructor(protected readonly agent: Agent) {} + constructor(protected readonly agent: Agent) { + for (const key of agent.initialStrippedMediaKeys ?? []) { + this.mediaStripAccumulator.add(key); + } + } + + /** Content-identity digests the provider rejected and that must stay stripped. */ + get mediaStripKeys(): readonly string[] { + return [...this.mediaStripAccumulator]; + } + + /** + * Merges media-strip digests into the accumulator and notifies the persistence + * hook only when the set actually grows. + */ + addMediaStripKeys(keys: Iterable): void { + let changed = false; + for (const key of keys) { + if (!this.mediaStripAccumulator.has(key)) { + this.mediaStripAccumulator.add(key); + changed = true; + } + } + if (changed) { + this.agent.onStrippedMediaKeysChange?.(this.mediaStripKeys); + } + } /** Best-effort agent id (main / generated id) derived from the agent homedir. */ private get agentId(): string { @@ -822,11 +848,19 @@ export class TurnFlow { // appended only when the loadable set actually changed, so quiet turns // keep the prompt cache fully warm. this.agent.injection.injectToolsDiff(); - let mediaStripSnapshot: MediaStripSnapshot | undefined; + let mediaStripMergedThisTurn = false; + const buildMessages = (): Message[] => { + const messages = this.agent.context.messages; + if (this.mediaStripAccumulator.size === 0) return messages; + return stripMediaPartsBySnapshot(messages, this.mediaStripAccumulator); + }; const buildMessagesMediaStripped = (): Message[] => { const messages = this.agent.context.messages; - mediaStripSnapshot ??= captureMediaStripSnapshot(messages); - return stripMediaPartsBySnapshot(messages, mediaStripSnapshot); + if (!mediaStripMergedThisTurn) { + this.addMediaStripKeys(captureMediaStripSnapshot(messages)); + mediaStripMergedThisTurn = true; + } + return stripMediaPartsBySnapshot(messages, this.mediaStripAccumulator); }; while (true) { signal.throwIfAborted(); @@ -840,7 +874,7 @@ export class TurnFlow { turnId: String(turnId), signal, llm: this.agent.llm, - buildMessages: () => this.agent.context.messages, + buildMessages, buildMessagesStrict: () => this.agent.context.strictMessages, buildMessagesMediaDegraded: () => this.agent.context.mediaDegradedMessages, buildMessagesMediaStripped, diff --git a/packages/agent-core/src/config/env-model.ts b/packages/agent-core/src/config/env-model.ts index 2933da52c6d..3b9a0a6a336 100644 --- a/packages/agent-core/src/config/env-model.ts +++ b/packages/agent-core/src/config/env-model.ts @@ -24,7 +24,7 @@ const DEFAULT_BASE_URL: Partial> = { /** Default context window (256K) used when KIMI_MODEL_MAX_CONTEXT_SIZE is unset. */ const DEFAULT_MAX_CONTEXT_SIZE = 262144; -/** Default capabilities when KIMI_MODEL_CAPABILITIES is unset (kimi models support both). */ +/** Default capabilities for a kimi provider when KIMI_MODEL_CAPABILITIES is unset (kimi models support images and thinking). */ const DEFAULT_CAPABILITIES = ['image_in', 'thinking']; type Env = Readonly>; @@ -119,7 +119,9 @@ export function applyEnvModelConfig(config: KimiConfig, env: Env = process.env): maxOutputRaw !== undefined ? parsePositiveInt(maxOutputRaw, 'KIMI_MODEL_MAX_OUTPUT_SIZE') : undefined; - const capabilities = parseCapabilities(env['KIMI_MODEL_CAPABILITIES']) ?? DEFAULT_CAPABILITIES; + const capabilities = + parseCapabilities(env['KIMI_MODEL_CAPABILITIES']) ?? + (type === 'kimi' ? DEFAULT_CAPABILITIES : ['thinking']); const displayName = trimmed(env['KIMI_MODEL_DISPLAY_NAME']); const reasoningKey = trimmed(env['KIMI_MODEL_REASONING_KEY']); const adaptiveThinking = parseBooleanVar( diff --git a/packages/agent-core/src/mcp/output.ts b/packages/agent-core/src/mcp/output.ts index 27d68c21f13..dc2b3f01dcc 100644 --- a/packages/agent-core/src/mcp/output.ts +++ b/packages/agent-core/src/mcp/output.ts @@ -404,7 +404,7 @@ function applyBinaryPartCap(parts: readonly ContentPart[]): { const out: ContentPart[] = []; for (const part of parts) { - if (part.type === 'text' || part.type === 'think') { + if (part.type === 'text' || part.type === 'think' || part.type === 'file') { out.push(part); continue; } diff --git a/packages/agent-core/src/services/message/message.ts b/packages/agent-core/src/services/message/message.ts index 1d25ec626f1..1eb6bd0c70c 100644 --- a/packages/agent-core/src/services/message/message.ts +++ b/packages/agent-core/src/services/message/message.ts @@ -177,6 +177,11 @@ function mapContentPart(part: ContextMessage['content'][number]): MessageContent type: 'text', text: `[video:${part.videoUrl.url}]`, }; + case 'file': + return { + type: 'text', + text: `[file:${part.fileId}]`, + }; } } diff --git a/packages/agent-core/src/session/index.ts b/packages/agent-core/src/session/index.ts index 08a6667d4fd..fc769c6ee14 100644 --- a/packages/agent-core/src/session/index.ts +++ b/packages/agent-core/src/session/index.ts @@ -190,6 +190,11 @@ export interface SessionMeta { * these follow the session across close/resume without affecting any other * session opened in the same workspace. */ additionalDirs?: string[]; + /** + * Content-identity digests of media parts the provider rejected and that must + * stay stripped across a resumed session. + */ + strippedMediaKeys?: string[]; agents: Record; custom: Record; } @@ -255,6 +260,7 @@ export class Session { isCustomTitle: false, agents: {}, custom: {}, + strippedMediaKeys: [], }; private writeMetadataPromise = Promise.resolve(); private agentProfileSnapshot: AgentProfileCatalogSnapshot | undefined; @@ -1120,7 +1126,7 @@ export class Session { const text = await this.persistenceKaos.readText(this.metadataPath); const persisted = JSON.parse(text) as PersistedSessionState; const { agentProfileCatalog, ...metadata } = persisted; - this.metadata = metadata; + this.metadata = { ...metadata, strippedMediaKeys: metadata.strippedMediaKeys ?? [] }; if (agentProfileCatalog === undefined) { if (this.options.agents?.refreshPluginAgents === true) { this.agentProfileSnapshot = this.agentCatalog.snapshot(); @@ -1302,6 +1308,15 @@ export class Session { this.options.kimiHomeDir, { additionalDirs: agent.getAdditionalDirs() }, ), + initialStrippedMediaKeys: + type === 'main' ? (this.metadata.strippedMediaKeys ?? []) : undefined, + onStrippedMediaKeysChange: + type === 'main' + ? (keys) => { + this.metadata.strippedMediaKeys = [...keys]; + void this.writeMetadata(); + } + : undefined, }); return agent; } diff --git a/packages/agent-core/src/session/provider-manager.ts b/packages/agent-core/src/session/provider-manager.ts index fc6b901a6bb..35c6b1d9474 100644 --- a/packages/agent-core/src/session/provider-manager.ts +++ b/packages/agent-core/src/session/provider-manager.ts @@ -259,6 +259,7 @@ function resolveModelCapabilities( dynamically_loaded_tools: declared.has('dynamically_loaded_tools') || detected.dynamically_loaded_tools === true, + image_file_api: declared.has('image_file_api') || detected.image_file_api === true, }; } diff --git a/packages/agent-core/src/tools/builtin/file/read-media.ts b/packages/agent-core/src/tools/builtin/file/read-media.ts index fb2724b4695..017ede9b83a 100644 --- a/packages/agent-core/src/tools/builtin/file/read-media.ts +++ b/packages/agent-core/src/tools/builtin/file/read-media.ts @@ -27,6 +27,7 @@ import type { Kaos } from '@moonshot-ai/kaos'; import type { ContentPart, + ImageUploadInput, ModelCapability, VideoUploadInput as ProviderVideoUploadInput, } from '@moonshot-ai/kosong'; @@ -39,7 +40,12 @@ import type { TelemetryClient } from '../../../telemetry'; import { renderPrompt } from '../../../utils/render-prompt'; import { resolvePathAccessPath } from '../../policies/path-access'; import { MEDIA_SNIFF_BYTES, detectFileType, sniffImageDimensions } from '../../support/file-type'; -import { deliverVideoContent, type VideoUploader } from '../../support/video-delivery'; +import { + deliverImageContent, + deliverVideoContent, + type ImageUploader, + type VideoUploader, +} from '../../support/video-delivery'; import { IMAGE_BYTE_BUDGET, MAX_IMAGE_DECODE_BYTES, @@ -99,7 +105,7 @@ function buildFullResolutionLimitError(path: string, finalBytes: number): string export type VideoUploadInput = ProviderVideoUploadInput; -export type { VideoUploader }; +export type { ImageUploader, ImageUploadInput, VideoUploader }; // ── Input schema ───────────────────────────────────────────────────── @@ -265,6 +271,7 @@ export class ReadMediaFileTool implements BuiltinTool { private readonly videoUploader?: VideoUploader | undefined, telemetry?: TelemetryClient, imageLimits?: ImageLimits, + private readonly imageUploader?: ImageUploader | undefined, ) { if (!capabilities.image_in && !capabilities.video_in) { const skip = new Error('ReadMediaFile requires image_in or video_in capability'); @@ -297,6 +304,24 @@ export class ReadMediaFileTool implements BuiltinTool { ); } + private imageContentPart( + data: Uint8Array, + mimeType: string, + safePath: string, + ): Promise { + if (this.capabilities.image_file_api === true) { + return deliverImageContent( + { data, mimeType, filename: safePath.split(/[\\/]/).at(-1) }, + this.imageUploader, + ); + } + const base64 = Buffer.from(data).toString('base64'); + return Promise.resolve({ + type: 'image_url', + imageUrl: { url: `data:${mimeType};base64,${base64}` }, + }); + } + resolveExecution(args: ReadMediaFileInput): ToolExecution { const path = resolvePathAccessPath(args.path, { kaos: this.kaos, @@ -467,11 +492,11 @@ export class ReadMediaFileTool implements BuiltinTool { if (!outcome.ok) { return { isError: true, output: `Cannot read region from "${args.path}": ${outcome.error}` }; } - const base64 = Buffer.from(outcome.data).toString('base64'); - mediaPart = { - type: 'image_url', - imageUrl: { url: `data:${outcome.mimeType};base64,${base64}` }, - }; + mediaPart = await this.imageContentPart( + Buffer.from(outcome.data), + outcome.mimeType, + safePath, + ); delivery = { kind: 'crop', width: outcome.width, @@ -496,11 +521,7 @@ export class ReadMediaFileTool implements BuiltinTool { output: buildFullResolutionLimitError(args.path, data.length), }; } - const base64 = Buffer.from(data).toString('base64'); - mediaPart = { - type: 'image_url', - imageUrl: { url: `data:${fileType.mimeType};base64,${base64}` }, - }; + mediaPart = await this.imageContentPart(data, fileType.mimeType, safePath); delivery = { kind: 'full', width: dimensions?.width ?? 0, @@ -536,11 +557,7 @@ export class ReadMediaFileTool implements BuiltinTool { }), }; } - const base64 = Buffer.from(compressed.data).toString('base64'); - mediaPart = { - type: 'image_url', - imageUrl: { url: `data:${compressed.mimeType};base64,${base64}` }, - }; + mediaPart = await this.imageContentPart(compressed.data, compressed.mimeType, safePath); delivery = { kind: compressed.changed ? 'downsampled' : 'untouched', width: compressed.width, diff --git a/packages/agent-core/src/tools/support/video-delivery.ts b/packages/agent-core/src/tools/support/video-delivery.ts index aec6ac364e6..0b93402ced7 100644 --- a/packages/agent-core/src/tools/support/video-delivery.ts +++ b/packages/agent-core/src/tools/support/video-delivery.ts @@ -16,7 +16,13 @@ * ladder so their delivery and fallback semantics stay identical. */ -import type { ContentPart, VideoUploadInput, VideoURLPart } from '@moonshot-ai/kosong'; +import type { + ContentPart, + FilePart, + ImageUploadInput, + VideoUploadInput, + VideoURLPart, +} from '@moonshot-ai/kosong'; import { ErrorCodes } from '../../errors'; import { abortReason } from '../../utils/abort'; @@ -27,6 +33,12 @@ export type VideoUploader = ( options?: { signal?: AbortSignal }, ) => Promise; +/** Uploads a local image and returns the provider-issued `file` part. */ +export type ImageUploader = ( + input: ImageUploadInput, + options?: { signal?: AbortSignal }, +) => Promise; + /** * Auth rejections from the upload channel that must surface (they drive * credential refresh and a clear auth error). The auth layer wraps provider @@ -87,3 +99,27 @@ export async function deliverVideoContent( videoUrl: { url: `data:${input.mimeType};base64,${base64}` }, }; } + +/** + * Deliver an image through the provider's upload channel when available, + * falling back to an inline base64 part when the channel is missing or the + * upload fails for a non-auth reason. Mirrors {@link deliverVideoContent} so + * the image and video delivery semantics stay identical. + */ +export async function deliverImageContent( + input: ImageUploadInput, + uploader: ImageUploader | undefined, + signal?: AbortSignal, +): Promise { + if (uploader !== undefined) { + try { + return await (signal === undefined ? uploader(input) : uploader(input, { signal })); + } catch (error) { + if (signal?.aborted) throw abortReason(signal); + if (isAuthUploadError(error)) throw error; + } + } + if (signal?.aborted) throw abortReason(signal); + const base64 = Buffer.from(input.data).toString('base64'); + return { type: 'image_url', imageUrl: { url: `data:${input.mimeType};base64,${base64}` } }; +} diff --git a/packages/agent-core/src/utils/tokens.ts b/packages/agent-core/src/utils/tokens.ts index cb4a366627a..bcf551b8a00 100644 --- a/packages/agent-core/src/utils/tokens.ts +++ b/packages/agent-core/src/utils/tokens.ts @@ -110,6 +110,8 @@ export function estimateTokensForContentPart(part: ContentPart): number { case 'audio_url': case 'video_url': return MEDIA_TOKEN_ESTIMATE; + case 'file': + return estimateTokens(part.fileId); default: { // Exhaustiveness guard: a new ContentPart kind must declare its estimate // here rather than silently counting as 0 (the CMP-03 defect). diff --git a/packages/agent-core/test/agent/basic.test.ts b/packages/agent-core/test/agent/basic.test.ts index d806d3def50..447f512f353 100644 --- a/packages/agent-core/test/agent/basic.test.ts +++ b/packages/agent-core/test/agent/basic.test.ts @@ -506,3 +506,23 @@ describe('prompt-attached video resolution', () => { } }); }); + +describe('media-strip accumulator persistence', () => { + it('seeds the media-strip accumulator from persisted stripped-media keys', () => { + const ctx = testAgent({ initialStrippedMediaKeys: ['key-1', 'key-2'] }); + expect(ctx.agent.turn.mediaStripKeys).toEqual(['key-1', 'key-2']); + }); + + it('notifies the persistence hook when the media-strip accumulator grows', () => { + const onStrippedMediaKeysChange = vi.fn(); + const ctx = testAgent({ + initialStrippedMediaKeys: ['key-1'], + onStrippedMediaKeysChange, + }); + + ctx.agent.turn.addMediaStripKeys(['key-2']); + + expect(ctx.agent.turn.mediaStripKeys).toEqual(['key-1', 'key-2']); + expect(onStrippedMediaKeysChange).toHaveBeenCalledWith(['key-1', 'key-2']); + }); +}); diff --git a/packages/agent-core/test/agent/harness/agent.ts b/packages/agent-core/test/agent/harness/agent.ts index 06041c25999..c143ba03fbf 100644 --- a/packages/agent-core/test/agent/harness/agent.ts +++ b/packages/agent-core/test/agent/harness/agent.ts @@ -109,6 +109,8 @@ export interface TestAgentOptions { readonly telemetry?: TelemetryClient | undefined; readonly log?: Logger; readonly experimentalFlags?: AgentOptions['experimentalFlags']; + readonly initialStrippedMediaKeys?: AgentOptions['initialStrippedMediaKeys']; + readonly onStrippedMediaKeysChange?: AgentOptions['onStrippedMediaKeysChange']; } interface ConfigureOptions { @@ -197,6 +199,8 @@ export class AgentTestContext { telemetry: options.telemetry, log: options.log, experimentalFlags: options.experimentalFlags, + initialStrippedMediaKeys: options.initialStrippedMediaKeys, + onStrippedMediaKeysChange: options.onStrippedMediaKeysChange, }); if (options.goal !== undefined) { (this.agent as unknown as { goal: GoalMode }).goal = options.goal; diff --git a/packages/agent-core/test/agent/kosong-llm.test.ts b/packages/agent-core/test/agent/kosong-llm.test.ts index eee33880926..705a6ac816c 100644 --- a/packages/agent-core/test/agent/kosong-llm.test.ts +++ b/packages/agent-core/test/agent/kosong-llm.test.ts @@ -460,6 +460,25 @@ describe('downgradeUnsupportedMedia', () => { expect(message.content[0]).toEqual(videoPart); }); + it('gates a remote https image_url when the model lacks image_in', () => { + const capability: ModelCapability = { + ...makeCapability(1000), + video_in: true, + audio_in: true, + }; + const remoteImage = { + type: 'image_url', + imageUrl: { url: 'https://example.com/pic.png' }, + } as const; + const input = [mediaMessage([remoteImage])]; + + const out = downgradeUnsupportedMedia(input, capability); + + expect(out[0]?.content).toEqual([ + { type: 'text', text: '[image omitted: current model has no image input]' }, + ]); + }); + it('KosongLLM strips unsupported video from messages sent to generate', async () => { let captured: readonly Message[] | undefined; const generate: GenerateFn = async (_p, _s, _t, messages) => { diff --git a/packages/agent-core/test/agent/turn.test.ts b/packages/agent-core/test/agent/turn.test.ts index 9bbc7fe3abe..0677f769387 100644 --- a/packages/agent-core/test/agent/turn.test.ts +++ b/packages/agent-core/test/agent/turn.test.ts @@ -512,6 +512,73 @@ describe('Agent turn flow', () => { .filter((text) => text.includes('[image omitted for provider compatibility;')), ).toHaveLength(2); }); + + it('keeps a poisoned image stripped on the next turn (sticky snapshot)', async () => { + let attempts = 0; + const histories: Message[][] = []; + const generate: GenerateFn = async (_p, _s, _t, history) => { + attempts += 1; + histories.push(structuredClone(history)); + const hasImage = history.flatMap((m) => m.content).some((p) => p.type === 'image_url'); + if (hasImage) throw new APIStatusError(400, 'unsupported image format'); + return okResponse(); + }; + const ctx = testAgent({ generate }); + ctx.configure({ + provider: { type: 'kimi', apiKey: 'test-key', model: 'kimi-code' }, + modelCapabilities: IMAGE_CAPABLE, + }); + plantPoisonedImage(ctx); + + await ctx.rpc.prompt({ input: [{ type: 'text', text: 'first turn' }] }); + await ctx.untilTurnEnd(); + expect(attempts).toBe(2); + + await ctx.rpc.prompt({ input: [{ type: 'text', text: 'second turn' }] }); + await ctx.untilTurnEnd(); + + expect(attempts).toBe(3); + expect(histories[2]!.flatMap((m) => m.content).some((p) => p.type === 'image_url')).toBe(false); + }); + + it('sends a new image pasted after a strip while keeping the old image stripped', async () => { + const POISON_URL = 'data:image/avif;base64,QUJD'; + const NEW_URL = 'data:image/png;base64,TkVX'; + let attempts = 0; + const histories: Message[][] = []; + const generate: GenerateFn = async (_p, _s, _t, history) => { + attempts += 1; + histories.push(structuredClone(history)); + const hasPoison = history + .flatMap((m) => m.content) + .some((p) => p.type === 'image_url' && p.imageUrl.url === POISON_URL); + if (hasPoison) throw new APIStatusError(400, 'unsupported image format'); + return okResponse(); + }; + const ctx = testAgent({ generate }); + ctx.configure({ + provider: { type: 'kimi', apiKey: 'test-key', model: 'kimi-code' }, + modelCapabilities: IMAGE_CAPABLE, + }); + plantPoisonedImage(ctx); + + await ctx.rpc.prompt({ input: [{ type: 'text', text: 'first turn' }] }); + await ctx.untilTurnEnd(); + expect(attempts).toBe(2); + + await ctx.rpc.prompt({ + input: [{ type: 'text', text: 'second turn' }, { type: 'image_url', imageUrl: { url: NEW_URL, id: 'new-id' } }], + }); + await ctx.untilTurnEnd(); + + expect(attempts).toBe(3); + const urls = histories + .at(-1)! + .flatMap((m) => m.content) + .filter((p) => p.type === 'image_url') + .map((p) => (p.type === 'image_url' ? p.imageUrl.url : undefined)); + expect(urls).toEqual([NEW_URL]); + }); }); it('tracks turn_started and turn_interrupted telemetry', async () => { diff --git a/packages/agent-core/test/config/env-model.test.ts b/packages/agent-core/test/config/env-model.test.ts index b9ac80e127f..b748cbdbc8f 100644 --- a/packages/agent-core/test/config/env-model.test.ts +++ b/packages/agent-core/test/config/env-model.test.ts @@ -88,6 +88,13 @@ describe('applyEnvModelConfig', () => { expect(anthropic?.baseUrl).toBeUndefined(); }); + it('defaults non-kimi providers to thinking-only (no image_in)', () => { + expect( + apply({ ...MIN, KIMI_MODEL_PROVIDER_TYPE: 'openai' }) + .models?.[ENV_MODEL_ALIAS_KEY]?.capabilities, + ).toEqual(['thinking']); + }); + it('rejects unsupported provider types', () => { expectConfigInvalid(() => apply({ ...MIN, KIMI_MODEL_PROVIDER_TYPE: 'google-genai' }), diff --git a/packages/agent-core/test/session/init.test.ts b/packages/agent-core/test/session/init.test.ts index a215e9bb826..6f891e04752 100644 --- a/packages/agent-core/test/session/init.test.ts +++ b/packages/agent-core/test/session/init.test.ts @@ -1014,3 +1014,30 @@ function wrapReadTextWithError(inner: Kaos, cause: Error): Kaos { }, }); } + +describe('Session metadata stripped-media keys', () => { + it('persists and restores stripped-media keys across a state.json round-trip', async () => { + const homedir = await makeTempDir(); + const first = new Session({ + id: 'test-strip-keys', + kaos: testKaos.withCwd(await makeTempDir()), + homedir, + rpc: createSessionRpc([]), + providerManager: testProviderManager(), + }); + first.metadata.strippedMediaKeys = ['key-1', 'key-2']; + await first.writeMetadata(); + await first.close(); + + const resumed = new Session({ + id: 'test-strip-keys', + kaos: testKaos.withCwd(await makeTempDir()), + homedir, + rpc: createSessionRpc([]), + providerManager: testProviderManager(), + }); + await resumed.readMetadata(); + expect(resumed.metadata.strippedMediaKeys).toEqual(['key-1', 'key-2']); + await resumed.close(); + }); +}); diff --git a/packages/agent-core/test/session/provider-manager.test.ts b/packages/agent-core/test/session/provider-manager.test.ts index 108abd1bf0e..79641fd336c 100644 --- a/packages/agent-core/test/session/provider-manager.test.ts +++ b/packages/agent-core/test/session/provider-manager.test.ts @@ -28,6 +28,39 @@ function makeOAuthProviderManager(): ProviderManager { }); } +function makeImageFileApiProviderManager(): ProviderManager { + return new ProviderManager({ + config: { + providers: { + kimi: { type: 'kimi', model: 'kimi-k2' }, + }, + models: { + 'kimi-k2': { + provider: 'kimi', + model: 'kimi-k2', + maxContextSize: 262144, + capabilities: ['image_file_api'], + }, + }, + } as KimiConfig, + resolveOAuthTokenProvider: () => ({ getAccessToken: async () => 'tok' }), + }); +} + +describe('ProviderManager.resolveProviderConfig — image_file_api capability', () => { + it('maps a declared image_file_api capability onto the resolved model capabilities', () => { + const manager = makeImageFileApiProviderManager(); + expect(manager.resolveProviderConfig('kimi-k2').modelCapabilities.image_file_api).toBe(true); + }); + + it('defaults image_file_api to false when not declared', () => { + const manager = makeOAuthProviderManager(); + expect(manager.resolveProviderConfig('kimi-code/k3-256k').modelCapabilities.image_file_api).not.toBe( + true, + ); + }); +}); + describe('ProviderManager.resolveAuth — 401 refresh gate', () => { it('does not force a token refresh for a context-limit 401', async () => { const manager = makeOAuthProviderManager(); diff --git a/packages/agent-core/test/tools/read-media.test.ts b/packages/agent-core/test/tools/read-media.test.ts index 38ed3d4e217..9c007895194 100644 --- a/packages/agent-core/test/tools/read-media.test.ts +++ b/packages/agent-core/test/tools/read-media.test.ts @@ -15,6 +15,7 @@ import type { ExecutableToolResult } from '../../src/loop'; import { ReadMediaFileInputSchema, ReadMediaFileTool, + type ImageUploader, } from '../../src/tools/builtin/file/read-media'; import { MAX_IMAGE_DECODE_BYTES } from '../../src/tools/support/image-compress'; import { ImageLimits } from '../../src/tools/support/image-limits'; @@ -95,6 +96,7 @@ function makeReadMediaTool( readonly modelCapabilities?: ModelCapability | undefined; readonly telemetry?: TelemetryClient | undefined; readonly imageLimits?: ImageLimits | undefined; + readonly imageUploader?: ImageUploader | undefined; } = {}, ): ReadMediaFileTool { const kaos = createFakeKaos({ @@ -108,6 +110,7 @@ function makeReadMediaTool( undefined, input.telemetry, input.imageLimits, + input.imageUploader, ); } @@ -196,6 +199,76 @@ describe('ReadMediaFileTool', () => { expect(parts[2]).toEqual({ type: 'text', text: '' }); }); + it('uploads the image as a file part when image_file_api is declared', async () => { + const data = Buffer.concat([PNG_HEADER, Buffer.from('pngdata')]); + const uploader = vi + .fn() + .mockResolvedValue({ type: 'file', fileId: 'file-api-123' }); + const tool = makeReadMediaTool({ + stat: vi.fn().mockResolvedValue({ ...DEFAULT_STAT, stSize: data.length }), + readBytes: vi.fn().mockResolvedValue(data), + modelCapabilities: capabilities({ image_file_api: true }), + imageUploader: uploader, + }); + + const result = await executeTool(tool, { + turnId: 't1', + toolCallId: 'c1', + args: { path: '/workspace/sample.png' }, + signal, + }); + + const parts = outputParts(result); + expect(parts[1]).toEqual({ type: 'file', fileId: 'file-api-123' }); + expect(uploader).toHaveBeenCalledTimes(1); + }); + + it('inlines a base64 image when image_file_api is not declared', async () => { + const data = Buffer.concat([PNG_HEADER, Buffer.from('pngdata')]); + const uploader = vi.fn(); + const tool = makeReadMediaTool({ + stat: vi.fn().mockResolvedValue({ ...DEFAULT_STAT, stSize: data.length }), + readBytes: vi.fn().mockResolvedValue(data), + modelCapabilities: capabilities({ image_file_api: false }), + imageUploader: uploader, + }); + + const result = await executeTool(tool, { + turnId: 't1', + toolCallId: 'c1', + args: { path: '/workspace/sample.png' }, + signal, + }); + + const parts = outputParts(result); + expect(parts[1]).toMatchObject({ type: 'image_url' }); + expect(uploader).not.toHaveBeenCalled(); + }); + + it('falls back to a base64 image when the upload fails', async () => { + const data = Buffer.concat([PNG_HEADER, Buffer.from('pngdata')]); + const uploader = vi.fn().mockRejectedValue(new Error('files unavailable')); + const tool = makeReadMediaTool({ + stat: vi.fn().mockResolvedValue({ ...DEFAULT_STAT, stSize: data.length }), + readBytes: vi.fn().mockResolvedValue(data), + modelCapabilities: capabilities({ image_file_api: true }), + imageUploader: uploader, + }); + + const result = await executeTool(tool, { + turnId: 't1', + toolCallId: 'c1', + args: { path: '/workspace/sample.png' }, + signal, + }); + + const parts = outputParts(result); + expect(parts[1]).toMatchObject({ type: 'image_url' }); + expect((parts[1] as { imageUrl: { url: string } }).imageUrl.url).toBe( + `data:image/png;base64,${data.toString('base64')}`, + ); + }); + it('emits a summary with mime type and byte size for images', async () => { const data = Buffer.concat([PNG_HEADER, Buffer.from('pngdata')]); const tool = makeReadMediaTool({ diff --git a/packages/kosong/src/capability.ts b/packages/kosong/src/capability.ts index f2d82499086..8a89e62bd71 100644 --- a/packages/kosong/src/capability.ts +++ b/packages/kosong/src/capability.ts @@ -31,6 +31,7 @@ export interface ModelCapability { * message carrying `tools`. */ readonly dynamically_loaded_tools?: boolean; + readonly image_file_api?: boolean; } const UNKNOWN_CAPABILITY_MARKER = Symbol.for('moonshot-ai.kosong.UNKNOWN_CAPABILITY'); @@ -50,6 +51,7 @@ export const UNKNOWN_CAPABILITY: ModelCapability = Object.freeze( tool_use: false, max_context_tokens: 0, dynamically_loaded_tools: false, + image_file_api: false, }, UNKNOWN_CAPABILITY_MARKER, { value: true }, @@ -67,6 +69,7 @@ export function isUnknownCapability(capability: ModelCapability): boolean { !capability.audio_in && !capability.thinking && !capability.tool_use && + !capability.image_file_api && capability.dynamically_loaded_tools !== true && capability.max_context_tokens === 0 ); diff --git a/packages/kosong/src/errors.ts b/packages/kosong/src/errors.ts index dc589c236c2..90ede9c08ad 100644 --- a/packages/kosong/src/errors.ts +++ b/packages/kosong/src/errors.ts @@ -288,6 +288,12 @@ const IMAGE_FORMAT_STATUS_MESSAGE_PATTERNS = [ /unable to process (?:the |input )?image/, /failed to decode (?:the )?image/, /invalid image(?: data| type| format)?/, + // DeepSeek image-rejection 400s (image input unsupported on the selected model). + /this model does not support image/, + /unknown variant\s+\x60?image_url\x60?/, + /images are supported in user messages only/, + /image in (?:assistant|system|tool) message is not supported/, + /you have uploaded an unsupported image/, ] as const; // Anthropic `media_type` & Gemini `mime_type` enum violations name the field diff --git a/packages/kosong/src/index.ts b/packages/kosong/src/index.ts index cd0440637ba..65a34dd165b 100644 --- a/packages/kosong/src/index.ts +++ b/packages/kosong/src/index.ts @@ -12,6 +12,7 @@ export { export type { AudioURLPart, ContentPart, + FilePart, ImageURLPart, Message, Role, diff --git a/packages/kosong/src/message.ts b/packages/kosong/src/message.ts index 12db25c84ed..8d0aedd8b6e 100644 --- a/packages/kosong/src/message.ts +++ b/packages/kosong/src/message.ts @@ -28,6 +28,11 @@ export interface VideoURLPart { videoUrl: { url: string; id?: string | undefined }; } +export interface FilePart { + type: 'file'; + fileId: string; +} + /** * A single piece of content within a {@link Message}. * @@ -35,7 +40,7 @@ export interface VideoURLPart { * Providers convert these to their native content-block format during * {@link ChatProvider.generate}. */ -export type ContentPart = TextPart | ThinkPart | ImageURLPart | AudioURLPart | VideoURLPart; +export type ContentPart = TextPart | ThinkPart | ImageURLPart | AudioURLPart | VideoURLPart | FilePart; export interface ToolCall { type: 'function'; @@ -114,11 +119,16 @@ export interface Message { readonly tools?: readonly Tool[] | undefined; } -/** Check if a streamed part is a ContentPart (text, think, image_url, audio_url, video_url). */ +/** Check if a streamed part is a ContentPart (text, think, image_url, audio_url, video_url, file). */ export function isContentPart(part: StreamedMessagePart): part is ContentPart { const t = part.type; return ( - t === 'text' || t === 'think' || t === 'image_url' || t === 'audio_url' || t === 'video_url' + t === 'text' || + t === 'think' || + t === 'image_url' || + t === 'audio_url' || + t === 'video_url' || + t === 'file' ); } diff --git a/packages/kosong/src/provider.ts b/packages/kosong/src/provider.ts index 905b12c1534..7339f625e16 100644 --- a/packages/kosong/src/provider.ts +++ b/packages/kosong/src/provider.ts @@ -1,4 +1,4 @@ -import type { Message, StreamedMessagePart, VideoURLPart } from './message'; +import type { FilePart, Message, StreamedMessagePart, VideoURLPart } from './message'; import type { Tool } from './tool'; import type { TokenUsage } from './usage'; @@ -209,6 +209,16 @@ export interface VideoUploadInput { readonly filename?: string | undefined; } +/** + * In-memory image bytes for providers that require an uploaded file + * reference instead of an inline data URL. + */ +export interface ImageUploadInput { + readonly data: Uint8Array; + readonly mimeType: string; + readonly filename?: string | undefined; +} + /** * Unified interface for an LLM chat provider. * @@ -271,4 +281,6 @@ export interface ChatProvider { ): ChatProvider; /** Upload a video and return a content part that can be sent to this provider. */ uploadVideo?(input: string | VideoUploadInput, options?: GenerateOptions): Promise; + /** Upload an image and return a content part that can be sent to this provider. */ + uploadImage?(input: string | ImageUploadInput, options?: GenerateOptions): Promise; } diff --git a/packages/kosong/src/providers/kimi-files.ts b/packages/kosong/src/providers/kimi-files.ts index ef40938ee22..abef8396e61 100644 --- a/packages/kosong/src/providers/kimi-files.ts +++ b/packages/kosong/src/providers/kimi-files.ts @@ -3,8 +3,8 @@ import * as path from 'node:path'; import { Blob, File } from 'node:buffer'; import { ChatProviderError } from '#/errors'; -import type { VideoURLPart } from '#/message'; -import type { ProviderRequestAuth, VideoUploadInput } from '#/provider'; +import type { FilePart, VideoURLPart } from '#/message'; +import type { ImageUploadInput, ProviderRequestAuth, VideoUploadInput } from '#/provider'; import type OpenAI from 'openai'; import OpenAIClient from 'openai'; @@ -141,6 +141,65 @@ export class KimiFiles { }; } + /** + * Upload an image to Kimi/Moonshot for use in chat messages. + * + * Accepts either a local filesystem path or an in-memory + * {@link ImageUploadInput}. Returns a {@link FilePart} referencing the + * uploaded file by its Moonshot file id. + * + * @param input - Local path string or `{ data, mimeType }` object. + * @returns A `FilePart` whose `fileId` references the uploaded file. + * @throws {ChatProviderError} if the input is not an image or the upload + * fails. + */ + async uploadImage( + input: string | ImageUploadInput, + options?: KimiUploadOptions, + ): Promise { + let file: unknown; + + if (typeof input === 'string') { + if (!fs.existsSync(input)) { + throw new ChatProviderError(`Image file not found: ${input}`); + } + const filename = path.basename(input); + const mimeType = guessImageMimeTypeFromExt(filename); + if (mimeType === undefined || !mimeType.startsWith('image/')) { + throw new ChatProviderError( + `KimiFiles.uploadImage: file extension does not indicate an image type: ${filename}`, + ); + } + const data = await fs.promises.readFile(input); + const blob = new Blob([new Uint8Array(data)], { type: mimeType }); + file = new File([blob], filename, { type: mimeType }); + } else { + if (!input.mimeType.startsWith('image/')) { + throw new ChatProviderError(`Expected an image mime type, got ${input.mimeType}`); + } + const filename = input.filename ?? guessImageFilename(input.mimeType); + const bytes = input.data instanceof Uint8Array ? input.data : new Uint8Array(input.data); + const blob = new Blob([bytes], { type: input.mimeType }); + file = new File([blob], filename, { type: input.mimeType }); + } + + let uploaded: { id: string }; + try { + const client = this._createClient(options?.auth); + uploaded = (await client.files.create( + { + file: file as never, + purpose: 'user_data' as never, + }, + options?.signal ? { signal: options.signal } : undefined, + )) as unknown as { id: string }; + } catch (error: unknown) { + throw convertOpenAIError(error, classifyKimiQuotaError); + } + + return { type: 'file', fileId: uploaded.id }; + } + private _createClient(auth: ProviderRequestAuth | undefined): OpenAI { return resolveAuthBackedClient( { cachedClient: this._client, clientFactory: this._clientFactory }, @@ -191,3 +250,34 @@ function guessMimeTypeFromExt(filename: string): string | undefined { const ext = filename.slice(dot + 1).toLowerCase(); return EXT_TO_MIME[ext]; } + +/** + * Guess a filename for an upload from an image MIME type. + * Falls back to `upload.bin` for unknown types. + */ +function guessImageFilename(mimeType: string): string { + const ext = IMAGE_MIME_TO_EXT[mimeType.toLowerCase()] ?? 'bin'; + return `upload.${ext}`; +} + +const IMAGE_MIME_TO_EXT: Record = { + 'image/png': 'png', + 'image/jpeg': 'jpg', + 'image/webp': 'webp', + 'image/gif': 'gif', +}; + +const IMAGE_EXT_TO_MIME: Record = Object.fromEntries( + Object.entries(IMAGE_MIME_TO_EXT).map(([mime, ext]) => [ext, mime]), +); + +/** + * Guess a MIME type from a filename extension. Only recognises the image + * types listed in {@link IMAGE_MIME_TO_EXT}; returns `undefined` otherwise. + */ +function guessImageMimeTypeFromExt(filename: string): string | undefined { + const dot = filename.lastIndexOf('.'); + if (dot < 0) return undefined; + const ext = filename.slice(dot + 1).toLowerCase(); + return IMAGE_EXT_TO_MIME[ext]; +} diff --git a/packages/kosong/src/providers/kimi.ts b/packages/kosong/src/providers/kimi.ts index 4496a432d98..58d68219190 100644 --- a/packages/kosong/src/providers/kimi.ts +++ b/packages/kosong/src/providers/kimi.ts @@ -5,6 +5,7 @@ import type { ChatProvider, FinishReason, GenerateOptions, + ImageUploadInput, MaxCompletionTokensOptions, ProviderRequestAuth, ResponseFormat, @@ -473,6 +474,10 @@ export class KimiChatProvider implements ChatProvider { return this.files.uploadVideo(input, options); } + uploadImage(input: string | ImageUploadInput, options?: GenerateOptions) { + return this.files.uploadImage(input, options); + } + get thinkingEffort(): ThinkingEffort | null { const thinking = this._generationKwargs.extra_body?.thinking; if (thinking === undefined) return null; diff --git a/packages/kosong/src/providers/openai-common.ts b/packages/kosong/src/providers/openai-common.ts index 91d2e80f4ec..dac5bd33f0f 100644 --- a/packages/kosong/src/providers/openai-common.ts +++ b/packages/kosong/src/providers/openai-common.ts @@ -23,9 +23,10 @@ import { export interface OpenAIContentPart { type: string; text?: string | undefined; - image_url?: { url: string; id?: string | null } | undefined; + image_url?: { url: string } | undefined; audio_url?: { url: string; id?: string | null } | undefined; video_url?: { url: string; id?: string | null } | undefined; + file_id?: string | undefined; } /** @@ -42,10 +43,7 @@ export function convertContentPart(part: ContentPart): OpenAIContentPart | null case 'image_url': return { type: 'image_url', - image_url: - part.imageUrl.id === undefined - ? { url: part.imageUrl.url } - : { url: part.imageUrl.url, id: part.imageUrl.id }, + image_url: { url: part.imageUrl.url }, }; case 'audio_url': return { @@ -63,6 +61,8 @@ export function convertContentPart(part: ContentPart): OpenAIContentPart | null ? { url: part.videoUrl.url } : { url: part.videoUrl.url, id: part.videoUrl.id }, }; + case 'file': + return { type: 'file', file_id: part.fileId }; default: throw new Error(`Unknown content part type: ${(part as ContentPart).type}`); } diff --git a/packages/kosong/test/capability.test.ts b/packages/kosong/test/capability.test.ts index 93846103b06..fed75caf59e 100644 --- a/packages/kosong/test/capability.test.ts +++ b/packages/kosong/test/capability.test.ts @@ -16,6 +16,23 @@ describe('ModelCapability / UNKNOWN_CAPABILITY', () => { expect(UNKNOWN_CAPABILITY.tool_use).toBe(false); }); + it('UNKNOWN_CAPABILITY.image_file_api defaults to false', () => { + expect(UNKNOWN_CAPABILITY.image_file_api).toBe(false); + }); + + it('a capability declaring only image_file_api is not unknown', () => { + const cap: ModelCapability = { + image_in: false, + video_in: false, + audio_in: false, + thinking: false, + tool_use: false, + max_context_tokens: 0, + image_file_api: true, + }; + expect(isUnknownCapability(cap)).toBe(false); + }); + it('UNKNOWN_CAPABILITY.max_context_tokens is 0 (unknown)', () => { expect(UNKNOWN_CAPABILITY.max_context_tokens).toBe(0); }); diff --git a/packages/kosong/test/errors.test.ts b/packages/kosong/test/errors.test.ts index 5e20c764026..f93c2d7354f 100644 --- a/packages/kosong/test/errors.test.ts +++ b/packages/kosong/test/errors.test.ts @@ -611,6 +611,32 @@ describe('isImageFormatError', () => { ).toBe(true); }); + it('matches DeepSeek image-rejection 400 messages', () => { + expect( + isImageFormatError(new APIStatusError(400, 'This model does not support image input')), + ).toBe(true); + expect( + isImageFormatError( + new APIStatusError(400, 'unknown variant `image_url`, expected one of `input_image`'), + ), + ).toBe(true); + expect( + isImageFormatError(new APIStatusError(400, 'Images are supported in user messages only')), + ).toBe(true); + expect( + isImageFormatError(new APIStatusError(400, 'Image in assistant message is not supported')), + ).toBe(true); + expect( + isImageFormatError(new APIStatusError(400, 'Image in system message is not supported')), + ).toBe(true); + expect( + isImageFormatError(new APIStatusError(400, 'Image in tool message is not supported')), + ).toBe(true); + expect( + isImageFormatError(new APIStatusError(400, 'You have uploaded an unsupported image')), + ).toBe(true); + }); + it('matches kosong client-side image whitelist throws', () => { expect( isImageFormatError(new ChatProviderError('Unsupported media type for base64 image: image/avif')), @@ -681,6 +707,24 @@ describe('isImageFormatError', () => { expect(isImageFormatError(new APIStatusError(400, 'invalid media type'))).toBe(false); }); + it('does not classify a non-image media rejection or a non-400 status as an image error', () => { + expect( + isImageFormatError( + new APIStatusError( + 400, + "messages.0.content.1.audio.source.base64.media_type: Input should be 'audio/mp3'", + ), + ), + ).toBe(false); + expect( + isImageFormatError(new APIStatusError(400, 'This model does not support video input')), + ).toBe(false); + expect(isImageFormatError(new APIStatusError(429, 'Rate limit exceeded'))).toBe(false); + expect( + isImageFormatError(new APIStatusError(429, 'This model does not support image')), + ).toBe(false); + }); + it('is excluded from the transient-retry fallback so dedicated recovery fires first', () => { // A base ChatProviderError is normally retried as an unclassified // transient; image-format errors must not be, or the run would burn the diff --git a/packages/kosong/test/kimi-files.test.ts b/packages/kosong/test/kimi-files.test.ts index 27ebff9c107..93bad9b55be 100644 --- a/packages/kosong/test/kimi-files.test.ts +++ b/packages/kosong/test/kimi-files.test.ts @@ -32,6 +32,7 @@ describe('KimiFiles', () => { const files = provider.files; expect(files).toBeInstanceOf(KimiFiles); expect(typeof files.uploadVideo).toBe('function'); + expect(typeof files.uploadImage).toBe('function'); }); it('memoizes the files property', () => { @@ -230,4 +231,42 @@ describe('KimiFiles', () => { expect(isRetryableGenerateError(caught)).toBe(false); }); }); + + describe('uploadImage', () => { + it('uploads an image with purpose user_data and returns a file part', async () => { + const provider = createProvider(); + let captured: unknown; + provider.files['_client']!.files.create = vi.fn().mockImplementation((params: unknown) => { + captured = params; + return Promise.resolve({ + id: 'file-api-123', + object: 'file', + bytes: 3, + created_at: 1, + filename: 'pic.png', + purpose: 'user_data', + }); + }) as never; + + const part = await provider.files.uploadImage({ + data: Buffer.from([1, 2, 3]), + mimeType: 'image/png', + }); + + const call = captured as { file: File; purpose: string }; + expect(call.purpose).toBe('user_data'); + expect(call.file).toBeInstanceOf(File); + expect(part).toEqual({ type: 'file', fileId: 'file-api-123' }); + }); + + it('rejects a non-image mime type', async () => { + const provider = createProvider(); + await expect( + provider.files.uploadImage({ + data: Buffer.from([1, 2, 3]), + mimeType: 'video/mp4', + }), + ).rejects.toThrow(/image/i); + }); + }); }); diff --git a/packages/kosong/test/message.test.ts b/packages/kosong/test/message.test.ts index ac2b64f1c04..f0661b55ed1 100644 --- a/packages/kosong/test/message.test.ts +++ b/packages/kosong/test/message.test.ts @@ -246,6 +246,11 @@ describe('type guards', () => { expect(isContentPart(part)).toBe(true); }); + it('isContentPart returns true for file', () => { + const part: StreamedMessagePart = { type: 'file', fileId: 'file-api-1' }; + expect(isContentPart(part)).toBe(true); + }); + it('isContentPart returns false for tool call', () => { const part: StreamedMessagePart = { type: 'function', diff --git a/packages/kosong/test/openai-common-errors.test.ts b/packages/kosong/test/openai-common-errors.test.ts index 0b7b7ca898d..3761cf5a7ea 100644 --- a/packages/kosong/test/openai-common-errors.test.ts +++ b/packages/kosong/test/openai-common-errors.test.ts @@ -356,13 +356,13 @@ describe('convertContentPart', () => { ).toEqual({ type: 'image_url', image_url: { url: 'https://ex/img.png' } }); }); - it('converts ImageURLPart with id', () => { + it('drops id from ImageURLPart in the OpenAI image_url wire shape', () => { expect( convertContentPart({ type: 'image_url', imageUrl: { url: 'https://ex/img.png', id: 'img-1' }, }), - ).toEqual({ type: 'image_url', image_url: { url: 'https://ex/img.png', id: 'img-1' } }); + ).toEqual({ type: 'image_url', image_url: { url: 'https://ex/img.png' } }); }); it('converts AudioURLPart without id', () => { @@ -395,6 +395,13 @@ describe('convertContentPart', () => { ).toEqual({ type: 'video_url', video_url: { url: 'https://ex/v.mp4', id: 'v-1' } }); }); + it('converts FilePart to a DeepSeek file content block', () => { + expect(convertContentPart({ type: 'file', fileId: 'file-api-abc' })).toEqual({ + type: 'file', + file_id: 'file-api-abc', + }); + }); + it('throws on unknown content part type', () => { // Force an invalid type to exercise the defensive branch. const bogus = { type: 'bogus', text: 'x' } as unknown as ContentPart; diff --git a/packages/kosong/test/type-safety.test.ts b/packages/kosong/test/type-safety.test.ts index 5c3dec1a293..6e90d10c47e 100644 --- a/packages/kosong/test/type-safety.test.ts +++ b/packages/kosong/test/type-safety.test.ts @@ -1,5 +1,6 @@ import type { AudioURLPart, + FilePart, ImageURLPart, StreamedMessagePart, TextPart, @@ -26,6 +27,8 @@ function processPartSafely(part: StreamedMessagePart): string { return part.audioUrl.url; // AudioURLPart.audioUrl.url -> string case 'video_url': return part.videoUrl.url; // VideoURLPart.videoUrl.url -> string + case 'file': + return part.fileId; // FilePart.fileId -> string case 'function': return part.name; // ToolCall.name -> string case 'tool_call_part': @@ -111,6 +114,7 @@ describe('exhaustiveness check', () => { { type: 'image_url', imageUrl: { url: 'c' } }, { type: 'audio_url', audioUrl: { url: 'd' } }, { type: 'video_url', videoUrl: { url: 'e' } }, + { type: 'file', fileId: 'f' }, { type: 'function', id: 'f', name: 'g', arguments: null }, { type: 'tool_call_part', argumentsPart: 'h' }, ]; From f870b09c6030111f7c54ebf2d6c7041e99a604b7 Mon Sep 17 00:00:00 2001 From: Mark Buckaway Date: Wed, 2 Sep 2026 08:30:51 -0400 Subject: [PATCH 2/2] fix: resolve CI regressions from DeepSeek image support --- apps/kimi-code/src/tui/utils/message-replay.ts | 2 ++ apps/vscode/src/utils/session-context.ts | 1 + .../agent-core-v2/src/agent/media/mediaResolverService.ts | 5 ++--- packages/agent-core-v2/src/kosong/contract/errors.ts | 2 +- .../kap-server/src/services/messages/messageProjection.ts | 4 +++- packages/klient/test/e2e/invalid-input-matrix.test.ts | 3 +++ packages/kosong/src/errors.ts | 2 +- packages/node-sdk/test/v1-v2-parity.test.ts | 3 +++ 8 files changed, 16 insertions(+), 6 deletions(-) diff --git a/apps/kimi-code/src/tui/utils/message-replay.ts b/apps/kimi-code/src/tui/utils/message-replay.ts index 1ef5c544c41..9d1297a80a0 100644 --- a/apps/kimi-code/src/tui/utils/message-replay.ts +++ b/apps/kimi-code/src/tui/utils/message-replay.ts @@ -376,6 +376,8 @@ function contentPartToText(part: ContentPart): string { return mediaUrlPartToText('video', part.videoUrl.url); case 'audio_url': return mediaUrlPartToText('audio', part.audioUrl.url); + case 'file': + return `[file:${part.fileId}]`; } } diff --git a/apps/vscode/src/utils/session-context.ts b/apps/vscode/src/utils/session-context.ts index e4ff4ce5b63..b86eea10f90 100644 --- a/apps/vscode/src/utils/session-context.ts +++ b/apps/vscode/src/utils/session-context.ts @@ -196,6 +196,7 @@ function formatPartMarkdown(part: ContentPart): string { case "image_url": return "[image]"; case "audio_url": return "[audio]"; case "video_url": return "[video]"; + case "file": return "[file]"; } } diff --git a/packages/agent-core-v2/src/agent/media/mediaResolverService.ts b/packages/agent-core-v2/src/agent/media/mediaResolverService.ts index d548dd1dba8..94b639736c5 100644 --- a/packages/agent-core-v2/src/agent/media/mediaResolverService.ts +++ b/packages/agent-core-v2/src/agent/media/mediaResolverService.ts @@ -171,13 +171,12 @@ export class AgentMediaResolverService implements IAgentMediaResolverService { requester: ModelRequester, signal: AbortSignal | undefined, ): Promise { - const uploadImage = requester.uploadImage; - if (uploadImage === undefined) return undefined; try { - const uploaded = await uploadImage( + const uploaded = await requester.uploadImage?.( { data: source.bytes, mimeType, filename: source.filename }, signal === undefined ? undefined : { signal }, ); + if (uploaded === undefined) return undefined; return { type: 'file', fileId: uploaded.fileId }; } catch (error) { if (signal?.aborted) throw error; diff --git a/packages/agent-core-v2/src/kosong/contract/errors.ts b/packages/agent-core-v2/src/kosong/contract/errors.ts index dd32ecbc028..01dba0d470b 100644 --- a/packages/agent-core-v2/src/kosong/contract/errors.ts +++ b/packages/agent-core-v2/src/kosong/contract/errors.ts @@ -220,7 +220,7 @@ const IMAGE_FORMAT_STATUS_MESSAGE_PATTERNS = [ /failed to decode (?:the )?image/, /invalid image(?: data| type| format)?/, /this model does not support image/, - /unknown variant\s+\x60?image_url\x60?/, + /unknown variant\s+\u0060?image_url\u0060?/, /images are supported in user messages only/, /image in (?:assistant|system|tool) message is not supported/, /you have uploaded an unsupported image/, diff --git a/packages/kap-server/src/services/messages/messageProjection.ts b/packages/kap-server/src/services/messages/messageProjection.ts index 56f15fe6f0c..913e252ca3d 100644 --- a/packages/kap-server/src/services/messages/messageProjection.ts +++ b/packages/kap-server/src/services/messages/messageProjection.ts @@ -1,4 +1,4 @@ -import { daemonFileRefFromPart, parseDaemonFileUrl, type ContentPart, type ContextMessage } from '@moonshot-ai/agent-core-v2'; +import { BugIndicatingError, daemonFileRefFromPart, parseDaemonFileUrl, type ContentPart, type ContextMessage } from '@moonshot-ai/agent-core-v2'; import type { Message, MessageContent, MessageRole, ToolUseContent } from '../../protocol/message'; @@ -35,6 +35,8 @@ function mapContentPart(part: ContextMessage['content'][number]): MessageContent ? { type: 'video', source: { kind: 'session_media', file_id: ref.fileId } } : { type: 'video', source: { kind: 'url', url: part.videoUrl.url, id: part.videoUrl.id } }; } + case 'file': + throw new BugIndicatingError('file content part cannot be projected from stored context'); } } diff --git a/packages/klient/test/e2e/invalid-input-matrix.test.ts b/packages/klient/test/e2e/invalid-input-matrix.test.ts index 3a8cfceb67a..471a3dc3dd3 100644 --- a/packages/klient/test/e2e/invalid-input-matrix.test.ts +++ b/packages/klient/test/e2e/invalid-input-matrix.test.ts @@ -369,6 +369,7 @@ beforeAll(async () => { baseUrl: `${baseUrl}/v1`, auth: { method: 'api-key', apiKey: 'test-key' }, maxContextSize: 262_144, + capabilities: { video_in: true }, }); await klient.global.kosong.addProvider({ id: M_OPENAI_VISION, @@ -396,6 +397,7 @@ beforeAll(async () => { baseUrl: `${baseUrl}/v1`, auth: { method: 'api-key', apiKey: 'test-key' }, maxContextSize: 262_144, + capabilities: { video_in: true }, }); await klient.global.kosong.addProvider({ id: M_GOOGLE, @@ -404,6 +406,7 @@ beforeAll(async () => { baseUrl, auth: { method: 'api-key', apiKey: 'test-key' }, maxContextSize: 262_144, + capabilities: { video_in: true }, }); }, 60_000); diff --git a/packages/kosong/src/errors.ts b/packages/kosong/src/errors.ts index 90ede9c08ad..61accd6af3b 100644 --- a/packages/kosong/src/errors.ts +++ b/packages/kosong/src/errors.ts @@ -290,7 +290,7 @@ const IMAGE_FORMAT_STATUS_MESSAGE_PATTERNS = [ /invalid image(?: data| type| format)?/, // DeepSeek image-rejection 400s (image input unsupported on the selected model). /this model does not support image/, - /unknown variant\s+\x60?image_url\x60?/, + /unknown variant\s+\u0060?image_url\u0060?/, /images are supported in user messages only/, /image in (?:assistant|system|tool) message is not supported/, /you have uploaded an unsupported image/, diff --git a/packages/node-sdk/test/v1-v2-parity.test.ts b/packages/node-sdk/test/v1-v2-parity.test.ts index fdfaedb6eac..15eb3174d2b 100644 --- a/packages/node-sdk/test/v1-v2-parity.test.ts +++ b/packages/node-sdk/test/v1-v2-parity.test.ts @@ -398,6 +398,9 @@ function projectResumedSession(resumed: ResumedSessionSummary, home: HomePair): const metadata = projected['sessionMetadata'] as Record; delete metadata['createdAt']; delete metadata['updatedAt']; + // `strippedMediaKeys` is v1-only engine-internal state (the media resolver + // records which context keys it stripped); v2 has no equivalent field. + delete metadata['strippedMediaKeys']; // v1 defaults an untitled metadata document to 'New Session'; the v2 // mapping reports the same unset title as ''. if (metadata['title'] === 'New Session' || metadata['title'] === '') {