diff --git a/docs-site/src/content/docs/fr/reference/configuration/providers.md b/docs-site/src/content/docs/fr/reference/configuration/providers.md index edf41cccc0..9cc6e0f0a6 100644 --- a/docs-site/src/content/docs/fr/reference/configuration/providers.md +++ b/docs-site/src/content/docs/fr/reference/configuration/providers.md @@ -283,6 +283,13 @@ les paramètres de modèle propres à Cursor : Les variantes explicites envoient le modèle `default` de Cursor avec son paramètre `optimization`, ce qui préserve la sélection à chaque requête. Elles restent disponibles lorsque la découverte en direct omet `default`. +### Vision + +La vision native Cursor utilise `SelectedImage` (plafond JPEG souple + `blobIdWithData`) pour les modèles +qui voient les images nativement — Claude, Gemini, GPT, Kimi et Grok notamment — à partir des images +`data:` du tour actif uniquement. Auto, la famille Composer et GLM (`glm-5.2`, `glm-5.3`) restent +sur la liste curatée `noVisionModels` et passent par le sidecar de description d'images. + Les outils locaux pilotés par le serveur Cursor sont désactivés par défaut. Codex continue d'utiliser ses propres outils tels que `apply_patch` et `exec_command` avec sa propre politique d'approbation et de bac à sable : diff --git a/docs-site/src/content/docs/reference/configuration/providers.md b/docs-site/src/content/docs/reference/configuration/providers.md index 0b1b4d7627..9cc3ea4296 100644 --- a/docs-site/src/content/docs/reference/configuration/providers.md +++ b/docs-site/src/content/docs/reference/configuration/providers.md @@ -285,6 +285,13 @@ Cursor-specific model parameters: Explicit variants send Cursor's `default` model with its `optimization` parameter, preserving the selection on every request. They remain available when live discovery omits `default`. +### Vision + +Native Cursor vision uses `SelectedImage` (JPEG soft-cap + `blobIdWithData`) for models that can +see images natively — Claude, Gemini, GPT, Kimi, and Grok among them — using active-turn `data:` +images only. Auto, the Composer family, and GLM (`glm-5.2`, `glm-5.3`) stay on the curated `noVisionModels` +list and use the vision describe sidecar instead. + Cursor server-driven local tools are disabled by default. Codex continues using its own tools such as `apply_patch` and `exec_command` with its own approval and sandbox policy: diff --git a/src/adapters/cursor/discovery.ts b/src/adapters/cursor/discovery.ts index 888f1c1f79..99ea79bb08 100644 --- a/src/adapters/cursor/discovery.ts +++ b/src/adapters/cursor/discovery.ts @@ -103,6 +103,28 @@ export const CURSOR_ROUTER_MODEL_IDS = [ ...CURSOR_ROUTING_LEVELS.map(level => `${CURSOR_AUTO_MODEL_ID}-${level}`), ] as const; +/** + * Cursor models that cannot see images natively. OpenCodex routes them through the vision + * sidecar (the catalog still advertises image so Codex can attach). Evidence: + * - Composer family: Cursor staff — text-only; "Model does not support images" + * - Auto / router modes: Cursor docs omit Images for Auto Cost; staff — pick Claude/GPT for images + * - glm-5.2: Cursor docs omit Images; Z.ai GLM-5.2 is text-only (vision is GLM-5V) + * - glm-5.3: same family; seeded as text-only ahead of Cursor's lineup update + * + * Composer ids are enumerated explicitly — prefix wildcard matching is deliberately out of + * scope here; a live-discovered new Composer slug stays native-path until curated. Everyone + * else in the static seed (Claude, Gemini, GPT, Kimi, Grok) takes SelectedImage. Other + * live-discovered ids stay unclassified (native path) until curated. + */ +export const CURSOR_NO_VISION_MODELS = [ + ...CURSOR_ROUTER_MODEL_IDS, + "composer-1", + "composer-2.5", + "composer-2.5-fast", + "glm-5.2", + "glm-5.3", +] as const; + /** Wire id Cursor Connect expects for the auto-router (GetUsableModels returns `default`, not `auto`). */ export const CURSOR_AUTO_WIRE_MODEL_ID = "default"; diff --git a/src/adapters/cursor/images.ts b/src/adapters/cursor/images.ts new file mode 100644 index 0000000000..62b8513831 --- /dev/null +++ b/src/adapters/cursor/images.ts @@ -0,0 +1,699 @@ +import { randomUUID } from "node:crypto"; +import { create } from "@bufbuild/protobuf"; +import type { OcxContentPart, OcxImageContent, OcxMessage } from "../../types"; +import { + SelectedContextSchema, + SelectedImageSchema, + SelectedImage_BlobIdWithDataSchema, + SelectedImage_DimensionSchema, + type SelectedContext, + type SelectedImage, +} from "./gen/agent_pb"; +import { + storeCursorBlob, + type CursorBlobRequestScopeToken, +} from "./native-exec"; + +/** Final per-image byte cap after prep (OmniRoute / composer-api style). */ +export const MAX_CURSOR_IMAGE_BYTES = 1024 * 1024; + +/** + * Inbound decode/fetch bomb ceiling before JPEG prep. Large clipboard PNGs may exceed + * {@link MAX_CURSOR_IMAGE_BYTES} raw but shrink under the wire cap after re-encode. + */ +export const MAX_CURSOR_IMAGE_DECODE_BYTES = 16 * 1024 * 1024; + +/** + * Soft target for Cursor vision hydration. Live A/B: ~430 KiB PNG failed ("gray"/wrong UI) + * while the same visual as ~75 KiB JPEG succeeded. Prefer JPEG at or under this size. + */ +export const CURSOR_VISION_SOFT_MAX_BYTES = 100 * 1024; + +/** Soft target when the client requests `detail: original` or `high`. */ +export const CURSOR_VISION_SOFT_MAX_BYTES_HIGH = 256 * 1024; + +/** Longest edge after Cursor vision prep (Cursor staff guidance: ≤ 2000 px). */ +export const CURSOR_VISION_MAX_EDGE = 2000; + +/** + * Decode bomb: reject images whose sniffed longest edge exceeds this before Bun.Image. + * Separate from {@link CURSOR_VISION_MAX_EDGE} (output resize target). + */ +export const MAX_CURSOR_IMAGE_DECODE_EDGE = 8192; + +/** Decode bomb: reject images whose sniffed pixel count exceeds this before Bun.Image. */ +export const MAX_CURSOR_IMAGE_PIXELS = 25_000_000; + +const CURSOR_VISION_JPEG_QUALITIES_DEFAULT = [85, 70, 55, 40] as const; +const CURSOR_VISION_JPEG_QUALITIES_HIGH = [90, 80, 65, 50] as const; +/** Stop shrinking below this longest edge when chasing the soft byte cap. */ +const CURSOR_VISION_SOFT_MIN_EDGE = 256; +const CURSOR_VISION_SOFT_SHRINK = 0.85; + +const CURSOR_VISION_PASSTHROUGH_MIME = new Set([ + "image/jpeg", + "image/jpg", + "image/png", + "image/gif", + "image/webp", +]); + +/** Upper bound on images attached to one Cursor turn. */ +export const MAX_CURSOR_IMAGES = 12; + +/** Marker when an image cannot be prepared for the Cursor vision wire. */ +export const CURSOR_VISION_IMAGE_OMITTED = + "[image omitted: undecodable or unsupported type]"; + +/** Short text-only stand-in for an image part on replayed (historical) turns. Never includes bytes. */ +export const CURSOR_VISION_IMAGE_HISTORY_MARKER = "[image attached]"; + +export class CursorImageError extends Error { + readonly status: number; + + constructor(message: string, status = 400) { + super(message); + this.name = "CursorImageError"; + this.status = status; + } +} + +export interface ResolvedCursorImage { + data: Uint8Array; + mimeType: string; + uuid: string; + /** Codex/OpenAI image detail hint; affects JPEG soft-cap tier. */ + detail?: string; +} + +export type PrepareCursorImageOutcome = + | { status: "ready"; image: ResolvedCursorImage } + | { status: "omitted"; reason: string }; + +function isImagePart(part: OcxContentPart): part is OcxImageContent { + return part.type === "image"; +} + +function estimatedBase64DecodedBytes(payload: string): number { + return Math.floor((payload.length * 3) / 4); +} + +function isHighDetail(detail: string | undefined): boolean { + const normalized = (detail ?? "").trim().toLowerCase(); + return normalized === "original" || normalized === "high"; +} + +function softMaxBytesForDetail(detail: string | undefined): number { + return isHighDetail(detail) ? CURSOR_VISION_SOFT_MAX_BYTES_HIGH : CURSOR_VISION_SOFT_MAX_BYTES; +} + +function jpegQualitiesForDetail(detail: string | undefined): readonly number[] { + return isHighDetail(detail) ? CURSOR_VISION_JPEG_QUALITIES_HIGH : CURSOR_VISION_JPEG_QUALITIES_DEFAULT; +} + +export function decodeCursorImageDataUrl(url: string): { data: Uint8Array; mimeType: string } { + const comma = url.indexOf(","); + if (comma < 0) throw new CursorImageError("Image data URL is malformed."); + const header = url.slice(5, comma); + const payload = url.slice(comma + 1); + const isBase64 = /;base64/i.test(header); + const mimeType = (header.split(";")[0] || "").trim().toLowerCase() || "application/octet-stream"; + + if (!mimeType.startsWith("image/")) { + throw new CursorImageError("Image data URL must have an image/* media type."); + } + if (!isBase64) { + throw new CursorImageError("Image data URL must be base64-encoded."); + } + if (payload.length > MAX_CURSOR_IMAGE_DECODE_BYTES * 2) { + throw new CursorImageError("Image input is too large to process safely."); + } + + const normalized = payload.replace(/\s/g, ""); + if (normalized.length === 0) { + throw new CursorImageError("Image data URL contains invalid base64 data."); + } + // Reject lenient Buffer.from acceptances (wrong alphabet, bad padding, truncated groups). + if (normalized.length % 4 !== 0 || !/^[A-Za-z0-9+/]+={0,2}$/.test(normalized)) { + throw new CursorImageError("Image data URL contains invalid base64 data."); + } + if (estimatedBase64DecodedBytes(normalized) > MAX_CURSOR_IMAGE_DECODE_BYTES) { + throw new CursorImageError("Image input is too large to process safely."); + } + + let data: Uint8Array; + try { + data = Buffer.from(normalized, "base64"); + } catch { + throw new CursorImageError("Image data URL contains invalid base64 data."); + } + if (data.byteLength === 0) { + throw new CursorImageError("Image data URL contains invalid base64 data."); + } + // Round-trip guard: Node/Bun can silently drop trailing garbage. + if (Buffer.from(data).toString("base64").replace(/=+$/, "") !== normalized.replace(/=+$/, "")) { + throw new CursorImageError("Image data URL contains invalid base64 data."); + } + if (data.byteLength > MAX_CURSOR_IMAGE_DECODE_BYTES) { + throw new CursorImageError("Image input is too large to process safely."); + } + return { data, mimeType }; +} + +function throwIfImagePhaseAborted(signal?: AbortSignal): void { + if (!signal?.aborted) return; + if (signal.reason instanceof Error) throw signal.reason; + const err = new Error("Cursor image phase aborted"); + err.name = "AbortError"; + throw err; +} + +/** Magic-byte format sniff (independent of declared MIME). */ +export function sniffCursorImageFormat( + data: Uint8Array, +): "png" | "jpeg" | "gif" | "webp" | undefined { + if ( + data.byteLength >= 8 + && data[0] === 0x89 && data[1] === 0x50 && data[2] === 0x4e && data[3] === 0x47 + && data[4] === 0x0d && data[5] === 0x0a && data[6] === 0x1a && data[7] === 0x0a + ) { + return "png"; + } + if ( + data.byteLength >= 6 + && data[0] === 0x47 && data[1] === 0x49 && data[2] === 0x46 && data[3] === 0x38 + ) { + return "gif"; + } + if (data.byteLength >= 4 && data[0] === 0xff && data[1] === 0xd8) return "jpeg"; + if ( + data.byteLength >= 12 + && data[0] === 0x52 && data[1] === 0x49 && data[2] === 0x46 && data[3] === 0x46 + && data[8] === 0x57 && data[9] === 0x45 && data[10] === 0x42 && data[11] === 0x50 + ) { + return "webp"; + } + return undefined; +} + +/** Collect image URLs from one message's content parts, preserving order. */ +export function extractCursorImageUrls(content: string | readonly OcxContentPart[]): string[] { + return extractCursorImageParts(content).map(part => part.imageUrl); +} + +export interface CursorImagePartRef { + imageUrl: string; + detail?: string; +} + +/** Collect image parts (URL + optional detail) from one message's content. */ +export function extractCursorImageParts( + content: string | readonly OcxContentPart[], +): CursorImagePartRef[] { + if (typeof content === "string" || !Array.isArray(content)) return []; + const parts: CursorImagePartRef[] = []; + for (const part of content) { + if (isImagePart(part) && typeof part.imageUrl === "string" && part.imageUrl.length > 0) { + parts.push({ + imageUrl: part.imageUrl, + ...(typeof part.detail === "string" && part.detail.length > 0 ? { detail: part.detail } : {}), + }); + } + } + return parts; +} + +/** + * Resolve OpenCodex image parts (data: URLs only) into bytes for SelectedImage. + * Prep (JPEG soft-cap) runs before the 1 MiB wire cap so large clipboard PNGs can shrink. + * Unsupported / undecodable images are omitted (fail-closed). + */ +export async function resolveCursorImages( + imageUrls: readonly string[], + signal?: AbortSignal, + options?: { details?: readonly (string | undefined)[] }, +): Promise { + if (imageUrls.length > MAX_CURSOR_IMAGES) { + throw new CursorImageError(`Too many images in one request (max ${MAX_CURSOR_IMAGES}).`); + } + + const out: ResolvedCursorImage[] = []; + for (let i = 0; i < imageUrls.length; i++) { + throwIfImagePhaseAborted(signal); + const url = imageUrls[i]; + if (typeof url !== "string" || url.length === 0) { + // Soft-omit missing URLs rather than aborting a mixed turn. + continue; + } + // Remote URL fetching is deliberately out of scope here; https:// images are omitted. + if (!url.toLowerCase().startsWith("data:")) continue; + try { + const resolved = decodeCursorImageDataUrl(url); + if (resolved.data.byteLength === 0) continue; + const outcome = await prepareCursorImageForWire({ + data: resolved.data, + mimeType: resolved.mimeType, + uuid: randomUUID(), + ...(options?.details?.[i] ? { detail: options.details[i] } : {}), + }, signal); + if (outcome.status === "omitted") continue; + if (outcome.image.data.byteLength > MAX_CURSOR_IMAGE_BYTES) continue; + out.push(outcome.image); + } catch (err) { + if (signal?.aborted || (err instanceof Error && err.name === "AbortError")) throw err; + if (err instanceof CursorImageError) continue; + continue; + } + } + return out; +} + +export async function resolveCursorImageParts( + parts: readonly CursorImagePartRef[], + signal?: AbortSignal, +): Promise { + return resolveCursorImages( + parts.map(part => part.imageUrl), + signal, + { details: parts.map(part => part.detail) }, + ); +} + +/** Filename Cursor clients typically put on SelectedImage.path (shunt / agent parity). */ +export function cursorImageAttachmentPath(uuid: string, mimeType: string): string { + const normalized = mimeType.toLowerCase(); + const ext = normalized === "image/jpeg" || normalized === "image/jpg" ? "jpg" + : normalized === "image/gif" ? "gif" + : normalized === "image/webp" ? "webp" + : "png"; + return `attachment-${uuid}.${ext}`; +} + +/** + * Re-encode toward a JPEG under the soft vision cap when Bun can decode the payload. + * Unsupported MIME, oversize dimensions/pixels, or undecodable bytes are omitted (fail-closed). + * After the quality ladder, edges shrink iteratively until the soft byte cap is met + * (or the min edge floor is hit) so large clipboard PNGs do not leave >softMax JPEGs + * that Cursor vision hallucinates on. + */ +export async function prepareCursorImageForWire( + image: ResolvedCursorImage, + signal?: AbortSignal, +): Promise { + throwIfImagePhaseAborted(signal); + const mime = image.mimeType.toLowerCase(); + const softMax = softMaxBytesForDetail(image.detail); + const qualities = jpegQualitiesForDetail(image.detail); + const lowestQuality = qualities[qualities.length - 1]!; + + if (!CURSOR_VISION_PASSTHROUGH_MIME.has(mime)) { + return { status: "omitted", reason: CURSOR_VISION_IMAGE_OMITTED }; + } + + const format = sniffCursorImageFormat(image.data); + // Peek headers before Bun.Image so huge compressed bombs fail closed cheaply. + const sniffed = sniffCursorImageDimensions(image.data); + if (sniffed) { + const edge = Math.max(sniffed.width, sniffed.height); + const pixels = sniffed.width * sniffed.height; + if (edge > MAX_CURSOR_IMAGE_DECODE_EDGE || pixels > MAX_CURSOR_IMAGE_PIXELS) { + return { status: "omitted", reason: CURSOR_VISION_IMAGE_OMITTED }; + } + } + + const declaredJpeg = mime === "image/jpeg" || mime === "image/jpg"; + + try { + throwIfImagePhaseAborted(signal); + // metadata() decodes; reuse it as the Anthropic-style validate pass. + const meta = await new Bun.Image(image.data).metadata(); + + // Passthrough only after a successful decode, and only when declared MIME + // matches actual JPEG magic (never PNG-as-JPEG or SOF-only junk). + if (declaredJpeg && format === "jpeg" && image.data.byteLength <= softMax) { + return { status: "ready", image }; + } + + throwIfImagePhaseAborted(signal); + const width = typeof meta.width === "number" ? meta.width : 0; + const height = typeof meta.height === "number" ? meta.height : 0; + if (width > 0 && height > 0) { + const edge = Math.max(width, height); + if (edge > MAX_CURSOR_IMAGE_DECODE_EDGE || width * height > MAX_CURSOR_IMAGE_PIXELS) { + return { status: "omitted", reason: CURSOR_VISION_IMAGE_OMITTED }; + } + } + let targetW = width; + let targetH = height; + if (width > 0 && height > 0 && Math.max(width, height) > CURSOR_VISION_MAX_EDGE) { + const scale = CURSOR_VISION_MAX_EDGE / Math.max(width, height); + targetW = Math.max(1, Math.round(width * scale)); + targetH = Math.max(1, Math.round(height * scale)); + } + + const encodeAt = async (w: number, h: number, quality: number): Promise => { + throwIfImagePhaseAborted(signal); + let pipeline = new Bun.Image(image.data); + if (w > 0 && h > 0 && (w !== width || h !== height)) { + pipeline = pipeline.resize(w, h); + } + return new Uint8Array(await pipeline.jpeg({ quality }).bytes()); + }; + + let best: Uint8Array | undefined; + for (const quality of qualities) { + const encoded = await encodeAt(targetW, targetH, quality); + if (!best || encoded.byteLength < best.byteLength) best = encoded; + if (encoded.byteLength <= softMax) { + return { + status: "ready", + image: { ...image, data: encoded, mimeType: "image/jpeg" }, + }; + } + } + + // Quality ladder missed the soft cap — shrink edges until it fits or we hit the floor. + while ( + best + && best.byteLength > softMax + && targetW > 0 + && targetH > 0 + && Math.max(targetW, targetH) > CURSOR_VISION_SOFT_MIN_EDGE + ) { + throwIfImagePhaseAborted(signal); + const nextW = Math.max(1, Math.round(targetW * CURSOR_VISION_SOFT_SHRINK)); + const nextH = Math.max(1, Math.round(targetH * CURSOR_VISION_SOFT_SHRINK)); + if (Math.max(nextW, nextH) < CURSOR_VISION_SOFT_MIN_EDGE) { + const scale = CURSOR_VISION_SOFT_MIN_EDGE / Math.max(targetW, targetH); + targetW = Math.max(1, Math.round(targetW * scale)); + targetH = Math.max(1, Math.round(targetH * scale)); + } else { + targetW = nextW; + targetH = nextH; + } + const encoded = await encodeAt(targetW, targetH, lowestQuality); + if (!best || encoded.byteLength < best.byteLength) best = encoded; + if (encoded.byteLength <= softMax) { + return { + status: "ready", + image: { ...image, data: encoded, mimeType: "image/jpeg" }, + }; + } + if (Math.max(targetW, targetH) <= CURSOR_VISION_SOFT_MIN_EDGE) break; + } + + if (best) { + return { + status: "ready", + image: { ...image, data: best, mimeType: "image/jpeg" }, + }; + } + // Undeclared/mismatched magic with no encode result — omit rather than lie about MIME. + if (declaredJpeg && format !== "jpeg") { + return { status: "omitted", reason: CURSOR_VISION_IMAGE_OMITTED }; + } + return { status: "ready", image }; + } catch (err) { + if (signal?.aborted || (err instanceof Error && err.name === "AbortError")) throw err; + return { status: "omitted", reason: CURSOR_VISION_IMAGE_OMITTED }; + } +} + +/** + * Sniff PNG/JPEG/GIF/WebP dimensions from raw bytes when the header is present. + * Best-effort only — unknown formats return undefined (dimension is optional). + */ +export function sniffCursorImageDimensions( + data: Uint8Array, +): { width: number; height: number } | undefined { + // PNG: signature + IHDR chunk (width/height at bytes 16..23) + if ( + data.byteLength >= 24 + && data[0] === 0x89 && data[1] === 0x50 && data[2] === 0x4e && data[3] === 0x47 + && data[4] === 0x0d && data[5] === 0x0a && data[6] === 0x1a && data[7] === 0x0a + ) { + const width = ((data[16]! << 24) | (data[17]! << 16) | (data[18]! << 8) | data[19]!) >>> 0; + const height = ((data[20]! << 24) | (data[21]! << 16) | (data[22]! << 8) | data[23]!) >>> 0; + if (width > 0 && height > 0) return { width, height }; + } + // GIF: "GIF8" + width/height as little-endian u16 at bytes 6..9 + if ( + data.byteLength >= 10 + && data[0] === 0x47 && data[1] === 0x49 && data[2] === 0x46 && data[3] === 0x38 + ) { + const width = data[6]! | (data[7]! << 8); + const height = data[8]! | (data[9]! << 8); + if (width > 0 && height > 0) return { width, height }; + } + // WebP: RIFF....WEBP + VP8X / VP8 / VP8L (same layout as anthropic-image-guard). + if ( + data.byteLength >= 30 + && data[0] === 0x52 && data[1] === 0x49 && data[2] === 0x46 && data[3] === 0x46 + && data[8] === 0x57 && data[9] === 0x45 && data[10] === 0x42 && data[11] === 0x50 + ) { + const fourcc = String.fromCharCode(data[12]!, data[13]!, data[14]!, data[15]!); + if (fourcc === "VP8X") { + const width = 1 + (data[24]! | (data[25]! << 8) | (data[26]! << 16)); + const height = 1 + (data[27]! | (data[28]! << 8) | (data[29]! << 16)); + if (width > 0 && height > 0) return { width, height }; + } else if (fourcc === "VP8 ") { + if (data[23] === 0x9d && data[24] === 0x01 && data[25] === 0x2a) { + const width = (data[26]! | (data[27]! << 8)) & 0x3fff; + const height = (data[28]! | (data[29]! << 8)) & 0x3fff; + if (width > 0 && height > 0) return { width, height }; + } + } else if (fourcc === "VP8L" && data[20] === 0x2f) { + const raw = data[21]! | (data[22]! << 8) | (data[23]! << 16) | (data[24]! << 24); + const width = (raw & 0x3fff) + 1; + const height = ((raw >> 14) & 0x3fff) + 1; + if (width > 0 && height > 0) return { width, height }; + } + } + // JPEG: scan for SOF0/SOF2 marker with dimensions + if (data.byteLength >= 4 && data[0] === 0xff && data[1] === 0xd8) { + let offset = 2; + while (offset + 8 < data.byteLength) { + if (data[offset] !== 0xff) break; + const marker = data[offset + 1]!; + // Standalone markers (TEM, RSTn, SOI, EOI) carry no length payload. + if (marker === 0x01 || (marker >= 0xd0 && marker <= 0xd9)) { + offset += 2; + continue; + } + const length = (data[offset + 2]! << 8) | data[offset + 3]!; + // SOFn frame headers share the dimension layout. 0xc4/0xc8/0xcc are DHT/JPG/DAC, not SOF. + if (marker >= 0xc0 && marker <= 0xcf && marker !== 0xc4 && marker !== 0xc8 && marker !== 0xcc) { + const height = (data[offset + 5]! << 8) | data[offset + 6]!; + const width = (data[offset + 7]! << 8) | data[offset + 8]!; + if (width > 0 && height > 0) return { width, height }; + break; + } + if (length < 2) break; + offset += 2 + length; + } + } + return undefined; +} + +/** + * Build SelectedImage messages for the AgentService vision path: + * store bytes in the local KV map under sha256(blobId), and encode + * `blobIdWithData` so the server can populate its cache without relying solely + * on getBlobArgs timing. Also set `path` like native/shunt clients. + */ +export function buildSelectedImages( + images: readonly ResolvedCursorImage[], + requestScope?: CursorBlobRequestScopeToken, +): SelectedImage[] { + return images.map(image => { + const blobId = storeCursorBlob(image.data, requestScope); + const dims = sniffCursorImageDimensions(image.data); + return create(SelectedImageSchema, { + uuid: image.uuid, + path: cursorImageAttachmentPath(image.uuid, image.mimeType), + mimeType: image.mimeType, + ...(dims + ? { dimension: create(SelectedImage_DimensionSchema, dims) } + : {}), + dataOrBlobId: { + case: "blobIdWithData", + value: create(SelectedImage_BlobIdWithDataSchema, { + blobId, + data: image.data, + }), + }, + }); + }); +} + +/** + * Always send `UserMessage.selected_context`, even when empty — matches cursor-agent. + * When images are present, they are blobIdWithData refs backed by the request-scoped KV store. + */ +export function buildSelectedContext( + images: readonly ResolvedCursorImage[] = [], + requestScope?: CursorBlobRequestScopeToken, +): SelectedContext { + return create(SelectedContextSchema, { + selectedImages: buildSelectedImages(images, requestScope), + }); +} + +/** + * Resolve data: images for the active user/developer turn onto SelectedImage. + * Tool-result image promotion is intentionally out of scope in this slice. + */ +export async function resolveActiveCursorImages( + messages: readonly OcxMessage[] | undefined, + signal?: AbortSignal, + preparedImages?: readonly ResolvedCursorImage[], +): Promise { + if (!messages?.length) return []; + // Same window the prepare pass rewrote; a divergent rule would attach unprepared bytes. + const message = messages[cursorVisionPrepareStartIndex(messages)]; + if (!message || (message.role !== "user" && message.role !== "developer")) return []; + if (preparedImages) return [...preparedImages]; + return resolveCursorImageParts(extractCursorImageParts(message.content), signal); +} + +function imageDataUrlFromPrepared(image: ResolvedCursorImage): string { + return `data:${image.mimeType};base64,${Buffer.from(image.data).toString("base64")}`; +} + +/** + * Re-encode a single image URL through {@link prepareCursorImageForWire}. + * data: URLs only. Omitted images become text (caller replaces the part). + */ +export async function prepareCursorImageDataUrl( + imageUrl: string, + detail?: string, + signal?: AbortSignal, +): Promise< + | { status: "ready"; imageUrl: string; image: ResolvedCursorImage } + | { status: "omitted"; reason: string } +> { + try { + const resolved = imageUrl.toLowerCase().startsWith("data:") + ? decodeCursorImageDataUrl(imageUrl) + : null; + if (!resolved) { + return { status: "omitted", reason: CURSOR_VISION_IMAGE_OMITTED }; + } + if (resolved.data.byteLength === 0) { + return { status: "omitted", reason: CURSOR_VISION_IMAGE_OMITTED }; + } + const outcome = await prepareCursorImageForWire({ + data: resolved.data, + mimeType: resolved.mimeType, + uuid: randomUUID(), + ...(detail ? { detail } : {}), + }, signal); + if (outcome.status === "omitted") return outcome; + if (outcome.image.data.byteLength > MAX_CURSOR_IMAGE_BYTES) { + return { status: "omitted", reason: CURSOR_VISION_IMAGE_OMITTED }; + } + if ( + imageUrl.toLowerCase().startsWith("data:") + && outcome.image.data === resolved.data + && outcome.image.mimeType === resolved.mimeType + ) { + return { status: "ready", imageUrl, image: outcome.image }; + } + return { status: "ready", imageUrl: imageDataUrlFromPrepared(outcome.image), image: outcome.image }; + } catch (err) { + if (signal?.aborted || (err instanceof Error && err.name === "AbortError")) throw err; + return { status: "omitted", reason: CURSOR_VISION_IMAGE_OMITTED }; + } +} + +async function prepareCursorContentParts( + content: string | readonly OcxContentPart[], + signal?: AbortSignal, +): Promise<{ content: string | readonly OcxContentPart[]; images: ResolvedCursorImage[] }> { + if (typeof content === "string" || !Array.isArray(content)) { + return { content, images: [] }; + } + let changed = false; + const next: OcxContentPart[] = []; + const images: ResolvedCursorImage[] = []; + for (const part of content) { + if (part.type === "image" && typeof part.imageUrl === "string" && part.imageUrl.length > 0) { + throwIfImagePhaseAborted(signal); + const prepared = await prepareCursorImageDataUrl(part.imageUrl, part.detail, signal); + if (prepared.status === "omitted") { + changed = true; + next.push({ type: "text", text: prepared.reason }); + continue; + } + images.push(prepared.image); + if (prepared.imageUrl !== part.imageUrl) changed = true; + next.push({ ...part, imageUrl: prepared.imageUrl }); + } else { + next.push(part); + } + } + return { content: changed ? next : content, images }; +} + +/** + * First original-message index that still needs image prep for the active vision window. + * Historical messages before this index are left untouched (no decode). + */ +export function cursorVisionPrepareStartIndex(messages: readonly OcxMessage[]): number { + // Tool-result image preparation is out of scope in this slice. + if (messages.at(-1)?.role === "toolResult") return messages.length; + for (let i = messages.length - 1; i >= 0; i--) { + const role = messages[i]?.role; + if (role === "user" || role === "developer") return i; + } + return messages.length; +} + +/** + * Rewrite image data URLs in the active vision window (last user/developer turn) through + * the JPEG soft-cap path before protobuf encode. Historical messages are left by + * reference. Undecodable images become {@link CURSOR_VISION_IMAGE_OMITTED} text so + * image-only turns stay userMessageAction. + */ +export interface PreparedCursorRawMessages { + messages: readonly OcxMessage[] | undefined; + images: ResolvedCursorImage[]; +} + +export async function prepareCursorRawMessages( + messages: readonly OcxMessage[] | undefined, + signal?: AbortSignal, +): Promise { + if (!messages?.length) return { messages, images: [] }; + throwIfImagePhaseAborted(signal); + const prepareFrom = cursorVisionPrepareStartIndex(messages); + const active = messages[prepareFrom]; + if ( + active + && (active.role === "user" || active.role === "developer") + && extractCursorImageParts(active.content).length > MAX_CURSOR_IMAGES + ) { + throw new CursorImageError(`Too many images in one request (max ${MAX_CURSOR_IMAGES}).`); + } + let changed = false; + const out: OcxMessage[] = []; + const images: ResolvedCursorImage[] = []; + for (let i = 0; i < messages.length; i++) { + throwIfImagePhaseAborted(signal); + const message = messages[i]!; + if ( + i >= prepareFrom + && (message.role === "user" || message.role === "developer") + ) { + const prepared = await prepareCursorContentParts(message.content, signal); + images.push(...prepared.images); + if (prepared.content !== message.content) { + changed = true; + out.push({ ...message, content: prepared.content } as OcxMessage); + continue; + } + } + out.push(message); + } + return { messages: changed ? out : messages, images }; +} diff --git a/src/adapters/cursor/live-transport.ts b/src/adapters/cursor/live-transport.ts index afc411e243..2696d2fb25 100644 --- a/src/adapters/cursor/live-transport.ts +++ b/src/adapters/cursor/live-transport.ts @@ -13,6 +13,8 @@ import { type TranslatorBudget, } from "../../lib/translator-budget"; import { activePromptText, prepareCursorRunRequest } from "./protobuf-request"; +import { prepareCursorRawMessages, resolveActiveCursorImages } from "./images"; +import { cursorRequestMessagesFromRaw } from "./request-builder"; import { createCursorContextUsageTracker, createCursorProtobufEventState, @@ -537,10 +539,29 @@ class LiveCursorTransport implements CursorTransport { // Advertise MCP tools before the stream opens — the server only calls tools it was told about. await this.prepareMcp(); - const activeText = activePromptText(request); - this.activeClientToolFinalizeGraceMs = clientToolFinalizeGraceMsForRequest(request, this.clientToolFinalizeGraceMs); - const cursorVisibleTools = cursorToolsForActivePrompt(request.tools, activeText, request.toolChoice); - const clientToolDefs = buildCursorToolDefinitions(cursorVisibleTools, request.toolChoice); + // JPEG soft-cap rewrite for active-turn data: images before encode. Rebuild text + // messages from the prepared raw channel so omission markers replace stale + // pre-rewrite content that activePromptText and the tool filter would otherwise see. + const preparedRaw = await prepareCursorRawMessages(request.rawMessages, signal); + const preparedRawMessages = preparedRaw.messages; + const selectedImages = await resolveActiveCursorImages( + preparedRawMessages, + signal, + preparedRaw.images, + ); + const preparedMessages = preparedRawMessages === request.rawMessages + ? request.messages + : cursorRequestMessagesFromRaw(preparedRawMessages); + const activeRequest: CursorRunRequest = { + ...request, + messages: preparedMessages, + rawMessages: preparedRawMessages, + selectedImages, + }; + const activeText = activePromptText(activeRequest); + this.activeClientToolFinalizeGraceMs = clientToolFinalizeGraceMsForRequest(activeRequest, this.clientToolFinalizeGraceMs); + const cursorVisibleTools = cursorToolsForActivePrompt(activeRequest.tools, activeText, activeRequest.toolChoice); + const clientToolDefs = buildCursorToolDefinitions(cursorVisibleTools, activeRequest.toolChoice); // `request.tools` is the catalog already filtered and budgeted by request-builder. Derive // conversion provenance only from tagged synthetic tools that also survive this final prompt // filter; a client tool with the same wire name can never opt into conversion by collision. @@ -576,7 +597,7 @@ class LiveCursorTransport implements CursorTransport { }); // Build the payload once. The estimate is only worth deriving when there is no // carry-forward to fall back on — with a carry present it would never be used (#373). - const prepared = prepareCursorRunRequest(request, { + const prepared = prepareCursorRunRequest(activeRequest, { estimateInputTokens: contextUsage.carryForwardTokens === undefined, }); this.blobRequestScope = prepared.blobRequestScope; diff --git a/src/adapters/cursor/protobuf-request.ts b/src/adapters/cursor/protobuf-request.ts index 4ede0a482f..81d304d03c 100644 --- a/src/adapters/cursor/protobuf-request.ts +++ b/src/adapters/cursor/protobuf-request.ts @@ -13,6 +13,7 @@ import { storeCursorBlob, type CursorBlobRequestScopeToken, } from "./native-exec"; +import { buildSelectedContext, CURSOR_VISION_IMAGE_HISTORY_MARKER } from "./images"; import { estimateTokens } from "../../lib/token-estimate"; import { AgentClientMessageSchema, @@ -203,7 +204,7 @@ function rootPromptMessages(request: CursorRunRequest, requestScope: CursorBlobR const message = messages[i]; if (!message) continue; if (message.role === "user" || message.role === "developer") { - const text = contentText(message).trim(); + const text = historyContentText(message).trim(); // Cursor root replay expects OpenAI-style content parts for historical user messages. // A bare string survives blob hydration but external workers reject the completed replay // before tokenization (`usedTokens: 0`, then invalid_argument). @@ -318,7 +319,7 @@ function contentText(message: OcxMessage): string { .map(part => { if (part.type === "text") return part.text; if (part.type === "thinking") return part.thinking; - if (part.type === "image") return `[image input unsupported by Cursor adapter phase 3: ${part.detail ?? "auto"}]`; + if (part.type === "image") return undefined; return undefined; }) .filter((value): value is string => typeof value === "string" && value.length > 0) @@ -328,7 +329,26 @@ function contentText(message: OcxMessage): string { function contentToText(content: OcxToolResultMessage["content"]): string { if (typeof content === "string") return content; return content - .map(part => part.type === "text" ? part.text : `[image input unsupported by Cursor adapter phase 3: ${part.detail ?? "auto"}]`) + .map(part => { + if (part.type === "text") return part.text; + if (part.type === "image") return CURSOR_VISION_IMAGE_HISTORY_MARKER; + return undefined; + }) + .filter((value): value is string => typeof value === "string" && value.length > 0) + .join("\n"); +} + +/** History serializer. Replayed turns are text-only; never embed image bytes. */ +function historyContentText(message: OcxMessage): string { + if (message.role === "toolResult" || typeof message.content === "string") return contentText(message); + return message.content + .map(part => { + if (part.type === "text") return part.text; + if (part.type === "thinking") return part.thinking; + if (part.type === "image") return CURSOR_VISION_IMAGE_HISTORY_MARKER; + return undefined; + }) + .filter((value): value is string => typeof value === "string" && value.length > 0) .join("\n"); } @@ -507,8 +527,10 @@ function conversationTurns( flush(); current = { userMessage: storeCursorBlob(toBinary(UserMessageSchema, create(UserMessageSchema, { - text: contentText(message), + text: historyContentText(message), messageId: crypto.randomUUID(), + selectedContext: buildSelectedContext([], requestScope), + mode: 1, })), requestScope), steps: [], }; @@ -579,7 +601,10 @@ function buildPreparedCursorRunRequest( : rawText; // Tool-result-only turns resume the remembered Cursor conversation with results in history. const lastRawIsToolResult = request.rawMessages?.at(-1)?.role === "toolResult"; - const actionCase = !lastRawIsToolResult && text.trim().length > 0 + const selectedImages = request.selectedImages ?? []; + // Image-only active turns (including soft-omitted images) stay userMessageAction. + // Tool-result continuations always resume; they never carry an active-turn image. + const actionCase = !lastRawIsToolResult && (text.trim().length > 0 || selectedImages.length > 0) ? "userMessageAction" : "resumeAction"; const action = create(ConversationActionSchema, { @@ -590,6 +615,9 @@ function buildPreparedCursorRunRequest( userMessage: create(UserMessageSchema, { text, messageId: crypto.randomUUID(), + selectedContext: buildSelectedContext(selectedImages, requestScope), + // OmniRoute / cursor-agent always send mode=1 on UserMessage. + mode: 1, }), requestContext: buildRequestContext(), }), diff --git a/src/adapters/cursor/request-builder.ts b/src/adapters/cursor/request-builder.ts index 8338080178..80860bdd37 100644 --- a/src/adapters/cursor/request-builder.ts +++ b/src/adapters/cursor/request-builder.ts @@ -25,6 +25,7 @@ import { isCursorWaitTool, } from "./tool-definitions"; import { lookupCursorThreadConversation } from "./thread-continuity"; +import { extractCursorImageUrls } from "./images"; /** Probe-verified Cursor Connect boundaries, with byte headroom for the enclosing field. */ export const CURSOR_TOOL_COUNT_LIMIT = 330; @@ -205,7 +206,8 @@ function contentPartToText(part: OcxContentPart | OcxAssistantContentPart): stri case "thinking": return part.thinking; case "image": - return `[image input unsupported by Cursor adapter phase 3: ${part.detail ?? "auto"}]`; + // Images ride UserMessage.selected_context (SelectedImage) instead of text. + return undefined; case "toolCall": // Cursor does not accept OpenAI Responses assistant tool-call parts as native history here. // Rendering them as visible "[tool_call]" text leaks synthetic protocol markers back into @@ -238,9 +240,19 @@ function requestMessage(message: OcxMessage): CursorRequestMessage | undefined { switch (message.role) { case "user": case "developer": - return { role: message.role, content: contentToText(message.content) }; + { + const content = contentToText(message.content); + // Image-only turns survive as empty content; the encoder keeps them userMessageAction. + if (content.length === 0 && extractCursorImageUrls(message.content).length === 0) { + return undefined; + } + return { role: message.role, content }; + } case "assistant": - return { role: "assistant", content: contentToText(message.content) }; + { + const content = contentToText(message.content); + return content.length > 0 ? { role: "assistant", content } : undefined; + } case "toolResult": return { role: "tool", @@ -249,6 +261,19 @@ function requestMessage(message: OcxMessage): CursorRequestMessage | undefined { } } +/** + * Rebuild the text `messages` channel from prepared `rawMessages` so omission markers + * and JPEG-rewritten parts stay visible to activePromptText after image preparation. + */ +export function cursorRequestMessagesFromRaw( + messages: readonly OcxMessage[] | undefined, +): CursorRequestMessage[] { + if (!messages?.length) return []; + return messages + .map(requestMessage) + .filter((message): message is CursorRequestMessage => !!message); +} + export function generatedCursorConversationId(): string { return `cursor_${crypto.randomUUID().replace(/-/g, "")}`; } @@ -299,9 +324,7 @@ export function createCursorRequest( parsed: OcxParsedRequest, options: CreateCursorRequestOptions = {}, ): CursorRunRequest { - const messages = parsed.context.messages - .map(requestMessage) - .filter((message): message is CursorRequestMessage => !!message && message.content.length > 0); + const messages = cursorRequestMessagesFromRaw(parsed.context.messages); const activeText = [...messages].reverse().find(message => message.role === "user" || message.role === "developer")?.content ?? ""; const visibleTools = cursorToolsForActivePrompt(parsed.context.tools, activeText, parsed.options.toolChoice); const budget = applyCursorToolBudget(visibleTools, parsed.options.toolChoice); diff --git a/src/adapters/cursor/types.ts b/src/adapters/cursor/types.ts index b32026a07d..ece91f4fee 100644 --- a/src/adapters/cursor/types.ts +++ b/src/adapters/cursor/types.ts @@ -1,6 +1,7 @@ import type { OcxUsage } from "../../types"; import type { OcxMessage, OcxRequestOptions, OcxTool } from "../../types"; import type { CursorRoutingLevel } from "./discovery"; +import type { ResolvedCursorImage } from "./images"; export interface CursorRequestedModelParameter { id: string; @@ -16,7 +17,13 @@ export interface CursorRunRequest { conversationId: string; system: string[]; messages: CursorRequestMessage[]; - rawMessages?: OcxMessage[]; + rawMessages?: readonly OcxMessage[]; + /** + * Images for the active user/developer turn. Encoded as SelectedImage blobIdWithData refs under + * UserMessage.selected_context (bytes live in the request-scoped KV store for getBlobArgs + * hydration). History stays text-only. data: URLs only in this slice. + */ + selectedImages?: readonly ResolvedCursorImage[]; tools?: OcxTool[]; toolChoice?: OcxRequestOptions["toolChoice"]; parallelToolCalls?: boolean; diff --git a/src/providers/registry.ts b/src/providers/registry.ts index d188185ed3..995284823c 100644 --- a/src/providers/registry.ts +++ b/src/providers/registry.ts @@ -9,6 +9,7 @@ import { MOONSHOT_BASE_URL_CHOICES, MOONSHOT_INTL_BASE_URL, } from "./base-url-choices"; import { + CURSOR_NO_VISION_MODELS, CURSOR_STATIC_MODELS, cursorModelContextWindows, cursorModelIds, @@ -975,11 +976,10 @@ export const PROVIDER_REGISTRY: readonly ProviderRegistryEntry[] = [ // no-effort fallback to `kimi-k3-max` would never be reached. Mirrors the other K3 // routes (kimi, kimi-code, opencode-go). modelDefaultReasoningEfforts: { "kimi-k3": "max" }, - // Cursor's wire protocol never forwards image parts (request-builder emits an unsupported- - // content marker), so the vision sidecar covers ALL cursor models regardless of what the - // upstream model could natively do. Live-discovered models outside the static list fall back - // to the same marker until they appear here. - noVisionModels: cursorModelIds(CURSOR_STATIC_MODELS), + // Blind Cursor models (Auto routers, Composer, GLM-5.2) go through the vision sidecar; + // multimodal hosts (Claude/Gemini/GPT/Kimi/Grok) take native SelectedImage. The catalog + // still advertises image for noVision members so Codex can attach (sidecar option B). + noVisionModels: [...CURSOR_NO_VISION_MODELS], }, { id: "xai", diff --git a/tests/catalog-vision-sidecar-modalities.test.ts b/tests/catalog-vision-sidecar-modalities.test.ts index c49347025e..3cbe66c1b4 100644 --- a/tests/catalog-vision-sidecar-modalities.test.ts +++ b/tests/catalog-vision-sidecar-modalities.test.ts @@ -5,6 +5,8 @@ import type { OcxProviderConfig } from "../src/types"; import { deriveComboCatalogModel } from "../src/codex/catalog"; import { PROVIDER_REGISTRY } from "../src/providers/registry"; import { enrichProviderFromRegistry } from "../src/providers/derive"; +import { CURSOR_NO_VISION_MODELS, CURSOR_STATIC_MODELS } from "../src/adapters/cursor/discovery"; +import { modelInList } from "../src/types"; import type { CatalogModel } from "../src/types"; const base: OcxProviderConfig = { @@ -231,3 +233,22 @@ describe("vision-capable provider models feed combo modalities", () => { expect(hinted.inputModalities).toEqual(["text", "image"]); }); }); + +describe("Cursor native vs sidecar vision registry", () => { + test("curates noVisionModels for Auto/Composer/GLM while advertising image for all static ids", () => { + const cursor = PROVIDER_REGISTRY.find(entry => entry.id === "cursor"); + expect(cursor?.noVisionModels).toEqual([...CURSOR_NO_VISION_MODELS]); + for (const model of ["auto", "composer-1", "composer-2.5", "composer-2.5-fast", "glm-5.2", "glm-5.3"]) { + expect(modelInList(cursor?.noVisionModels, model), `${model} should match noVision`).toBe(true); + } + for (const model of ["auto", "composer-2.5", "glm-5.2", "glm-5.3", "gpt-5.5", "gemini-3-pro", "grok-4.5", "kimi-k3"]) { + expect(cursor?.modelInputModalities?.[model]).toEqual(["text", "image"]); + } + for (const model of ["gpt-5.5", "gemini-3-pro", "grok-4.5", "kimi-k3"]) { + expect(modelInList(cursor?.noVisionModels, model)).toBe(false); + } + for (const model of CURSOR_STATIC_MODELS) { + expect(cursor?.modelInputModalities?.[model.id]).toEqual(["text", "image"]); + } + }); +}); diff --git a/tests/cursor-blob.test.ts b/tests/cursor-blob.test.ts index e8df19da3e..9f59985484 100644 --- a/tests/cursor-blob.test.ts +++ b/tests/cursor-blob.test.ts @@ -36,6 +36,7 @@ import { GetBlobArgsSchema, KvServerMessageSchema, SetBlobArgsSchema, + UserMessageSchema, } from "../src/adapters/cursor/gen/agent_pb"; beforeEach(() => { @@ -114,6 +115,35 @@ function actionText(bytes: Uint8Array): string | undefined { return action?.case === "userMessageAction" ? action.value.userMessage?.text : undefined; } +/** Minimal valid 1×1 PNG for SelectedImage fixtures (not signature-only). */ +const PNG_1X1 = Uint8Array.from( + Buffer.from( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==", + "base64", + ), +); + +function activeUserMessage(bytes: Uint8Array) { + const msg = fromBinary(AgentClientMessageSchema, bytes); + const run = msg.message.case === "runRequest" ? msg.message.value : undefined; + const action = run?.action?.action; + return action?.case === "userMessageAction" ? action.value.userMessage : undefined; +} + +function activeSelectedImages(bytes: Uint8Array) { + return activeUserMessage(bytes)?.selectedContext?.selectedImages; +} + +function nativeTurnUserMessages(bytes: Uint8Array) { + const msg = fromBinary(AgentClientMessageSchema, bytes); + const run = msg.message.case === "runRequest" ? msg.message.value : undefined; + return (run?.conversationState?.turns ?? []).map(turnId => { + const turn = fromBinary(ConversationTurnStructureSchema, blobData(turnId)); + if (turn.turn.case !== "agentConversationTurn") return undefined; + return fromBinary(UserMessageSchema, blobData(turn.turn.value.userMessage)); + }).filter((message): message is NonNullable => message !== undefined); +} + /** The `toolName`s advertised in the top-level AgentRunRequest.mcp_tools channel (undefined when unset). */ function mcpToolNames(bytes: Uint8Array): string[] | undefined { const msg = fromBinary(AgentClientMessageSchema, bytes); @@ -129,6 +159,221 @@ describe("Cursor blob handshake", () => { expect(Array.from(id)).toEqual(Array.from(sha256(data))); }); + test("encodeCursorRunRequest attaches selectedContext with blobIdWithData image refs on the active user turn", () => { + const imageBytes = PNG_1X1; + const expectedBlobId = sha256(imageBytes); + const bytes = encodeCursorRunRequest({ + modelId: "claude-4.6-opus-high", + conversationId: "c1", + system: [], + messages: [{ role: "user", content: "see this" }], + selectedImages: [{ + data: imageBytes, + mimeType: "image/png", + uuid: "img-uuid-1", + }], + }); + + expect(actionText(bytes)).toBe("see this"); + const userMessage = activeUserMessage(bytes); + expect(userMessage?.mode).toBe(1); + expect(userMessage?.selectedContext).toBeDefined(); + const images = activeSelectedImages(bytes); + expect(images?.length).toBe(1); + expect(images?.[0]?.uuid).toBe("img-uuid-1"); + expect(images?.[0]?.mimeType).toBe("image/png"); + expect(images?.[0]?.path).toBe("attachment-img-uuid-1.png"); + expect(images?.[0]?.dataOrBlobId.case).toBe("blobIdWithData"); + const withData = images?.[0]?.dataOrBlobId.value as { blobId: Uint8Array; data: Uint8Array }; + expect(Array.from(withData.blobId)).toEqual(Array.from(expectedBlobId)); + expect(Array.from(withData.data)).toEqual(Array.from(imageBytes)); + expect(Array.from(blobData(expectedBlobId))).toEqual(Array.from(imageBytes)); + }); + + test("encodeCursorRunRequest always sends empty selectedContext and mode=1 on text-only turns", () => { + const bytes = encodeCursorRunRequest({ + modelId: "claude-4.6-opus-high", + conversationId: "c1", + system: [], + messages: [{ role: "user", content: "hi" }], + }); + + const userMessage = activeUserMessage(bytes); + expect(userMessage?.text).toBe("hi"); + expect(userMessage?.mode).toBe(1); + expect(userMessage?.selectedContext).toBeDefined(); + expect(userMessage?.selectedContext?.selectedImages.length).toBe(0); + }); + + test("encodeCursorRunRequest keeps selectedContext only on the active user turn", () => { + const activeImageBytes = PNG_1X1; + const bytes = encodeCursorRunRequest({ + modelId: "composer-2.5", + conversationId: "c1", + system: ["You are helpful."], + messages: [{ role: "user", content: "active turn" }], + rawMessages: [ + { + role: "user", + content: [ + { type: "text", text: "old turn" }, + { type: "image", imageUrl: "data:image/png;base64,old", detail: "auto" }, + ], + timestamp: 1, + }, + { + role: "assistant", + model: "cursor/composer-2.5", + content: [{ type: "text", text: "ack" }], + timestamp: 2, + }, + { role: "user", content: "active turn", timestamp: 3 }, + ], + selectedImages: [{ + data: activeImageBytes, + mimeType: "image/png", + uuid: "active-img", + }], + }); + + const roots = decodeRootMessages(bytes) as Array<{ role?: string; selectedContext?: unknown }>; + expect(roots.some(root => root.selectedContext !== undefined)).toBe(false); + + const historicalUser = nativeTurnUserMessages(bytes)[0]; + expect(historicalUser?.text).toBe("old turn\n[image attached]"); + expect(historicalUser?.mode).toBe(1); + expect(historicalUser?.selectedContext).toBeDefined(); + expect(historicalUser?.selectedContext?.selectedImages.length).toBe(0); + + const activeMessage = activeUserMessage(bytes); + expect(activeMessage?.mode).toBe(1); + const images = activeSelectedImages(bytes); + expect(images?.length).toBe(1); + expect(images?.[0]?.uuid).toBe("active-img"); + }); + + test("historical image-only turns replay a short text marker, not empty text", () => { + const bytes = encodeCursorRunRequest({ + modelId: "composer-2.5", + conversationId: "c1", + system: [], + messages: [{ role: "user", content: "follow-up" }], + rawMessages: [ + { + role: "user", + content: [{ type: "image", imageUrl: "data:image/png;base64,abc", detail: "auto" }], + timestamp: 1, + }, + { + role: "assistant", + model: "cursor/composer-2.5", + content: [{ type: "text", text: "ack" }], + timestamp: 2, + }, + { role: "user", content: "follow-up", timestamp: 3 }, + ], + }); + const historicalUser = nativeTurnUserMessages(bytes)[0]; + expect(historicalUser?.text).toBe("[image attached]"); + expect(historicalUser?.selectedContext?.selectedImages.length).toBe(0); + }); + + test("external root-prompt replay keeps image-only history as the text marker", () => { + const bytes = encodeCursorRunRequest({ + modelId: "grok-4.5", + conversationId: "c1", + system: [], + messages: [{ role: "user", content: "follow-up" }], + rawMessages: [ + { + role: "user", + content: [{ type: "image", imageUrl: "data:image/png;base64,abc", detail: "auto" }], + timestamp: 1, + }, + { + role: "assistant", + model: "cursor/grok-4.5", + content: [{ type: "text", text: "ack" }], + timestamp: 2, + }, + { role: "user", content: "follow-up", timestamp: 3 }, + ], + }); + const roots = decodeRootMessages(bytes) as Array<{ + role?: string; + content?: Array<{ type?: string; text?: string }>; + }>; + const rootsJson = JSON.stringify(roots); + expect(rootsJson).toContain("[image attached]"); + expect(rootsJson).not.toContain("data:image/png;base64,"); + expect(rootsJson).not.toContain("abc"); + expect(roots).toContainEqual({ + role: "user", + content: [{ type: "text", text: "[image attached]" }], + }); + }); + + test("encodeCursorRunRequest uses userMessageAction for image-only turns with selectedImages", () => { + const imageBytes = PNG_1X1; + const bytes = encodeCursorRunRequest({ + modelId: "claude-4.6-opus-high", + conversationId: "c1", + system: [], + messages: [{ role: "user", content: "" }], + rawMessages: [{ + role: "user", + content: [{ type: "image", imageUrl: "data:image/png;base64,abc", detail: "auto" }], + timestamp: 1, + }], + selectedImages: [{ + data: imageBytes, + mimeType: "image/png", + uuid: "image-only", + }], + }); + + expect(activeUserMessage(bytes)).toBeDefined(); + expect(actionText(bytes)).toBe(""); + expect(activeSelectedImages(bytes)?.length).toBe(1); + }); + + test("encodeCursorRunRequest uses userMessageAction for image-only turns after assistant reply", () => { + const imageBytes = PNG_1X1; + const bytes = encodeCursorRunRequest({ + modelId: "composer-2.5", + conversationId: "c1", + system: [], + messages: [ + { role: "user", content: "first" }, + { role: "assistant", content: "ack" }, + { role: "user", content: "" }, + ], + rawMessages: [ + { role: "user", content: "first", timestamp: 1 }, + { + role: "assistant", + model: "cursor/composer-2.5", + content: [{ type: "text", text: "ack" }], + timestamp: 2, + }, + { + role: "user", + content: [{ type: "image", imageUrl: "data:image/png;base64,abc", detail: "auto" }], + timestamp: 3, + }, + ], + selectedImages: [{ + data: imageBytes, + mimeType: "image/png", + uuid: "follow-up-image", + }], + }); + + expect(activeUserMessage(bytes)).toBeDefined(); + expect(actionText(bytes)).toBe(""); + expect(activeSelectedImages(bytes)?.length).toBe(1); + }); + test("encodeCursorRunRequest sends rootPromptMessagesJson as blob IDs, not inline JSON", () => { const bytes = encodeCursorRunRequest({ modelId: "claude-4.6-opus-high", diff --git a/tests/cursor-discovery.test.ts b/tests/cursor-discovery.test.ts index 7e9595aca2..fc3bcfdeb1 100644 --- a/tests/cursor-discovery.test.ts +++ b/tests/cursor-discovery.test.ts @@ -4,6 +4,7 @@ import { CURSOR_DEFAULT_CONTEXT_WINDOW, CURSOR_ROUTER_MODEL_IDS, CURSOR_ROUTING_LEVELS, + CURSOR_NO_VISION_MODELS, CURSOR_STATIC_MODELS, cursorCodexToWireModelId, filterCursorConfiguredModelsByLiveDiscovery, @@ -20,6 +21,23 @@ import { } from "../src/adapters/cursor/discovery"; describe("Cursor discovery metadata", () => { + test("no-vision list is a curated explicit subset of the static seed", () => { + const ids = new Set(cursorModelIds(CURSOR_STATIC_MODELS)); + expect([...CURSOR_NO_VISION_MODELS]).toEqual([ + ...CURSOR_ROUTER_MODEL_IDS, + "composer-1", + "composer-2.5", + "composer-2.5-fast", + "glm-5.2", + "glm-5.3", + ]); + for (const id of CURSOR_NO_VISION_MODELS) { + expect(ids.has(id), `${id} must be in the static Cursor seed`).toBe(true); + } + for (const id of ["grok-4.5", "grok-4.5-fast", "gpt-5.5", "claude-sonnet-5", "kimi-k3", "gemini-3-pro"]) { + expect(CURSOR_NO_VISION_MODELS as readonly string[]).not.toContain(id); + } + }); test("static seed includes Cursor's public model families plus the safe auto model", () => { const ids = cursorModelIds(CURSOR_STATIC_MODELS); diff --git a/tests/cursor-images.test.ts b/tests/cursor-images.test.ts new file mode 100644 index 0000000000..be75b8636c --- /dev/null +++ b/tests/cursor-images.test.ts @@ -0,0 +1,635 @@ +import { describe, expect, test } from "bun:test"; +import { createHash } from "node:crypto"; +import { create, fromBinary } from "@bufbuild/protobuf"; +import { + CursorImageError, + CURSOR_VISION_IMAGE_OMITTED, + CURSOR_VISION_SOFT_MAX_BYTES, + CURSOR_VISION_SOFT_MAX_BYTES_HIGH, + MAX_CURSOR_IMAGE_BYTES, + MAX_CURSOR_IMAGE_DECODE_BYTES, + MAX_CURSOR_IMAGE_DECODE_EDGE, + MAX_CURSOR_IMAGE_PIXELS, + MAX_CURSOR_IMAGES, + buildSelectedImages, + cursorVisionPrepareStartIndex, + decodeCursorImageDataUrl, + prepareCursorImageForWire, + prepareCursorRawMessages, + resolveActiveCursorImages, + resolveCursorImages, + sniffCursorImageDimensions, + sniffCursorImageFormat, +} from "../src/adapters/cursor/images"; +import { + handleCursorNativeKv, + resetCursorBlobStateForTests, +} from "../src/adapters/cursor/native-exec"; +import { cursorRequestMessagesFromRaw } from "../src/adapters/cursor/request-builder"; +import { activePromptText, encodeCursorRunRequest } from "../src/adapters/cursor/protobuf-request"; +import { + AgentClientMessageSchema, + GetBlobArgsSchema, + KvServerMessageSchema, +} from "../src/adapters/cursor/gen/agent_pb"; + +/** Minimal valid 1×1 PNG (real IHDR; not a signature-only stub). */ +const PNG_BYTES = Uint8Array.from( + Buffer.from( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==", + "base64", + ), +); +const PNG_DATA_URL = `data:image/png;base64,${Buffer.from(PNG_BYTES).toString("base64")}`; + +async function oversizedDecodablePng(): Promise { + const pngPath = new URL("./helpers/cursor-grumpy-fixture.png", import.meta.url); + const src = new Uint8Array(await Bun.file(pngPath).arrayBuffer()); + return new Uint8Array(await new Bun.Image(src).resize(2400, 2400).png().bytes()); +} + +describe("Cursor image resolver", () => { + test("rejects more than MAX_CURSOR_IMAGES in one request", async () => { + const urls = Array.from({ length: MAX_CURSOR_IMAGES + 1 }, () => PNG_DATA_URL); + await expect(resolveCursorImages(urls)).rejects.toMatchObject({ + name: "CursorImageError", + message: `Too many images in one request (max ${MAX_CURSOR_IMAGES}).`, + }); + await expect(prepareCursorRawMessages([{ + role: "user", + content: urls.map(imageUrl => ({ type: "image" as const, imageUrl })), + timestamp: 1, + }])).rejects.toMatchObject({ + name: "CursorImageError", + message: `Too many images in one request (max ${MAX_CURSOR_IMAGES}).`, + }); + }); + + test("omits data URLs above the inbound decode bomb ceiling", async () => { + const oversized = "A".repeat(Math.ceil((MAX_CURSOR_IMAGE_DECODE_BYTES + 1) * 4 / 3)); + const padded = oversized + "=".repeat((4 - (oversized.length % 4)) % 4); + const url = `data:image/png;base64,${padded}`; + // Pin the guard itself: the resolver soft-omits every failure reason identically. + expect(() => decodeCursorImageDataUrl(url)).toThrow("Image input is too large to process safely."); + // Soft-omit: one bad URL must not abort a mixed turn. + const resolved = await resolveCursorImages([url]); + expect(resolved).toEqual([]); + }); + + test("prep-before-cap accepts PNG over 1 MiB that JPEG-encodes under the soft and wire caps", async () => { + const png = await oversizedDecodablePng(); + expect(png.byteLength).toBeGreaterThan(MAX_CURSOR_IMAGE_BYTES); + const url = `data:image/png;base64,${Buffer.from(png).toString("base64")}`; + const resolved = await resolveCursorImages([url]); + expect(resolved).toHaveLength(1); + expect(resolved[0]?.mimeType).toBe("image/jpeg"); + expect(resolved[0]!.data.byteLength).toBeLessThan(png.byteLength); + expect(resolved[0]!.data.byteLength).toBeLessThanOrEqual(CURSOR_VISION_SOFT_MAX_BYTES); + expect(resolved[0]!.data.byteLength).toBeLessThanOrEqual(MAX_CURSOR_IMAGE_BYTES); + }); + + test("omits undecodable payloads under the decode ceiling instead of sending them", async () => { + const junk = "A".repeat(Math.ceil((MAX_CURSOR_IMAGE_BYTES + 1) * 4 / 3)); + // Pad to valid base64 length so alphabet/padding checks pass and Bun decode fails. + const padded = junk + "=".repeat((4 - (junk.length % 4)) % 4); + const resolved = await resolveCursorImages([`data:image/png;base64,${padded}`]); + expect(resolved).toEqual([]); + }); + + test("decodes valid base64 data URLs through JPEG prep", async () => { + const resolved = await resolveCursorImages([PNG_DATA_URL]); + expect(resolved).toHaveLength(1); + expect(resolved[0]?.mimeType).toBe("image/jpeg"); + expect(resolved[0]!.data.byteLength).toBeGreaterThan(0); + expect(resolved[0]!.data[0]).toBe(0xff); + expect(resolved[0]!.data[1]).toBe(0xd8); + expect(resolved[0]?.uuid.length).toBeGreaterThan(0); + }); + + test("soft-omits malformed and non-image data URLs", async () => { + expect(await resolveCursorImages(["data:image/png,not-base64"])).toEqual([]); + expect(await resolveCursorImages(["data:text/plain;base64,YQ=="])).toEqual([]); + expect(await resolveCursorImages(["data:image/png;base64"])).toEqual([]); + expect(await resolveCursorImages(["data:image/png;base64,"])).toEqual([]); + }); + + test("omits remote URLs — this slice is data: only", async () => { + expect(await resolveCursorImages(["http://example.com/image.png"])).toEqual([]); + expect(await resolveCursorImages(["https://example.com/image.png"])).toEqual([]); + }); + + test("resolveActiveCursorImages selects the last user turn and ignores earlier images", async () => { + const resolved = await resolveActiveCursorImages([ + { + role: "user", + content: [{ type: "image", imageUrl: PNG_DATA_URL }], + timestamp: 1, + }, + { + role: "assistant", + model: "cursor/auto", + content: [{ type: "text", text: "seen" }], + timestamp: 2, + }, + { + role: "user", + content: [ + { type: "text", text: "active" }, + { type: "image", imageUrl: PNG_DATA_URL }, + ], + timestamp: 3, + }, + ]); + expect(resolved).toHaveLength(1); + expect(resolved[0]?.mimeType).toBe("image/jpeg"); + }); + + test("resolveActiveCursorImages supports developer turns", async () => { + const resolved = await resolveActiveCursorImages([ + { + role: "developer", + content: [{ type: "image", imageUrl: PNG_DATA_URL }], + timestamp: 1, + }, + ]); + expect(resolved).toHaveLength(1); + }); + + test("resolveActiveCursorImages returns empty for text-only trailing toolResult", async () => { + const resolved = await resolveActiveCursorImages([ + { + role: "user", + content: [{ type: "image", imageUrl: PNG_DATA_URL }], + timestamp: 1, + }, + { + role: "toolResult", + toolCallId: "call-1", + toolName: "read_file", + content: "done", + isError: false, + timestamp: 2, + }, + ]); + expect(resolved).toEqual([]); + }); + + test("user message after toolResult does not promote stale tool images", async () => { + const resolved = await resolveActiveCursorImages([ + { role: "user", content: "first", timestamp: 1 }, + { + role: "toolResult", + toolCallId: "call_view", + toolName: "view_image", + content: [{ type: "image", imageUrl: PNG_DATA_URL }], + isError: false, + timestamp: 2, + }, + { role: "user", content: "new question without an image", timestamp: 3 }, + ]); + expect(resolved).toEqual([]); + }); + + test("CursorImageError carries HTTP status for callers", () => { + const error = new CursorImageError("blocked", 403); + expect(error.status).toBe(403); + expect(error.name).toBe("CursorImageError"); + }); + + test("buildSelectedImages uses blobIdWithData + attachment path and keeps KV hydrated", () => { + resetCursorBlobStateForTests(); + // Minimal PNG signature + IHDR claiming 2x3 (not Bun-decodable — stays PNG) + const png = Uint8Array.from([ + 137, 80, 78, 71, 13, 10, 26, 10, + 0, 0, 0, 13, 73, 72, 68, 82, + 0, 0, 0, 2, 0, 0, 0, 3, + 8, 2, 0, 0, 0, 0, 0, 0, 0, + ]); + expect(sniffCursorImageDimensions(png)).toEqual({ width: 2, height: 3 }); + + // Standalone RST0 before SOF0 must not be parsed as a length-bearing segment. + const jpegWithRst = Uint8Array.from([ + 0xff, 0xd8, // SOI + 0xff, 0xd0, // RST0 (no length) + 0xff, 0xc0, 0x00, 0x0b, 0x08, 0x00, 0x03, 0x00, 0x02, 0x03, 0x01, 0x11, 0x00, // SOF0 2x3 + ]); + expect(sniffCursorImageDimensions(jpegWithRst)).toEqual({ width: 2, height: 3 }); + + // Extended-sequential SOF1 (0xC1) shares the same dimension layout as SOF0. + const jpegSof1 = Uint8Array.from([ + 0xff, 0xd8, + 0xff, 0xc1, 0x00, 0x0b, 0x08, 0x00, 0x03, 0x00, 0x02, 0x03, 0x01, 0x11, 0x00, + ]); + expect(sniffCursorImageDimensions(jpegSof1)).toEqual({ width: 2, height: 3 }); + + const [selected] = buildSelectedImages([{ + data: png, + mimeType: "image/png", + uuid: "u-dim", + }]); + expect(selected?.dataOrBlobId.case).toBe("blobIdWithData"); + expect(selected?.path).toBe("attachment-u-dim.png"); + expect(selected?.dimension?.width).toBe(2); + expect(selected?.dimension?.height).toBe(3); + const withData = selected!.dataOrBlobId.value as { blobId: Uint8Array; data: Uint8Array }; + expect(Array.from(withData.blobId)).toEqual(Array.from(createHash("sha256").update(png).digest())); + expect(Array.from(withData.data)).toEqual(Array.from(png)); + + const reply = fromBinary(AgentClientMessageSchema, handleCursorNativeKv(create(KvServerMessageSchema, { + id: 1, + message: { case: "getBlobArgs", value: create(GetBlobArgsSchema, { blobId: withData.blobId }) }, + }))); + const kv = reply.message.case === "kvClientMessage" ? reply.message.value : undefined; + const data = kv?.message.case === "getBlobResult" ? kv.message.value.blobData : undefined; + expect(Array.from(data ?? [])).toEqual(Array.from(png)); + }); + + test("prepareCursorImageForWire re-encodes large PNG as JPEG under the soft cap", async () => { + const pngPath = new URL("./helpers/cursor-grumpy-fixture.png", import.meta.url); + const png = new Uint8Array(await Bun.file(pngPath).arrayBuffer()); + expect(png.byteLength).toBeGreaterThan(CURSOR_VISION_SOFT_MAX_BYTES); + + const prepared = await prepareCursorImageForWire({ + data: png, + mimeType: "image/png", + uuid: "big-png", + }); + expect(prepared.status).toBe("ready"); + if (prepared.status !== "ready") throw new Error("expected ready"); + expect(prepared.image.mimeType).toBe("image/jpeg"); + expect(prepared.image.data.byteLength).toBeLessThan(png.byteLength); + expect(prepared.image.data.byteLength).toBeLessThanOrEqual(CURSOR_VISION_SOFT_MAX_BYTES); + expect(prepared.image.data[0]).toBe(0xff); + expect(prepared.image.data[1]).toBe(0xd8); + }); + + test("detail original/high uses a higher soft tier than auto", async () => { + const pngPath = new URL("./helpers/cursor-grumpy-fixture.png", import.meta.url); + const png = new Uint8Array(await Bun.file(pngPath).arrayBuffer()); + const auto = await prepareCursorImageForWire({ + data: png, + mimeType: "image/png", + uuid: "auto", + detail: "auto", + }); + const original = await prepareCursorImageForWire({ + data: png, + mimeType: "image/png", + uuid: "original", + detail: "original", + }); + expect(auto.status).toBe("ready"); + expect(original.status).toBe("ready"); + if (auto.status !== "ready" || original.status !== "ready") throw new Error("expected ready"); + expect(original.image.data.byteLength).toBeGreaterThan(auto.image.data.byteLength); + expect(auto.image.data.byteLength).toBeLessThanOrEqual(CURSOR_VISION_SOFT_MAX_BYTES); + expect(original.image.data.byteLength).toBeLessThanOrEqual(CURSOR_VISION_SOFT_MAX_BYTES_HIGH); + expect(original.image.data.byteLength).toBeLessThanOrEqual(MAX_CURSOR_IMAGE_BYTES); + }); + + test("exotic MIME and corrupt PNG fail closed", async () => { + const bmp = await prepareCursorImageForWire({ + data: new Uint8Array([0x42, 0x4d, 0, 0, 0, 0]), + mimeType: "image/bmp", + uuid: "bmp", + }); + expect(bmp).toEqual({ status: "omitted", reason: CURSOR_VISION_IMAGE_OMITTED }); + + const corrupt = await prepareCursorImageForWire({ + data: new Uint8Array(128).fill(0x41), + mimeType: "image/png", + uuid: "corrupt", + }); + expect(corrupt).toEqual({ status: "omitted", reason: CURSOR_VISION_IMAGE_OMITTED }); + + // Soft-cap-sized labeled JPEG must still decode; junk under the soft max is omitted. + const fakeJpeg = await prepareCursorImageForWire({ + data: new Uint8Array(128).fill(0xff), + mimeType: "image/jpeg", + uuid: "fake-jpeg", + }); + expect(fakeJpeg).toEqual({ status: "omitted", reason: CURSOR_VISION_IMAGE_OMITTED }); + }); + + test("prepareCursorRawMessages JPEG-preps active-turn user data URLs", async () => { + const pngPath = new URL("./helpers/cursor-grumpy-fixture.png", import.meta.url); + const png = new Uint8Array(await Bun.file(pngPath).arrayBuffer()); + const imageUrl = `data:image/png;base64,${Buffer.from(png).toString("base64")}`; + const prepared = await prepareCursorRawMessages([ + { + role: "user", + content: [ + { type: "text", text: "describe" }, + { type: "image", imageUrl, detail: "auto" }, + ], + timestamp: 1, + }, + ]); + const user = prepared.messages?.[0]; + expect(user?.role).toBe("user"); + if (user?.role !== "user" || typeof user.content === "string") throw new Error("expected image parts"); + const part = user.content.find(item => item.type === "image"); + expect(part?.type).toBe("image"); + if (part?.type !== "image") throw new Error("expected image"); + expect(part.imageUrl.startsWith("data:image/jpeg;base64,")).toBe(true); + const payload = part.imageUrl.slice(part.imageUrl.indexOf(",") + 1); + const bytes = Buffer.from(payload, "base64"); + expect(bytes.byteLength).toBeLessThanOrEqual(CURSOR_VISION_SOFT_MAX_BYTES); + expect(bytes[0]).toBe(0xff); + expect(bytes[1]).toBe(0xd8); + }); + + test("prepareCursorRawMessages replaces exotic images with omission text", async () => { + const bmpUrl = `data:image/bmp;base64,${Buffer.from([0x42, 0x4d, 0, 0]).toString("base64")}`; + const prepared = await prepareCursorRawMessages([ + { + role: "user", + content: [{ type: "image", imageUrl: bmpUrl }], + timestamp: 1, + }, + ]); + const user = prepared.messages?.[0]; + expect(user?.role).toBe("user"); + if (user?.role !== "user" || typeof user.content === "string") throw new Error("expected parts"); + expect(user.content).toEqual([{ type: "text", text: CURSOR_VISION_IMAGE_OMITTED }]); + }); + + test("cursorRequestMessagesFromRaw surfaces omission text after prepare", async () => { + const bmpUrl = `data:image/bmp;base64,${Buffer.from([0x42, 0x4d, 0, 0]).toString("base64")}`; + const prepared = await prepareCursorRawMessages([ + { + role: "user", + content: [{ type: "image", imageUrl: bmpUrl }], + timestamp: 1, + }, + ]); + const raw = prepared.messages; + const messages = cursorRequestMessagesFromRaw(raw); + expect(messages).toEqual([{ role: "user", content: CURSOR_VISION_IMAGE_OMITTED }]); + expect(activePromptText({ + modelId: "grok-4.5", + conversationId: "cursor_test", + system: [], + messages, + rawMessages: raw, + })).toBe(CURSOR_VISION_IMAGE_OMITTED); + }); + + test("live-transport image phase: prepare rawMessages then resolve SelectedImage", async () => { + // Mirrors live-transport.ts: prepareCursorRawMessages → resolveActiveCursorImages. + const rawIn = [ + { + role: "user" as const, + content: [ + { type: "text" as const, text: "What is in this image?" }, + { type: "image" as const, imageUrl: PNG_DATA_URL, detail: "high" }, + ], + timestamp: 1, + }, + ]; + const prepared = await prepareCursorRawMessages(rawIn); + const rawMessages = prepared.messages; + const messages = cursorRequestMessagesFromRaw(rawMessages); + const selectedImages = await resolveActiveCursorImages(rawMessages, undefined, prepared.images); + expect(selectedImages).toHaveLength(1); + + resetCursorBlobStateForTests(); + const bytes = encodeCursorRunRequest({ + modelId: "grok-4.5", + conversationId: "c-wire", + system: ["You are helpful."], + messages, + rawMessages, + selectedImages, + }); + const msg = fromBinary(AgentClientMessageSchema, bytes); + const run = msg.message.case === "runRequest" ? msg.message.value : undefined; + expect(run?.action?.action.case).toBe("userMessageAction"); + if (run?.action?.action.case !== "userMessageAction") throw new Error("expected userMessageAction"); + expect(run.action.action.value.userMessage?.text).toContain("What is in this image?"); + expect(run.action.action.value.userMessage?.selectedContext?.selectedImages.length).toBe(1); + }); + + test("resolveActiveCursorImages reuses prepared bytes instead of re-encoding", async () => { + const prepared = await prepareCursorRawMessages([ + { + role: "user", + content: [ + { type: "text", text: "What is in this image?" }, + { type: "image", imageUrl: PNG_DATA_URL, detail: "high" }, + ], + timestamp: 1, + }, + ]); + expect(prepared.images).toHaveLength(1); + const selectedImages = await resolveActiveCursorImages( + prepared.messages, + undefined, + prepared.images, + ); + expect(selectedImages).toHaveLength(1); + expect(selectedImages[0]).toBe(prepared.images[0]); + expect(selectedImages[0]?.data).toBe(prepared.images[0]?.data); + }); + + test("image-only remote soft-omit yields userMessageAction with omission text", async () => { + const prepared = await prepareCursorRawMessages([ + { + role: "user", + content: [{ type: "image", imageUrl: "https://example.com/missing.png" }], + timestamp: 1, + }, + ]); + const raw = prepared.messages; + const messages = cursorRequestMessagesFromRaw(raw); + expect(messages).toEqual([{ role: "user", content: CURSOR_VISION_IMAGE_OMITTED }]); + const selectedImages = await resolveActiveCursorImages(raw, undefined, prepared.images); + expect(selectedImages).toEqual([]); + resetCursorBlobStateForTests(); + const bytes = encodeCursorRunRequest({ + modelId: "grok-4.5", + conversationId: "c-https-omit", + system: [], + messages, + rawMessages: raw, + selectedImages, + }); + const msg = fromBinary(AgentClientMessageSchema, bytes); + const run = msg.message.case === "runRequest" ? msg.message.value : undefined; + expect(run?.action?.action.case).toBe("userMessageAction"); + expect(actionTextFrom(bytes)).toBe(CURSOR_VISION_IMAGE_OMITTED); + }); + + test("remote image with valid text continues text-only", async () => { + const prepared = await prepareCursorRawMessages([ + { + role: "user", + content: [ + { type: "text", text: "what color is the sky?" }, + { type: "image", imageUrl: "https://example.com/missing.png" }, + ], + timestamp: 1, + }, + ]); + const raw = prepared.messages; + const messages = cursorRequestMessagesFromRaw(raw); + expect(typeof messages[0]?.content).toBe("string"); + expect(messages[0]?.content).toContain("what color is the sky?"); + expect(messages[0]?.content).toContain(CURSOR_VISION_IMAGE_OMITTED); + expect(await resolveActiveCursorImages(raw, undefined, prepared.images)).toEqual([]); + }); + + test("strict base64 rejects truncated and wrong-alphabet payloads", () => { + expect(() => decodeCursorImageDataUrl("data:image/png;base64,iVBOR")).toThrow(CursorImageError); + expect(() => decodeCursorImageDataUrl("data:image/png;base64,!!!!")).toThrow(CursorImageError); + // Signature-only 8-byte stub is valid base64 but must not bypass prepare (no ≤64 passthrough). + const stubUrl = `data:image/png;base64,${Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]).toString("base64")}`; + const decoded = decodeCursorImageDataUrl(stubUrl); + expect(decoded.data.byteLength).toBe(8); + }); + + test("signature-only PNG stub is omitted by prepare (no ≤64 bypass)", async () => { + const outcome = await prepareCursorImageForWire({ + data: new Uint8Array([137, 80, 78, 71, 13, 10, 26, 10]), + mimeType: "image/png", + uuid: "stub", + }); + expect(outcome).toEqual({ status: "omitted", reason: CURSOR_VISION_IMAGE_OMITTED }); + }); + + test("oversize sniffed dimensions omit without Bun decode bomb", async () => { + // PNG IHDR with absurd width/height; sniff rejects before Bun.Image. + const ihdr = new Uint8Array([ + 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, + 0x00, 0x00, 0x00, 0x0d, 0x49, 0x48, 0x44, 0x52, + 0x00, 0x01, 0x00, 0x00, // width 65536 + 0x00, 0x01, 0x00, 0x00, // height 65536 + ]); + expect(sniffCursorImageDimensions(ihdr)).toEqual({ width: 65536, height: 65536 }); + expect(65536).toBeGreaterThan(MAX_CURSOR_IMAGE_DECODE_EDGE); + expect(65536 * 65536).toBeGreaterThan(MAX_CURSOR_IMAGE_PIXELS); + const outcome = await prepareCursorImageForWire({ + data: ihdr, + mimeType: "image/png", + uuid: "huge", + }); + expect(outcome).toEqual({ status: "omitted", reason: CURSOR_VISION_IMAGE_OMITTED }); + }); + + test("truncated FF D8 JPEG under soft cap is omitted (no SOI-only fast path)", async () => { + const truncated = new Uint8Array([0xff, 0xd8, 0x00, 0x00]); + expect(sniffCursorImageFormat(truncated)).toBe("jpeg"); + expect(sniffCursorImageDimensions(truncated)).toBeUndefined(); + const outcome = await prepareCursorImageForWire({ + data: truncated, + mimeType: "image/jpeg", + uuid: "soi-only", + }); + expect(outcome).toEqual({ status: "omitted", reason: CURSOR_VISION_IMAGE_OMITTED }); + }); + + test("truncated JPEG with a valid SOF is omitted (no header-only passthrough)", async () => { + // SOI + SOF0 claiming 2x3, then EOF. Sniff succeeds; Bun.Image must still reject it. + const sofOnly = Uint8Array.from([ + 0xff, 0xd8, + 0xff, 0xc0, 0x00, 0x0b, 0x08, 0x00, 0x03, 0x00, 0x02, 0x03, 0x01, 0x11, 0x00, + ]); + expect(sniffCursorImageFormat(sofOnly)).toBe("jpeg"); + expect(sniffCursorImageDimensions(sofOnly)).toEqual({ width: 2, height: 3 }); + const outcome = await prepareCursorImageForWire({ + data: sofOnly, + mimeType: "image/jpeg", + uuid: "sof-only", + }); + expect(outcome).toEqual({ status: "omitted", reason: CURSOR_VISION_IMAGE_OMITTED }); + }); + + test("PNG bytes labeled image/jpeg are re-encoded as JPEG, not passthrough", async () => { + const outcome = await prepareCursorImageForWire({ + data: PNG_BYTES, + mimeType: "image/jpeg", + uuid: "mislabeled", + }); + expect(outcome.status).toBe("ready"); + if (outcome.status !== "ready") throw new Error("expected ready"); + expect(outcome.image.mimeType).toBe("image/jpeg"); + expect(outcome.image.data[0]).toBe(0xff); + expect(outcome.image.data[1]).toBe(0xd8); + expect(sniffCursorImageFormat(outcome.image.data)).toBe("jpeg"); + }); + + test("oversized WebP VP8X header omits before Bun decode", async () => { + // RIFF....WEBP + VP8X with canvas size 65536x65536 (stored as size-1). + const webp = new Uint8Array(30); + webp.set([0x52, 0x49, 0x46, 0x46], 0); // RIFF + webp.set([0x57, 0x45, 0x42, 0x50], 8); // WEBP + webp.set([0x56, 0x50, 0x38, 0x58], 12); // VP8X + // width-1 / height-1 as 24-bit LE at 24..29 → 65535 → displayed 65536 + webp[24] = 0xff; + webp[25] = 0xff; + webp[26] = 0x00; + webp[27] = 0xff; + webp[28] = 0xff; + webp[29] = 0x00; + expect(sniffCursorImageFormat(webp)).toBe("webp"); + expect(sniffCursorImageDimensions(webp)).toEqual({ width: 65536, height: 65536 }); + const outcome = await prepareCursorImageForWire({ + data: webp, + mimeType: "image/webp", + uuid: "huge-webp", + }); + expect(outcome).toEqual({ status: "omitted", reason: CURSOR_VISION_IMAGE_OMITTED }); + }); + + test("prepareCursorRawMessages leaves historical images untouched on a later user turn", async () => { + const oldUrl = `data:image/png;base64,${Buffer.from([...PNG_BYTES, 1]).toString("base64")}`; + const prepared = await prepareCursorRawMessages([ + { + role: "user", + content: [{ type: "image", imageUrl: oldUrl }], + timestamp: 1, + }, + { + role: "assistant", + model: "cursor/grok-4.5", + content: [{ type: "text", text: "seen" }], + timestamp: 2, + }, + { role: "user", content: "thanks, no image", timestamp: 3 }, + ]); + expect(prepared.messages?.[0]).toEqual({ + role: "user", + content: [{ type: "image", imageUrl: oldUrl }], + timestamp: 1, + }); + expect(cursorVisionPrepareStartIndex(prepared.messages ?? [])).toBe(2); + }); + + test("aborted image-phase signal stops further local prepare work", async () => { + const controller = new AbortController(); + controller.abort(); + await expect(prepareCursorImageForWire({ + data: PNG_BYTES, + mimeType: "image/png", + uuid: "aborted", + }, controller.signal)).rejects.toMatchObject({ name: "AbortError" }); + + await expect(prepareCursorRawMessages([ + { + role: "user", + content: [ + { type: "image", imageUrl: PNG_DATA_URL }, + { type: "image", imageUrl: PNG_DATA_URL }, + ], + timestamp: 1, + }, + ], controller.signal)).rejects.toMatchObject({ name: "AbortError" }); + }); +}); + +function actionTextFrom(bytes: Uint8Array): string | undefined { + const msg = fromBinary(AgentClientMessageSchema, bytes); + const run = msg.message.case === "runRequest" ? msg.message.value : undefined; + const action = run?.action?.action; + return action?.case === "userMessageAction" ? action.value.userMessage?.text : undefined; +} diff --git a/tests/cursor-request-builder.test.ts b/tests/cursor-request-builder.test.ts index be977c2b90..a16ed9efd5 100644 --- a/tests/cursor-request-builder.test.ts +++ b/tests/cursor-request-builder.test.ts @@ -180,7 +180,7 @@ describe("Cursor request builder", () => { ]); }); - test("uses an explicit image placeholder for unsupported image parts", () => { + test("omits image parts from text — they ride SelectedImage, not markers", () => { const request = createCursorRequest({ ...base, context: { @@ -198,8 +198,58 @@ describe("Cursor request builder", () => { }); expect(request.messages[0]?.content).toContain("see"); - expect(request.messages[0]?.content).toContain("image input unsupported"); - expect(request.messages[0]?.content).toContain("high"); + expect(request.messages[0]?.content).not.toContain("image input unsupported"); + expect(request.messages[0]?.content).not.toContain("data:image/png"); + }); + + test("preserves image-only user turns as empty-string active messages", () => { + const request = createCursorRequest({ + ...base, + context: { + messages: [ + { + role: "user", + content: [{ type: "image", imageUrl: "data:image/png;base64,abc", detail: "high" }], + timestamp: 1, + }, + ], + }, + }); + + expect(request.messages).toEqual([{ role: "user", content: "" }]); + expect(request.rawMessages?.length).toBe(1); + }); + + test("preserves image-only active user turn after assistant reply", () => { + const request = createCursorRequest({ + ...base, + context: { + messages: [ + { + role: "user", + content: [{ type: "image", imageUrl: "data:image/png;base64,abc", detail: "high" }], + timestamp: 1, + }, + { + role: "assistant", + model: "cursor/composer-2.5", + content: [{ type: "text", text: "ack" }], + timestamp: 2, + }, + { + role: "user", + content: [{ type: "image", imageUrl: "data:image/png;base64,def", detail: "high" }], + timestamp: 3, + }, + ], + }, + }); + + expect(request.messages).toEqual([ + { role: "user", content: "" }, + { role: "assistant", content: "ack" }, + { role: "user", content: "" }, + ]); }); test("preserves Responses tools and tool choice for Cursor request context", () => { diff --git a/tests/cursor-static-catalog.test.ts b/tests/cursor-static-catalog.test.ts index 4dbd772e86..fa8be304f3 100644 --- a/tests/cursor-static-catalog.test.ts +++ b/tests/cursor-static-catalog.test.ts @@ -111,5 +111,12 @@ describe("Cursor static Codex catalog", () => { ]); expect(entries.find(item => item.slug === "cursor/glm-5.2")?.supported_reasoning_levels) .toMatchObject([{ effort: "high" }, { effort: "max" }, { effort: "ultra" }]); + + for (const modelId of ["auto", "composer-2.5", "gpt-5.5", "gemini-3-pro"]) { + expect( + entries.find(item => item.slug === `cursor/${modelId}`)?.input_modalities, + `cursor/${modelId} should advertise image input`, + ).toEqual(["text", "image"]); + } }); }); diff --git a/tests/cursor-vision-wire-harness.test.ts b/tests/cursor-vision-wire-harness.test.ts new file mode 100644 index 0000000000..6b26c36931 --- /dev/null +++ b/tests/cursor-vision-wire-harness.test.ts @@ -0,0 +1,127 @@ +import { describe, expect, test } from "bun:test"; +import { fromBinary } from "@bufbuild/protobuf"; +import { encodeCursorRunRequest } from "../src/adapters/cursor/protobuf-request"; +import { CURSOR_VISION_IMAGE_HISTORY_MARKER } from "../src/adapters/cursor/images"; +import { + AgentClientMessageSchema, + ConversationStepSchema, + ConversationTurnStructureSchema, +} from "../src/adapters/cursor/gen/agent_pb"; +import { handleCursorNativeKv, resetCursorBlobStateForTests } from "../src/adapters/cursor/native-exec"; +import { GetBlobArgsSchema, KvServerMessageSchema } from "../src/adapters/cursor/gen/agent_pb"; +import { create } from "@bufbuild/protobuf"; + +function blobData(blobId: Uint8Array): Uint8Array { + const reply = fromBinary(AgentClientMessageSchema, handleCursorNativeKv(create(KvServerMessageSchema, { + id: 1, + message: { case: "getBlobArgs", value: create(GetBlobArgsSchema, { blobId }) }, + }))); + const kv = reply.message.case === "kvClientMessage" ? reply.message.value : undefined; + const result = kv?.message.case === "getBlobResult" ? kv.message.value.blobData : undefined; + if (!result) throw new Error("missing blob data"); + return result; +} + +function activeSelectedImageBytes(bytes: Uint8Array): Uint8Array | undefined { + const msg = fromBinary(AgentClientMessageSchema, bytes); + const run = msg.message.case === "runRequest" ? msg.message.value : undefined; + const action = run?.action?.action; + const image = action?.case === "userMessageAction" + ? action.value.userMessage?.selectedContext?.selectedImages[0] + : undefined; + if (!image) return undefined; + if (image.dataOrBlobId.case === "data") return image.dataOrBlobId.value; + if (image.dataOrBlobId.case === "blobId") return blobData(image.dataOrBlobId.value); + if (image.dataOrBlobId.case === "blobIdWithData") return image.dataOrBlobId.value.data; + return undefined; +} + +function anyMcpImageContent(bytes: Uint8Array): boolean { + const msg = fromBinary(AgentClientMessageSchema, bytes); + const run = msg.message.case === "runRequest" ? msg.message.value : undefined; + for (const turnId of run?.conversationState?.turns ?? []) { + const turn = fromBinary(ConversationTurnStructureSchema, blobData(turnId)); + if (turn.turn.case !== "agentConversationTurn") continue; + for (const stepId of turn.turn.value.steps) { + const step = fromBinary(ConversationStepSchema, blobData(stepId)); + if (step.message.case !== "toolCall") continue; + const tool = step.message.value.tool; + if (tool.case !== "mcpToolCall") continue; + const result = tool.value.result?.result; + if (result?.case !== "success") continue; + for (const item of result.value.content) { + if (item.content.case === "image") return true; + } + } + } + return false; +} + +describe("Cursor vision wire harness", () => { + test("grok attach keeps non-empty PNG bytes on the wire; tool-result images stay text-only", () => { + resetCursorBlobStateForTests(); + const imageBytes = new Uint8Array([137, 80, 78, 71, 13, 10, 26, 10, 0, 0, 0, 13]); + const imageUrl = `data:image/png;base64,${Buffer.from(imageBytes).toString("base64")}`; + + const attachBytes = encodeCursorRunRequest({ + modelId: "grok-4.5", + conversationId: "c1", + system: [], + messages: [{ role: "user", content: "see this" }], + selectedImages: [{ uuid: "img-uuid-1", mimeType: "image/png", data: imageBytes }], + }); + const attachWireBytes = activeSelectedImageBytes(attachBytes); + expect(attachWireBytes).toBeDefined(); + expect(attachWireBytes!.byteLength).toBeGreaterThan(0); + expect(Array.from(attachWireBytes!.slice(0, 4))).toEqual([137, 80, 78, 71]); + + const viewBytes = encodeCursorRunRequest({ + modelId: "grok-4.5", + conversationId: "c1", + system: ["You are helpful."], + messages: [{ role: "tool", content: "[tool_result]\ncall_id: call_view\nname: view_image\nis_error: false\noutput:" }], + rawMessages: [ + { role: "user", content: "describe the image", timestamp: 1 }, + { + role: "assistant", + model: "cursor/grok-4.5", + timestamp: 2, + content: [{ type: "toolCall", id: "call_view", name: "view_image", arguments: { path: "/tmp/x.png" } }], + }, + { + role: "toolResult", + toolCallId: "call_view", + toolName: "view_image", + content: [{ type: "image", imageUrl, detail: "auto" }], + isError: false, + timestamp: 3, + }, + ], + }); + // Tool-result image promotion is out of scope in this slice: no McpImageContent on the wire. + expect(anyMcpImageContent(viewBytes)).toBe(false); + // Image bytes must never be serialized into text. Scan the whole encoded frame. + const base64Payload = imageUrl.slice(imageUrl.indexOf(",") + 1); + expect(new TextDecoder().decode(viewBytes)).not.toContain(base64Payload); + expect(new TextDecoder().decode(viewBytes)).not.toContain("data:image/png;base64,"); + const blobText = hydratedTurnText(viewBytes); + expect(blobText).not.toContain(base64Payload); + expect(blobText).not.toContain("data:image/png;base64,"); + expect(blobText).toContain(CURSOR_VISION_IMAGE_HISTORY_MARKER); + }); +}); + +function hydratedTurnText(bytes: Uint8Array): string { + const msg = fromBinary(AgentClientMessageSchema, bytes); + const run = msg.message.case === "runRequest" ? msg.message.value : undefined; + const parts: string[] = []; + for (const turnId of run?.conversationState?.turns ?? []) { + const turn = fromBinary(ConversationTurnStructureSchema, blobData(turnId)); + if (turn.turn.case !== "agentConversationTurn") continue; + parts.push(new TextDecoder().decode(blobData(turn.turn.value.userMessage))); + for (const stepId of turn.turn.value.steps) { + parts.push(new TextDecoder().decode(blobData(stepId))); + } + } + return parts.join("\n"); +} diff --git a/tests/helpers/cursor-grumpy-fixture.png b/tests/helpers/cursor-grumpy-fixture.png new file mode 100644 index 0000000000..a06cdc84a2 Binary files /dev/null and b/tests/helpers/cursor-grumpy-fixture.png differ diff --git a/tests/oauth-provider-reconcile.test.ts b/tests/oauth-provider-reconcile.test.ts index 8bbc3d0510..5027076633 100644 --- a/tests/oauth-provider-reconcile.test.ts +++ b/tests/oauth-provider-reconcile.test.ts @@ -6,6 +6,8 @@ import { loadConfig } from "../src/config"; import { OAUTH_PROVIDERS, reconcileOAuthProviders, upsertOAuthProvider } from "../src/oauth"; import { getCredential, saveCredential } from "../src/oauth/store"; import { routeModel } from "../src/router"; +import { CURSOR_NO_VISION_MODELS, CURSOR_STATIC_MODELS, cursorModelIds } from "../src/adapters/cursor/discovery"; +import { modelInList } from "../src/types"; import type { OcxConfig } from "../src/types"; const originalHome = process.env.OPENCODEX_HOME; @@ -18,6 +20,33 @@ afterEach(() => { }); describe("OAuth provider reconciliation", () => { + test("heals a stale Cursor all-models noVisionModels stamp down to the curated list", () => { + const home = mkdtempSync(join(tmpdir(), "ocx-cursor-novision-reconcile-")); + homes.push(home); + process.env.OPENCODEX_HOME = home; + const preset = OAUTH_PROVIDERS.cursor.providerConfig; + const stale = cursorModelIds(CURSOR_STATIC_MODELS); + expect(stale.length).toBeGreaterThan((preset.noVisionModels ?? []).length); + const config = { + port: 10100, + defaultProvider: "cursor", + providers: { + cursor: { + ...structuredClone(preset), + authMode: "oauth", + noVisionModels: [...stale], + }, + }, + } satisfies OcxConfig; + + expect(reconcileOAuthProviders(config)).toBe(true); + expect(config.providers.cursor.noVisionModels).toEqual(preset.noVisionModels); + expect(config.providers.cursor.noVisionModels).toEqual([...CURSOR_NO_VISION_MODELS]); + expect(config.providers.cursor.noVisionModels).not.toContain("grok-4.5"); + expect(config.providers.cursor.noVisionModels).toContain("auto"); + expect(modelInList(config.providers.cursor.noVisionModels, "composer-2.5")).toBe(true); + expect(reconcileOAuthProviders(config)).toBe(false); + }); test("refreshes a saved Antigravity 3.5 preset without touching credentials or user fields", async () => { const home = mkdtempSync(join(tmpdir(), "ocx-gemini-36-reconcile-")); homes.push(home); diff --git a/tests/provider-registry-parity.test.ts b/tests/provider-registry-parity.test.ts index 850409d8df..3b7d2b9e19 100644 --- a/tests/provider-registry-parity.test.ts +++ b/tests/provider-registry-parity.test.ts @@ -1,5 +1,6 @@ import { describe, expect, test } from "bun:test"; import { buildCatalogEntries } from "../src/codex/catalog"; +import { CURSOR_NO_VISION_MODELS } from "../src/adapters/cursor/discovery"; import { getModelMetadata, resolveMetadataProvider } from "../src/generated/model-metadata"; import { buildInitProviders } from "../src/cli/init"; import { OAUTH_PROVIDERS } from "../src/oauth"; @@ -653,6 +654,13 @@ describe("provider registry parity", () => { expect(seed.modelContextWindows?.["gpt-5.6-luna"]).toBe(1_000_000); expect(seed.modelReasoningEfforts?.["gpt-5.5"]).toEqual(["low", "medium", "high"]); expect(seed.modelReasoningEfforts?.["gpt-5.6-sol"]).toEqual(["low", "medium", "high", "xhigh", "max"]); + expect(cursor?.noVisionModels).toEqual([...CURSOR_NO_VISION_MODELS]); + expect(seed.noVisionModels).toEqual([...CURSOR_NO_VISION_MODELS]); + expect(seed.noVisionModels).toContain("composer-2.5"); + expect(seed.noVisionModels).toContain("glm-5.3"); + expect(seed.noVisionModels).not.toContain("grok-4.5"); + expect(seed.modelInputModalities?.auto).toEqual(["text", "image"]); + expect(seed.modelInputModalities?.["composer-2.5"]).toEqual(["text", "image"]); const savedCursor: OcxProviderConfig = { adapter: "cursor", baseUrl: "https://api2.cursor.sh" }; enrichProviderFromCatalog("cursor", savedCursor);