feat: vision and multimodal input across all providers

.changeset/vision-multimodal-input.md

@@ -0,0 +1,34 @@
+---
+'@ts-dspy/anthropic': minor
+'@ts-dspy/gemini': minor
+'@ts-dspy/openai': minor
+'@ts-dspy/core': minor
+---
+
+Send images, not just text. `ChatMessage.content` is widened from `string` to
+`string | ContentPart[]`, where a `ContentPart` is either text or an image
+carried as an `https://` URL, a `data:` URI, or base64 plus a media type. Plain
+strings remain valid content and behave exactly as before, so text-only code —
+`generate()`, `generateStructured()`, and every module — is untouched.
+
+Signature inputs can now be declared as images with `@ImageField` (or the
+`image` type in a string signature), and the new `buildPromptContent()` renders
+such a signature as content parts, returning a plain string when every input is
+text. `buildPrompt()` still returns a string, rendering an image input as an
+`[image: image/png]` placeholder.
+
+Each provider maps parts onto its own SDK shape: OpenAI `image_url` parts (only
+on user turns, since system and assistant messages accept text alone),
+Anthropic `image` blocks with a base64 or URL source, and Gemini `inlineData` or
+`fileData`. Anthropic's merging of consecutive same-role turns now concatenates
+block arrays rather than strings; it previously merged only when both turns were
+strings, which silently skipped the merge for image turns and produced two
+adjacent user messages that the Messages API rejects.
+
+Widening `ChatMessage.content` is a breaking change to a public type — code
+that treats it as a `string` without narrowing will need a narrowing step. Per
+the pre-1.0 convention this ships as a minor.
+
+`supportsVision` is reported per model rather than hardcoded to `true`: false
+for `gpt-3.5`, `o1-mini` and `o3-mini`, for `claude-3-5-haiku` and older Claude
+models, and for Gemini embedding models.

README.md

@@ -457,6 +457,39 @@ Tool descriptions are what the model uses to decide when to call each tool, so
 they earn the detail. Never pass model output to `eval()` — see
 [`examples/utils.ts`](examples/utils.ts) for a bounded arithmetic evaluator.
 
+### Images
+
+`ChatMessage.content` is `string | ContentPart[]`, so one message can carry text
+and images together. Declare an image input with `@ImageField` — or the `image`
+type in a string signature — and render the message with `buildPromptContent`:
+
+```ts
+import { Signature, ImageField, OutputField, buildPromptContent } from '@ts-dspy/core';
+
+class ReadSign extends Signature {
+  static description = 'Read the sign in the photo.';
+
+  @ImageField({ description: 'photo of the sign' })
+  photo!: string;
+
+  @OutputField({ description: 'the words on the sign' })
+  words!: string;
+}
+
+const content = buildPromptContent(ReadSign, { photo: 'data:image/png;base64,...' });
+await lm.chat([{ role: 'user', content }]);
+```
+
+`imagePart()` accepts an `https://` URL, a `data:` URI, or an explicit
+`{ kind: 'base64', data, mediaType }` source. OpenAI receives `image_url` parts,
+Anthropic `image` blocks, and Gemini `inlineData` (Gemini fetches no arbitrary
+web URLs, so pass bytes or a Files API URI). Only user turns can carry an image,
+so system and assistant content is flattened to text. Modules still build string
+prompts, where an image input renders as an `[image: image/png]` placeholder —
+send images through `lm.chat()` for now. A plain string content behaves exactly
+as it did, and `supportsVision` is reported per model instead of being hardcoded
+to `true`.
+
 ### Testing
 
 `@ts-dspy/core/testing` ships the test doubles the library's own suite uses, so

packages/anthropic/src/anthropic-lm.test.ts

@@ -5,6 +5,8 @@ import {
     LMError,
     RateLimitError,
     TimeoutError,
+    imagePart,
+    textPart,
 } from '@ts-dspy/core';
 import {
     AnthropicLM,
@@ -66,6 +68,9 @@ beforeEach(() => {
     mocks.stream.mockReset();
 });
 
+const PNG = 'iVBORw0KGgo=';
+const DATA_URI = `data:image/png;base64,${PNG}`;
+
 describe('AnthropicLM', () => {
     it('defaults to the current Opus model with no date suffix', () => {
         expect(new AnthropicLM({ apiKey: 'k' }).getModelName()).toBe(DEFAULT_ANTHROPIC_MODEL);
@@ -659,4 +664,62 @@ describe('AnthropicLM', () => {
         expect(capabilities.supportsFunctionCalling).toBe(true);
         expect(capabilities.maxContextLength).toBe(1_000_000);
     });
+
+    describe('image content', () => {
+        it('merges a text turn and an adjacent image turn into one user message', () => {
+            const { messages } = toAnthropicMessages([
+                { role: 'user', content: 'what is this?' },
+                { role: 'user', content: [imagePart(DATA_URI)] },
+                { role: 'assistant', content: 'a logo' },
+            ]);
+
+            // Two adjacent user messages would be rejected outright: the
+            // Messages API requires strict user/assistant alternation.
+            expect(messages).toHaveLength(2);
+            expect(messages[0]).toEqual({
+                role: 'user',
+                content: [
+                    { type: 'text', text: 'what is this?' },
+                    {
+                        type: 'image',
+                        source: { type: 'base64', media_type: 'image/png', data: PNG },
+                    },
+                ],
+            });
+            expect(messages[1].role).toBe('assistant');
+        });
+
+        it('sends an image as a base64 block', () => {
+            const { messages } = toAnthropicMessages([
+                { role: 'user', content: [textPart('look'), imagePart(DATA_URI)] },
+            ]);
+
+            expect(messages[0].content).toEqual([
+                { type: 'text', text: 'look' },
+                {
+                    type: 'image',
+                    source: { type: 'base64', media_type: 'image/png', data: PNG },
+                },
+            ]);
+        });
+
+        it('sends a remote image as a url block', () => {
+            const { messages } = toAnthropicMessages([
+                { role: 'user', content: [imagePart('https://example.com/a.png')] },
+            ]);
+
+            expect(messages[0].content).toEqual([
+                { type: 'image', source: { type: 'url', url: 'https://example.com/a.png' } },
+            ]);
+        });
+
+        it('flattens an image in a system message, which is text-only', () => {
+            const { system } = toAnthropicMessages([
+                { role: 'system', content: [textPart('logo: '), imagePart(DATA_URI)] },
+                { role: 'user', content: 'hi' },
+            ]);
+
+            expect(system).toBe('logo: [image: image/png]');
+        });
+    });
 });

packages/anthropic/src/anthropic-lm.ts

@@ -4,17 +4,22 @@ import {
     LMError,
     TimeoutError,
     classify,
+    contentToText,
+    normalizeImageSource,
     type ChatMessage,
     type ChatResult,
     type FinishReason,
+    type ImageContentPart,
     type LLMCallOptions,
     type ModelCapabilities,
     type StreamChunk,
     type ToolCall,
 } from '@ts-dspy/core';
 import Anthropic, { APIConnectionTimeoutError, APIError } from '@anthropic-ai/sdk';
 import type {
+    Base64ImageSource,
     ContentBlockParam,
+    ImageBlockParam,
     Message,
     MessageParam,
 } from '@anthropic-ai/sdk/resources/messages';
@@ -259,7 +264,7 @@ export class AnthropicLM extends BaseLM {
             supportsStreaming: true,
             supportsStructuredOutput: true,
             supportsFunctionCalling: true,
-            supportsVision: true,
+            supportsVision: supportsVisionFor(this.model),
             maxContextLength: 1_000_000,
             supportedFormats: ['text', 'json_schema'],
         };
@@ -303,6 +308,13 @@ function textOf(message: Message): string {
         .join('');
 }
 
+/** Claude models that take text only; every other current model reads images. */
+const TEXT_ONLY_MODELS = [/^claude-3-5-haiku/, /^claude-2/, /^claude-instant/];
+
+function supportsVisionFor(model: string): boolean {
+    return !TEXT_ONLY_MODELS.some((pattern) => pattern.test(model));
+}
+
 /** Extract the `tool_use` blocks a turn requested. Anthropic sends `input` already parsed. */
 function toolCallsOf(message: Message): ToolCall[] {
     return message.content
@@ -324,8 +336,10 @@ function toolCallsOf(message: Message): ToolCall[] {
  * Convert ts-dspy messages into the Messages API shape.
  *
  * System messages become the top-level `system` parameter — Anthropic has no
- * system role inside `messages`. Consecutive same-role turns are merged, since
- * the API requires strict alternation.
+ * system role inside `messages`, and that parameter is text-only, so an image
+ * addressed to it is flattened to its placeholder rather than silently dropped.
+ * Consecutive same-role turns are merged, since the API requires strict
+ * alternation.
  *
  * Tool traffic is structural rather than textual: an assistant turn carrying
  * tool calls becomes `text` + `tool_use` blocks, and a `tool` result turn becomes
@@ -343,7 +357,7 @@ export function toAnthropicMessages(messages: ChatMessage[]): {
 
     for (const message of messages) {
         if (message.role === 'system') {
-            systemParts.push(message.content);
+            systemParts.push(contentToText(message.content));
             continue;
         }
 
@@ -356,10 +370,14 @@ export function toAnthropicMessages(messages: ChatMessage[]): {
             continue;
         }
 
+        // Two plain strings still merge as a string. Anything block-shaped — an
+        // image, a tool call, a tool result — merges at the block level instead:
+        // concatenating those as text would destroy the structure the API needs,
+        // and refusing to merge would break its strict role alternation.
         if (typeof previous.content === 'string' && typeof content === 'string') {
             previous.content = `${previous.content}\n\n${content}`;
         } else {
-            previous.content = [...asBlocks(previous.content), ...asBlocks(content)];
+            previous.content = mergeBlocks(toBlocks(previous.content), toBlocks(content));
         }
     }
 
@@ -369,12 +387,16 @@ export function toAnthropicMessages(messages: ChatMessage[]): {
     };
 }
 
-/** A plain turn stays a string; anything carrying tool traffic becomes blocks. */
+/**
+ * A plain turn stays a string; anything carrying tool traffic or an image
+ * becomes blocks.
+ */
 function toAnthropicContent(message: ChatMessage): string | ContentBlockParam[] {
     if (message.role === 'assistant' && message.toolCalls?.length) {
         const blocks: ContentBlockParam[] = [];
-        if (message.content) {
-            blocks.push({ type: 'text', text: message.content });
+        const text = contentToText(message.content);
+        if (text) {
+            blocks.push({ type: 'text', text });
         }
         message.toolCalls.forEach((call, index) => {
             blocks.push({
@@ -392,24 +414,74 @@ function toAnthropicContent(message: ChatMessage): string | ContentBlockParam[]
     }
 
     if (message.role === 'tool' || message.role === 'function') {
+        const text = contentToText(message.content);
         // Without a `tool_use_id` there is nothing to correlate against, so the
         // result degrades to ordinary user text rather than a rejected request.
-        if (!message.toolCallId) return message.content;
+        if (!message.toolCallId) return text;
         return [
             {
                 type: 'tool_result',
                 tool_use_id: message.toolCallId,
-                content: message.content,
+                content: text,
             },
         ];
     }
 
-    return message.content;
+    // An assistant turn cannot carry an image, so only user content keeps parts.
+    if (typeof message.content === 'string') return message.content;
+    if (message.role === 'assistant') return contentToText(message.content);
+    return message.content.map((part) =>
+        part.type === 'text'
+            ? { type: 'text' as const, text: part.text }
+            : toAnthropicImage(part)
+    );
 }
 
-function asBlocks(content: string | ContentBlockParam[]): ContentBlockParam[] {
-    if (typeof content !== 'string') return content;
-    return content ? [{ type: 'text', text: content }] : [];
+/**
+ * Images are either inline base64 with an explicit media type, or a URL the API
+ * fetches. A `data:` URI handed in as a URL is rewritten to the base64 form,
+ * which is the only shape `URLImageSource` will not accept.
+ */
+function toAnthropicImage(part: ImageContentPart): ImageBlockParam {
+    const source = normalizeImageSource(part.source);
+    if (source.kind === 'url') {
+        return { type: 'image', source: { type: 'url', url: source.url } };
+    }
+    return {
+        type: 'image',
+        source: {
+            type: 'base64',
+            // The SDK narrows media_type to the four types the API accepts;
+            // ours stays open so a new one needs no core release.
+            media_type: source.mediaType as Base64ImageSource['media_type'],
+            data: source.data,
+        },
+    };
+}
+
+function toBlocks(content: MessageParam['content']): ContentBlockParam[] {
+    return typeof content === 'string' ? [{ type: 'text', text: content }] : [...content];
+}
+
+/**
+ * Concatenate two turns' blocks, folding a text block that meets another text
+ * block into one. Without the fold the two turns would abut with no separator;
+ * the blank line is what the string merge used to provide.
+ */
+function mergeBlocks(
+    left: ContentBlockParam[],
+    right: ContentBlockParam[]
+): ContentBlockParam[] {
+    const last = left.at(-1);
+    const first = right[0];
+    if (last?.type === 'text' && first?.type === 'text') {
+        return [
+            ...left.slice(0, -1),
+            { ...last, text: `${last.text}\n\n${first.text}` },
+            ...right.slice(1),
+        ];
+    }
+    return [...left, ...right];
 }
 
 /** Translate tool declarations into the Messages API request shape. */

packages/core/src/core/signature.test.ts

@@ -1,4 +1,4 @@
-import { Signature, InputField, OutputField } from './signature';
+import { Signature, ImageField, InputField, OutputField, isImageFieldType } from './signature';
 
 describe('Signature', () => {
     describe('parseStringSignature', () => {
@@ -135,6 +135,25 @@ describe('Signature', () => {
         });
     });
 
+    describe('ImageField', () => {
+        it('records an input field typed image', () => {
+            class Caption extends Signature {
+                @ImageField({ description: 'the picture' })
+                picture!: string;
+
+                @OutputField({ description: 'one sentence' })
+                caption!: string;
+            }
+
+            const picture = Caption.getInputFields().picture;
+            expect(picture.type).toBe('image');
+            expect(picture.description).toBe('the picture');
+            expect(isImageFieldType(picture.type)).toBe(true);
+            // Images are inputs only: a model replies in text.
+            expect(Caption.getOutputFields().caption.type).toBe('string');
+        });
+    });
+
     describe('decorator mode', () => {
         it('explains what to change when legacy decorators are disabled', () => {
             // Under TC39 stage-3 decorators a field decorator is called with