diff --git a/.changeset/post-merge-followups.md b/.changeset/post-merge-followups.md new file mode 100644 index 0000000..a80ca1a --- /dev/null +++ b/.changeset/post-merge-followups.md @@ -0,0 +1,27 @@ +--- +'@ts-dspy/anthropic': minor +'@ts-dspy/gemini': minor +'@ts-dspy/openai': minor +'@ts-dspy/core': minor +--- + +Close the gaps left where the 0.6 features met each other. + +Images now reach the model through `Predict` and `ChainOfThought`. A signature +declaring an `image` input previously had it flattened to an `[image: …]` +placeholder before the request was built, so the model never saw the picture; +the prompt now travels as chat content whenever a field is declared `image`, +and as a plain string otherwise. Structured output over an image asks for the +schema in the prompt, since the provider methods that constrain decoding accept +only a string. + +Every provider now overrides `cacheScope()`. Two clients differing only in +`maxTokens`, `safetySettings`, `baseURL`, or declared capabilities hashed to the +same cache key, so one could be served a reply the other's configuration would +never have produced. + +`AnthropicRefusalError` is a subclass of `ContentFilterError` rather than an +alias of it. As an alias, `instanceof AnthropicRefusalError` also matched OpenAI +and Gemini content filters; as a subclass, a cross-provider `catch` on +`ContentFilterError` still works and narrowing to Anthropic means Anthropic +again. diff --git a/.prettierignore b/.prettierignore index d62a0e2..18251e5 100644 --- a/.prettierignore +++ b/.prettierignore @@ -5,3 +5,4 @@ package-lock.json CHANGELOG.md .changeset site/ +.claude diff --git a/eslint.config.js b/eslint.config.js index 21b9b31..7084578 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -3,7 +3,16 @@ import tseslint from 'typescript-eslint'; export default tseslint.config( { - ignores: ['site/**', '**/dist/**', '**/node_modules/**', '**/coverage/**', '**/*.d.ts'], + ignores: [ + 'site/**', + // Git worktrees live under .claude/, and linting another + // branch's checkout is never what you meant. + '.claude/**', + '**/dist/**', + '**/node_modules/**', + '**/coverage/**', + '**/*.d.ts', + ], }, js.configs.recommended, ...tseslint.configs.recommended, diff --git a/packages/anthropic/src/anthropic-lm.test.ts b/packages/anthropic/src/anthropic-lm.test.ts index f9ca144..ac0e9d2 100644 --- a/packages/anthropic/src/anthropic-lm.test.ts +++ b/packages/anthropic/src/anthropic-lm.test.ts @@ -152,7 +152,7 @@ describe('AnthropicLM', () => { ); }); - it('throws the shared ContentFilterError, which AnthropicRefusalError now aliases', async () => { + it('throws an AnthropicRefusalError, which is a ContentFilterError', async () => { mocks.create.mockResolvedValue({ content: [], stop_reason: 'refusal', @@ -163,7 +163,11 @@ describe('AnthropicLM', () => { const error = await new AnthropicLM({ apiKey: 'k' }).generate('Hi').catch((e) => e); expect(error).toBeInstanceOf(ContentFilterError); - expect(AnthropicRefusalError).toBe(ContentFilterError); + // A subclass, not an alias: narrowing to AnthropicRefusalError has to + // keep meaning "Anthropic", while a cross-provider catch on + // ContentFilterError still works. + expect(AnthropicRefusalError.prototype).toBeInstanceOf(ContentFilterError); + expect(AnthropicRefusalError).not.toBe(ContentFilterError); expect(error.provider).toBe('anthropic'); }); }); @@ -723,3 +727,20 @@ describe('AnthropicLM', () => { }); }); }); + +describe('cache scoping', () => { + it('keys two differently configured clients apart', () => { + // Before cacheScope() was overridden here, these two hashed identically, + // so a reply truncated at 64 tokens could be served to a client that + // allows 8192. + const scopeOf = (lm: AnthropicLM) => + JSON.stringify((lm as unknown as { cacheScope(): unknown }).cacheScope()); + + expect(scopeOf(new AnthropicLM({ apiKey: 'k', maxTokens: 64 }))).not.toBe( + scopeOf(new AnthropicLM({ apiKey: 'k', maxTokens: 8192 })) + ); + expect( + scopeOf(new AnthropicLM({ apiKey: 'k', baseURL: 'https://a.example' })) + ).not.toBe(scopeOf(new AnthropicLM({ apiKey: 'k', baseURL: 'https://b.example' }))); + }); +}); diff --git a/packages/anthropic/src/anthropic-lm.ts b/packages/anthropic/src/anthropic-lm.ts index 1114f9c..10cf1ee 100644 --- a/packages/anthropic/src/anthropic-lm.ts +++ b/packages/anthropic/src/anthropic-lm.ts @@ -51,21 +51,20 @@ export interface AnthropicConfig { /** * Raised when Claude's safety classifiers decline a request. * - * @deprecated Renamed to `ContentFilterError` in `@ts-dspy/core`, which every - * provider now throws for the same condition. This is an alias of that class, - * not a subclass of it, so two things changed: the constructor now takes - * `(provider, message, options)` rather than `(category, explanation)`, and an - * `instanceof` check now also matches an OpenAI or Gemini content filter. Check - * `error.provider === 'anthropic'` if you need to tell them apart. The alias - * will be removed in a future release. + * @deprecated Prefer `ContentFilterError` from `@ts-dspy/core`, which every + * provider throws for the same condition. This is a **subclass** of it, so + * `catch (e) { if (e instanceof ContentFilterError) … }` handles all three + * providers while `instanceof AnthropicRefusalError` still means Anthropic + * specifically. The constructor did change, though: it now takes + * `(provider, message, options)` rather than `(category, explanation)`. This + * subclass will be removed in a future release. */ -export const AnthropicRefusalError = ContentFilterError; -/** @deprecated Renamed to `ContentFilterError` in `@ts-dspy/core`. */ -export type AnthropicRefusalError = ContentFilterError; +export class AnthropicRefusalError extends ContentFilterError {} export class AnthropicLM extends BaseLM { private readonly client: Anthropic; private readonly defaultMaxTokens: number; + private readonly baseURL?: string; constructor(config: AnthropicConfig = {}) { super('anthropic', config.model ?? DEFAULT_ANTHROPIC_MODEL); @@ -77,6 +76,15 @@ export class AnthropicLM extends BaseLM { maxRetries: config.maxRetries, }); this.defaultMaxTokens = config.maxTokens ?? DEFAULT_MAX_TOKENS; + this.baseURL = config.baseURL; + } + + /** + * `maxTokens` is part of the request, so a client that truncates at 64 + * tokens must not serve a cache entry recorded by one that allows 8192. + */ + protected cacheScope(): unknown { + return { maxTokens: this.defaultMaxTokens, baseURL: this.baseURL ?? null }; } async chat(messages: ChatMessage[], options?: LLMCallOptions): Promise { @@ -283,7 +291,7 @@ export class AnthropicLM extends BaseLM { const category = details?.category ?? undefined; const explanation = details?.explanation ?? undefined; - throw new ContentFilterError( + throw new AnthropicRefusalError( 'anthropic', `Request was declined by safety classifiers${category ? ` (${category})` : ''}` + `${explanation ? `: ${explanation}` : ''}`, diff --git a/packages/core/src/evaluate/evaluate.ts b/packages/core/src/evaluate/evaluate.ts index f0a6262..9645182 100644 --- a/packages/core/src/evaluate/evaluate.ts +++ b/packages/core/src/evaluate/evaluate.ts @@ -1,4 +1,5 @@ import { type Example } from '../core/example'; +import { mapWithConcurrency } from '../utils/pool'; import { type Prediction } from '../core/prediction'; import { getDefaultLM } from '../core/config'; import type { ILanguageModel, LLMCallOptions, UsageStats } from '../types/language-model'; @@ -16,33 +17,33 @@ const DEFAULT_CONCURRENCY = 4; /** * Run `worker` over `items` with at most `limit` in flight, preserving input - * order in the returned array. + * order. * - * Small and local on purpose: an evaluation needs no more than this, and - * `worker` is expected never to reject. + * Delegates to the shared pool in `utils/pool`; the clamp lives here because + * evaluation treats a computed `0` as "one at a time" — a caller deriving the + * limit from a rate-limit budget must not get the default four — whereas the + * shared pool rejects a non-positive limit outright. */ -async function mapWithConcurrency( +async function runPooled( items: readonly T[], limit: number, worker: (item: T, index: number) => Promise ): Promise { - const results = new Array(items.length); - // Only a missing or unusable limit falls back to the default: a caller who - // computed `0` from a rate-limit budget must not get four in flight. + if (items.length === 0) return []; + const requested = Number.isFinite(limit) ? Math.floor(limit) : DEFAULT_CONCURRENCY; const lanes = Math.min(Math.max(1, requested), items.length); - let cursor = 0; - const runners = Array.from({ length: lanes }, async () => { - while (cursor < items.length) { - const index = cursor; - cursor += 1; - results[index] = await worker(items[index], index); - } + const settled = await mapWithConcurrency([...items], (item, index) => worker(item, index), { + concurrency: lanes, }); - await Promise.all(runners); - return results; + return settled.map((result) => { + // The evaluation worker captures its own failures, so a rejection here + // is a bug in this module rather than a bad example. + if (result.status === 'rejected') throw result.reason; + return result.value as R; + }); } function toError(cause: unknown): Error { @@ -183,7 +184,7 @@ export async function evaluate( const before = lm?.getUsage(); const startedAt = Date.now(); - const results = await mapWithConcurrency(prepared, concurrency, async (example, index) => { + const results = await runPooled(prepared, concurrency, async (example, index) => { const exampleStartedAt = Date.now(); let inputs: Record = {}; // Held outside the try so a metric that throws still reports what the diff --git a/packages/core/src/modules/predict.test.ts b/packages/core/src/modules/predict.test.ts index eed9634..5690e46 100644 --- a/packages/core/src/modules/predict.test.ts +++ b/packages/core/src/modules/predict.test.ts @@ -1,5 +1,5 @@ import { Predict } from './predict'; -import { Signature, InputField, OutputField } from '../core/signature'; +import { Signature, InputField, OutputField, ImageField } from '../core/signature'; import { ValidationError } from '../core/errors'; import { MockLM } from '../test-utils'; @@ -302,3 +302,44 @@ describe('Predict', () => { await expect(predict.forward({})).rejects.toThrow('No signature provided'); }); }); + +describe('image inputs', () => { + const PNG = 'iVBORw0KGgo='; + + class DescribeImage extends Signature { + static description = 'Describe the picture.'; + + @ImageField({ description: 'the picture' }) + picture!: string; + + @OutputField({ description: 'what it shows' }) + caption!: string; + } + + it('sends the image as content rather than a placeholder', async () => { + const lm = new MockLM({ responses: ['caption: a cat'] }); + + const result = await new Predict(DescribeImage, lm).forward({ + picture: `data:image/png;base64,${PNG}`, + }); + + expect(result.caption).toBe('a cat'); + + // Predict used to flatten the image to "[image: image/png]", so the + // model never actually saw it. + const content = lm.calls.at(-1)?.messages[0]?.content; + expect(Array.isArray(content)).toBe(true); + expect(content).toContainEqual({ + type: 'image', + source: { kind: 'base64', mediaType: 'image/png', data: PNG }, + }); + }); + + it('leaves a text-only prompt as a plain string', async () => { + const lm = new MockLM({ responses: ['answer: Paris\nconfidence: 0.9'] }); + + await new Predict(QA, lm).forward({ question: 'Capital of France?' }); + + expect(typeof lm.calls.at(-1)?.messages[0]?.content).toBe('string'); + }); +}); diff --git a/packages/core/src/modules/predict.ts b/packages/core/src/modules/predict.ts index b0ffecd..11c6e87 100644 --- a/packages/core/src/modules/predict.ts +++ b/packages/core/src/modules/predict.ts @@ -2,8 +2,15 @@ import { Module } from '../core/module'; import { Prediction } from '../core/prediction'; import { type Signature, type SignatureLike, type SignatureSource } from '../core/signature'; import { type Example } from '../core/example'; -import type { ChatMessage, ILanguageModel, LLMCallOptions } from '../types/language-model'; -import { parseOutput, buildPrompt } from '../utils/parsing'; +import { contentToText, textPart } from '../utils/content'; +import type { + ChatMessage, + ContentPart, + ILanguageModel, + LLMCallOptions, + MessageContent, +} from '../types/language-model'; +import { parseOutput, buildPrompt, buildPromptContent } from '../utils/parsing'; import { buildOutputSchema, buildOutputJsonSchema, @@ -188,7 +195,7 @@ export class Predict< options?: LLMCallOptions ): Promise & TOutput> { const prediction = await this.traced(inputs, async (span) => { - const prompt = this.buildPrompt(inputs); + const prompt = this.buildPromptContent(inputs); return (await this.complete(prompt, options, span)) as TOutput; }); @@ -362,13 +369,13 @@ export class Predict< * `span` is supplied when tracing is on, and records the call either way. */ protected async complete( - prompt: string, + prompt: MessageContent, options?: LLMCallOptions, span?: TraceSpan ): Promise> { const repairAttempts = normaliseRepairAttempts(options?.repairAttempts); const structured = this.lm.getCapabilities().supportsStructuredOutput; - let attemptPrompt = prompt; + let attemptPrompt: MessageContent = prompt; let previousError: ValidationError | undefined; for (let attempt = 0; ; attempt++) { @@ -388,11 +395,14 @@ export class Predict< // Repair from the original prompt, not the previous repair prompt, // so successive attempts do not stack up every earlier correction. - attemptPrompt = buildRepairPrompt( - prompt, - error, - structured ? 'structured' : 'text' - ); + const format = structured ? 'structured' : 'text'; + attemptPrompt = + typeof prompt === 'string' + ? buildRepairPrompt(prompt, error, format) + : // Rebuilding a content prompt would re-send the image on + // every attempt. Appending the correction as a trailing + // text part keeps the parts, and their order, intact. + [...prompt, textPart(`\n${buildRepairPrompt('', error, format)}`)]; } } } @@ -404,13 +414,20 @@ export class Predict< * round trip that was paid for rather than only the one that succeeded. */ protected async completeOnce( - prompt: string, + prompt: MessageContent, structured: boolean, options?: LLMCallOptions, span?: TraceSpan ): Promise> { const signature = this.requireSignature(); + // A prompt carrying images has to travel as chat content: neither + // `generate` nor `generateStructured` takes anything but a string, and + // flattening would reduce the image to its placeholder. + if (typeof prompt !== 'string') { + return this.completeFromContent(prompt, structured, options, span); + } + if (structured) { const schema = buildOutputJsonSchema(signature); span?.startCall(prompt); @@ -429,6 +446,34 @@ export class Predict< return parseOutput(signature, rawOutput); } + /** + * The multimodal path: one `chat` turn carrying text and image parts. + * + * Native structured output is unavailable here because the provider methods + * that constrain decoding take a string prompt, so the schema is requested + * in the prompt instead — the same fallback `BaseLM.generateStructured` uses + * for providers without a JSON-schema mode. Validation is identical either + * way. + */ + protected async completeFromContent( + content: ContentPart[], + structured: boolean, + options?: LLMCallOptions, + span?: TraceSpan + ): Promise> { + const signature = this.requireSignature(); + const parts = structured ? withSchemaInstruction(content, signature) : content; + + span?.startCall(contentToText(parts)); + const rawOutput = await this.lm.chat([{ role: 'user', content: parts }], options); + span?.endCall(rawOutput); + + if (!structured) { + return parseOutput(signature, rawOutput); + } + return this.validateStructured(extractJsonObject(rawOutput)); + } + /** Validate a provider's structured response against the signature. */ protected validateStructured(raw: Record): Record { const signature = this.requireSignature(); @@ -461,6 +506,16 @@ export class Predict< return this.signature; } + /** + * The prompt as chat content: a plain string when every input is text, and + * content parts when a field is declared `image`, so the image reaches the + * provider instead of the `[image: …]` placeholder a string is limited to. + */ + protected buildPromptContent(inputs: Record): MessageContent { + const format = this.lm.getCapabilities().supportsStructuredOutput ? 'json' : 'labelled'; + return buildPromptContent(this.requireSignature(), inputs, this.demos, { format }); + } + protected buildPrompt(inputs: Record): string { // Demos must demonstrate the shape the reply will actually take. A // provider with native structured output has its decoding constrained to @@ -559,3 +614,49 @@ function readPartialJsonFields(buffer: string): Record { } return fields; } + +/** Append the JSON-schema request as a trailing text part. */ +function withSchemaInstruction( + content: ContentPart[], + signature: SignatureLike +): ContentPart[] { + const schema = buildOutputJsonSchema(signature); + return [ + ...content, + textPart( + `\n\nRespond with JSON matching this schema. ` + + `Output only the JSON object, with no surrounding prose or code fences.\n` + + `${JSON.stringify(schema, null, 2)}` + ), + ]; +} + +/** Read a JSON object out of a reply, tolerating code fences and stray prose. */ +function extractJsonObject(raw: string): Record { + const fenced = raw.match(/```(?:json)?\s*([\s\S]*?)```/i); + const candidate = (fenced?.[1] ?? raw).trim(); + try { + return JSON.parse(candidate) as Record; + } catch { + const start = candidate.indexOf('{'); + const end = candidate.lastIndexOf('}'); + if (start !== -1 && end > start) { + try { + return JSON.parse(candidate.slice(start, end + 1)) as Record; + } catch { + // fall through + } + } + throw new ValidationError( + [ + { + field: '(root)', + expected: 'object', + received: raw, + message: 'model did not return a JSON object', + }, + ], + raw + ); + } +} diff --git a/packages/gemini/src/gemini-lm.ts b/packages/gemini/src/gemini-lm.ts index 85ab88b..b6e0c4e 100644 --- a/packages/gemini/src/gemini-lm.ts +++ b/packages/gemini/src/gemini-lm.ts @@ -105,6 +105,8 @@ export class GeminiLM extends BaseLM { private readonly safetySettings: SafetySetting[]; private readonly timeout?: number; private readonly maxRetries: number; + /** Constructor settings that change what a given prompt returns. */ + private readonly scope: Record; constructor(config: GeminiConfig = {}) { super('gemini', config.model ?? DEFAULT_GEMINI_MODEL); @@ -121,6 +123,22 @@ export class GeminiLM extends BaseLM { // 2 is what the OpenAI and Anthropic SDKs default to; matching them is // the point of running a retry loop here at all. this.maxRetries = config.maxRetries ?? 2; + this.scope = { + safetySettings: this.safetySettings, + vertexai: config.vertexai ?? null, + project: config.project ?? null, + location: config.location ?? null, + baseUrl: config.baseUrl ?? null, + }; + } + + /** + * Safety settings decide whether a reply comes back at all, and the Vertex + * and base-URL settings decide which endpoint answered, so none of them can + * be left out of the cache key. + */ + protected cacheScope(): unknown { + return this.scope; } async chat(messages: ChatMessage[], options?: LLMCallOptions): Promise { diff --git a/packages/openai/src/openai-compatible-lm.ts b/packages/openai/src/openai-compatible-lm.ts index 0399003..4f9d5ea 100644 --- a/packages/openai/src/openai-compatible-lm.ts +++ b/packages/openai/src/openai-compatible-lm.ts @@ -137,6 +137,16 @@ export class OpenAICompatibleLM extends OpenAILM { }; } + /** + * The declared capabilities decide whether a strict `json_schema` request is + * sent at all, so two clients pointed at the same endpoint with different + * capability declarations ask different questions and must not share cache + * entries. + */ + protected cacheScope(): unknown { + return { ...this.scope, capabilities: this.capabilities }; + } + /** * Native JSON-schema mode when the endpoint has it, prompt-based JSON when * it does not. Calling this directly must not send a `response_format` the diff --git a/packages/openai/src/openai-lm.ts b/packages/openai/src/openai-lm.ts index f100905..f8e3bd6 100644 --- a/packages/openai/src/openai-lm.ts +++ b/packages/openai/src/openai-lm.ts @@ -83,10 +83,17 @@ function supportsVisionFor(model: string): boolean { export class OpenAILM extends BaseLM { private readonly client: OpenAI; + /** Constructor settings that change what a given prompt returns. */ + protected readonly scope: Record; constructor(config: OpenAIConfig = {}) { super('openai', config.model ?? DEFAULT_OPENAI_MODEL); + this.scope = { + baseURL: config.baseURL ?? null, + organization: config.organization ?? null, + project: config.project ?? null, + }; this.client = new OpenAI({ apiKey: config.apiKey, organization: config.organization, @@ -97,6 +104,15 @@ export class OpenAILM extends BaseLM { }); } + /** + * A proxy or an alternate organization can answer the same prompt + * differently, so two clients configured that way must not share cache + * entries. + */ + protected cacheScope(): unknown { + return this.scope; + } + async chat(messages: ChatMessage[], options?: LLMCallOptions): Promise { return (await this.chatWithTools(messages, options)).content; } diff --git a/site/docs.html b/site/docs.html index 80e15c7..9a8a93e 100644 --- a/site/docs.html +++ b/site/docs.html @@ -1692,18 +1692,26 @@

Image input fields

// string signatures take a type too buildPromptContent('photo: image, question -> answer', { photo, question }) -
- Modules send text +
+ Strings stay strings

buildPromptContent() returns a plain string when every input is text, byte for byte what buildPrompt() - produces. buildPrompt() always returns a - string, rendering an image input as an - [image: image/png] placeholder — and that placeholder - is what Predict, ChainOfThought and - RespAct send, because they build string prompts. Until the - modules carry content parts, send images through lm.chat() - yourself. + produces; it returns content parts only once a field declared + image is supplied. Predict and + ChainOfThought follow the same rule, so a text-only program + sends exactly what it always did. +

+
+ +
+ Structured output with images +

+ The provider methods that constrain decoding to a schema take a string + prompt, so a prompt carrying an image asks for the schema in the prompt + instead — the same fallback used for providers with no JSON-schema + mode. Validation is identical either way; only the guarantee that the + model cannot emit the wrong shape is lost.

diff --git a/site/examples.html b/site/examples.html index 9f88f7d..a04371a 100644 --- a/site/examples.html +++ b/site/examples.html @@ -349,21 +349,12 @@

Testing without a network

the shapes you fear, not the one you hope for. 

import { describe, it, expect } from 'vitest'
-import { BaseLM, Predict, ValidationError } from '@ts-dspy/core'
-
-class StubLM extends BaseLM {
-  constructor(private reply: string) { super({ model: 'stub' }) }
-  async chat() { return this.reply }
-  getCapabilities() {
-    return { supportsStructuredOutput: false, supportsStreaming: false,
-             supportsFunctionCalling: false, supportsVision: false,
-             maxContextLength: 4096, supportedFormats: ['text'] }
-  }
-}
+import { Predict, ValidationError } from '@ts-dspy/core'
+import { MockLM } from '@ts-dspy/core/testing'
 
 describe('TriageTicket', () => {
   it('coerces a numeric string', async () => {
-    const lm = new StubLM('category: bug\nconfidence: 0.82\nurgent: yes')
+    const lm = new MockLM({ responses: ['category: bug\nconfidence: 0.82\nurgent: yes'] })
     const out = await new Predict(TriageTicket, lm).forward({ ticket: 'x' })
 
     expect(out.confidence).toBe(0.82)
@@ -372,7 +363,7 @@ 
Testing without a network

   })
 
   it('rejects prose where a number was declared', async () => {
-    const lm = new StubLM('category: bug\nconfidence: very high\nurgent: yes')
+    const lm = new MockLM({ responses: ['category: bug\nconfidence: very high\nurgent: yes'] })
 
     await expect(new Predict(TriageTicket, lm).forward({ ticket: 'x' }))
       .rejects.toBeInstanceOf(ValidationError)