diff --git a/packages/core/src/attachments.ts b/packages/core/src/attachments.ts index 27a5797136..40dced2748 100644 --- a/packages/core/src/attachments.ts +++ b/packages/core/src/attachments.ts @@ -58,6 +58,20 @@ export const READ_IMAGE_TOO_LARGE_MESSAGE = `Image exceeds the ${MAX_READ_IMAGE_ export const MAX_PROVIDER_IMAGE_REQUEST_BYTES = 12 * 1024 * 1024; export const PROVIDER_IMAGE_BUDGET_EXCEEDED_MESSAGE = `Image was read, but the per-request image budget (${MAX_PROVIDER_IMAGE_REQUEST_BYTES / 1024 / 1024}MB across all images this turn) was exceeded; earlier images were sent and this one was omitted. Read fewer or smaller images.`; +/** + * Native PDF requests are Base64 encoded. Sixteen raw MiB expands to roughly + * 21.4 MiB, leaving headroom under Anthropic's 32 MiB whole-request limit for + * text, tool schemas, JSON framing, and other content. + */ +export const MAX_PROVIDER_PDF_REQUEST_BYTES = 16 * 1024 * 1024; + +/** + * Shared raw-byte ceiling across image and PDF inputs. Eighteen raw MiB + * expands to 24 MiB in Base64, preserving 8 MiB of whole-request headroom on + * the strictest verified native PDF route. + */ +export const MAX_PROVIDER_BINARY_REQUEST_BYTES = 18 * 1024 * 1024; + const MIME_BY_EXTENSION: Readonly> = { png: 'image/png', jpg: 'image/jpeg', @@ -88,10 +102,9 @@ export function guessMimeFromName(fileName: string): string { /** * Route a MIME type to an {@link AttachmentRef} kind. The runtime - * consumption split is image vs. everything-else (images become provider - * image parts; other kinds are read on demand by the model via Read), so this - * only needs to single out the kinds that change - * consumption or display. Unknown / unmapped MIME falls back to `other`. + * consumption split singles out images and PDFs (authorized routes can send + * them as provider file parts); text-like kinds are read on demand by the + * model via Read. Unknown / unmapped MIME falls back to `other`. * * `fileName` is consulted for kinds whose MIME is unreliable across OSes * (Office documents arrive as `application/octet-stream` or a long diff --git a/packages/runtime-host/src/__tests__/execution-model-composition.test.ts b/packages/runtime-host/src/__tests__/execution-model-composition.test.ts index 4507920821..7ae8fb7c54 100644 --- a/packages/runtime-host/src/__tests__/execution-model-composition.test.ts +++ b/packages/runtime-host/src/__tests__/execution-model-composition.test.ts @@ -237,6 +237,106 @@ test('backend creation does not treat aliased provider metadata as inventory', a ); }); +test('Host composition carries verified native PDF input through the provider wire', async () => { + const modelId = 'gpt-4o'; + const provider = await startProvider(); + let attachmentReads = 0; + let backend: Awaited> | undefined; + try { + backend = await createHostAiSdkBackend( + backendCreationFixture({ + abortSignal: new AbortController().signal, + modelId, + resolveExecutionConnection: async () => ({ + kind: 'ready', + connection: { + slug: 'backend-creation-connection', + providerType: 'openai', + baseUrl: provider.baseUrl, + enabledModelIds: [modelId], + models: [ + { + id: modelId, + capabilities: { chat: true, functionCalling: true }, + modalities: { input: ['text', 'image', 'pdf'], output: ['text'] }, + contextWindow: 8_192, + maxOutputTokens: 1_024, + }, + ], + }, + networkProxy: { enabled: false }, + secretMaterial: { connection: { secret: API_KEY } }, + }), + readPricing: async () => ({ revision: 0, overrides: [] }), + artifacts: { + readDurableAttachmentBinary: async ({ + artifactId, + sessionId, + }: { + artifactId: string; + sessionId: string; + }) => { + attachmentReads += 1; + assert.equal(artifactId, 'brief'); + assert.equal(sessionId, 'backend-creation-session'); + return { ok: true, base64: 'JVBERi0=', mimeType: 'application/pdf' }; + }, + } as unknown as HostAiSdkBackendInput['artifacts'], + }), + ); + + const events = []; + for await (const event of backend.send({ + invocationId: 'pdf-composition-invocation', + runId: 'pdf-composition-run', + turnId: 'pdf-composition-turn', + text: 'Read the attached PDF.', + attachments: [ + { + kind: 'pdf', + name: 'brief.pdf', + mimeType: 'application/pdf', + bytes: 8, + ref: { + kind: 'session_file', + sessionId: 'backend-creation-session', + relativePath: 'brief', + }, + }, + ], + context: [], + runtimeContext: [], + })) { + events.push(event); + } + + assert.equal( + events.find((event) => event.type === 'complete')?.stopReason, + 'end_turn', + JSON.stringify({ events, providerRequests: provider.requests }), + ); + assert.equal(attachmentReads, 1); + assert.equal(provider.requests.length, 1); + const messages = provider.requests[0]?.body.messages; + assert.ok(Array.isArray(messages)); + const filePart = messages + .flatMap((message: { content?: unknown }) => + Array.isArray(message.content) ? message.content : [], + ) + .find((part: { type?: unknown }) => part.type === 'file'); + assert.deepEqual(filePart, { + type: 'file', + file: { + filename: 'brief.pdf', + file_data: 'data:application/pdf;base64,JVBERi0=', + }, + }); + } finally { + await backend?.dispose(); + await provider.close(); + } +}); + test('provider dispatch fails closed when the Run Composition commit fails', async () => { const provider = await startProvider(); let commits = 0; @@ -3180,6 +3280,7 @@ function backendCreationFixture(input: { recordModelCallAttempt?: BackendFactoryContext['recordModelCallAttempt']; createFetchTransport?: HostAiSdkBackendInput['createFetchTransport']; createRunComposer?: HostAiSdkBackendInput['createRunComposer']; + artifacts?: HostAiSdkBackendInput['artifacts']; }): HostAiSdkBackendInput { const runtimePolicy = input.runtimePolicy ?? @@ -3252,7 +3353,7 @@ function backendCreationFixture(input: { ...(input.oauthCredentials ? { oauthCredentials: input.oauthCredentials } : {}), ...(input.claudeDeviceId ? { claudeDeviceId: input.claudeDeviceId } : {}), createRunComposer, - artifacts: {}, + artifacts: input.artifacts ?? {}, executionArtifacts: { recordToolArtifacts: async () => undefined, toolResultArchive: createToolResultArchiveCapability({ diff --git a/packages/runtime-host/src/server/execution-model-composition.ts b/packages/runtime-host/src/server/execution-model-composition.ts index c9604bd6b0..fdd3d0b2a4 100644 --- a/packages/runtime-host/src/server/execution-model-composition.ts +++ b/packages/runtime-host/src/server/execution-model-composition.ts @@ -14,6 +14,7 @@ import { buildLlmHistorySummarizer } from '@maka/runtime/history-compact-summari import { buildOpenAiCodexHistoryCompactor } from '@maka/runtime/openai-codex-history-compactor'; import { buildPricingLookup, recordToolInvocation } from '@maka/runtime/telemetry'; import { buildProviderOptions, getAIModel } from '@maka/runtime/model-factory'; +import { resolveModelPdfInputContract } from '@maka/runtime/model-runtime'; import { createProviderRequestCaptureRecorder } from '@maka/runtime/provider-request-telemetry'; import { createProxiedFetchTransport, @@ -132,6 +133,7 @@ export async function createHostAiSdkBackend(input: HostAiSdkBackendInput): Prom input.context.header.thinkingLevel, ); const contextWindow = resolveSelectedModelContextWindow(target.connection, target.model); + const pdfInputContract = resolveModelPdfInputContract(target.connection, target.model); let modelComposition: HostRunComposer; try { modelComposition = await readDuringBackendCreation( @@ -362,6 +364,7 @@ export async function createHostAiSdkBackend(input: HostAiSdkBackendInput): Prom target.model, relayModelProfile(target.connection, target.model)?.vision, ), + ...(pdfInputContract ? { pdfInputContract } : {}), readAttachmentBytes: createAttachmentByteReader({ artifactStore: input.artifacts, sessionId: input.context.sessionId, diff --git a/packages/runtime/src/__tests__/ai-sdk-backend.test.ts b/packages/runtime/src/__tests__/ai-sdk-backend.test.ts index 297aee2152..516a987256 100644 --- a/packages/runtime/src/__tests__/ai-sdk-backend.test.ts +++ b/packages/runtime/src/__tests__/ai-sdk-backend.test.ts @@ -10,7 +10,7 @@ import type { AttachmentByteReader } from '@maka/core/attachments'; import type { BackendSendInput } from '@maka/core/backend-types'; import type { LlmConnection } from '@maka/core/llm-connections'; import type { SessionHeader } from '@maka/core/session'; -import type { StorageRef } from '@maka/core/events'; +import type { AttachmentRef, StorageRef } from '@maka/core/events'; import { encodeCanonicalRuntimeEvent } from '@maka/core/canonical-runtime-event'; import type { SessionEvent } from '@maka/core/events'; import type { RuntimeEvent } from '@maka/core/runtime-event'; @@ -2221,6 +2221,344 @@ describe('AiSdkBackend model history', () => { ); }); + test('materializes PDF attachments consistently for current, RuntimeEvent, and stored replay', async () => { + const pdfBytes = new Uint8Array([0x25, 0x50, 0x44, 0x46, 0x2d, 1, 2, 3]); + const pdf = { + kind: 'pdf' as const, + name: 'brief.pdf', + mimeType: 'application/pdf', + bytes: pdfBytes.length, + ref: { kind: 'session_file' as const, sessionId: 'session-1', relativePath: 'brief' }, + }; + const cases: Array<{ name: string; input: BackendSendInput }> = [ + { + name: 'current turn', + input: { + turnId: 'turn-current', + text: 'read the current PDF', + attachments: [pdf], + context: [], + runtimeContext: [], + }, + }, + { + name: 'RuntimeEvent replay', + input: { + turnId: 'turn-current', + text: 'continue', + context: [], + runtimeContext: [ + runtimeEvent({ + id: 'rt-pdf', + turnId: 'turn-prev', + role: 'user', + author: 'user', + content: { kind: 'text', text: 'read the replayed PDF', attachments: [pdf] }, + }), + runtimeTextEvent({ + id: 'rt-answer', + turnId: 'turn-prev', + role: 'model', + author: 'agent', + text: 'noted', + }), + ], + }, + }, + { + name: 'stored-message fallback', + input: { + turnId: 'turn-current', + text: 'continue', + context: [ + { + type: 'user', + id: 'stored-user', + turnId: 'turn-prev', + ts: 1, + text: 'read the stored PDF', + attachments: [pdf], + }, + { + type: 'assistant', + id: 'stored-assistant', + turnId: 'turn-prev', + ts: 2, + text: 'noted', + modelId: 'm', + }, + ], + runtimeContext: [ + { + id: 'rt-terminal', + invocationId: 'inv-prev', + runId: 'run-prev', + sessionId: 'session-1', + turnId: 'turn-prev', + ts: 1, + partial: false, + role: 'model', + author: 'agent', + status: 'completed', + actions: { endInvocation: true }, + }, + ], + }, + }, + ]; + + for (const scenario of cases) { + const model = completionModel(); + const backend = createTestAiSdkBackend({ + sessionId: 'session-1', + header: header(), + appendMessage: async () => {}, + connection: connection(), + apiKey: 'sk-test', + modelId: 'mock-model-id', + modelFactory: () => model, + tools: [], + newId: idGenerator(), + now: monotonicClock(), + pdfInputContract: { providerType: 'openai', wire: 'openai-responses' }, + readAttachmentBytes: async () => ({ ok: true, bytes: pdfBytes }), + }); + + await drain(backend.send(scenario.input)); + + const prompt = compactPrompt(model) as Array<{ content: unknown }>; + const pdfParts = prompt + .flatMap((message) => (Array.isArray(message.content) ? message.content : [])) + .filter( + (part: any) => + part.type === 'file' && + part.mediaType === 'application/pdf' && + part.filename === 'brief.pdf', + ); + assert.equal(pdfParts.length, 1, `${scenario.name}: ${JSON.stringify(prompt)}`); + assert.deepEqual(pdfParts[0]?.data, { type: 'data', data: pdfBytes }); + } + }); + + test('never reads or sends PDF bytes without a verified input contract', async () => { + let reads = 0; + const model = completionModel(); + const backend = createTestAiSdkBackend({ + sessionId: 'session-1', + header: header(), + appendMessage: async () => {}, + connection: connection(), + apiKey: 'sk-test', + modelId: 'mock-model-id', + modelFactory: () => model, + tools: [], + newId: idGenerator(), + now: monotonicClock(), + readAttachmentBytes: async () => { + reads += 1; + return { ok: true, bytes: new Uint8Array([0xde, 0xad, 0xbe, 0xef]) }; + }, + }); + + await drain( + backend.send({ + turnId: 'turn-current', + text: 'inspect this report', + attachments: [pdfAttachment('report', 4)], + context: [], + runtimeContext: [], + }), + ); + + const prompt = compactPrompt(model) as Array<{ content: unknown }>; + const parts = prompt.flatMap((message) => + Array.isArray(message.content) ? message.content : [], + ); + assert.equal(reads, 0); + assert.equal( + parts.some((part: any) => part.type === 'file' && part.mediaType === 'application/pdf'), + false, + ); + assert.match(JSON.stringify(prompt), /report\.pdf.*application\/pdf/); + }); + + test('rejects a mislabeled PDF before reading or sending its bytes', async () => { + let reads = 0; + const model = completionModel(); + const backend = createTestAiSdkBackend({ + sessionId: 'session-1', + header: header(), + appendMessage: async () => {}, + connection: connection(), + apiKey: 'sk-test', + modelId: 'mock-model-id', + modelFactory: () => model, + tools: [], + newId: idGenerator(), + now: monotonicClock(), + pdfInputContract: { providerType: 'openai', wire: 'openai-responses' }, + readAttachmentBytes: async () => { + reads += 1; + return { ok: true, bytes: new Uint8Array([0xde, 0xad, 0xbe, 0xef]) }; + }, + }); + + await drain( + backend.send({ + turnId: 'turn-current', + text: 'inspect this report', + attachments: [{ ...pdfAttachment('mislabeled', 4), mimeType: 'application/octet-stream' }], + context: [], + runtimeContext: [], + }), + ); + + const prompt = compactPrompt(model) as Array<{ content: unknown }>; + assert.equal(reads, 0); + assert.equal(JSON.stringify(prompt).includes('3q2+7w=='), false); + assert.match(JSON.stringify(prompt), /mislabeled\.pdf.*not application\/pdf/); + }); + + test('does not charge an unavailable PDF against the PDF subtype budget', async () => { + const model = completionModel(); + const backend = createTestAiSdkBackend({ + sessionId: 'session-1', + header: header(), + appendMessage: async () => {}, + connection: connection(), + apiKey: 'sk-test', + modelId: 'mock-model-id', + modelFactory: () => model, + tools: [], + newId: idGenerator(), + now: monotonicClock(), + pdfInputContract: { providerType: 'openai', wire: 'openai-responses' }, + maxProviderPdfRequestBytes: 10, + maxProviderBinaryRequestBytes: 10, + readAttachmentBytes: async (ref: StorageRef) => + ref.kind === 'session_file' && ref.relativePath === 'missing' + ? { ok: false, reason: 'not_found' } + : { ok: true, bytes: new Uint8Array(10) }, + }); + + await drain( + backend.send({ + turnId: 'turn-current', + text: 'inspect both reports', + attachments: [pdfAttachment('missing', 10), pdfAttachment('available', 10)], + context: [], + runtimeContext: [], + }), + ); + + const prompt = compactPrompt(model) as Array<{ content: unknown }>; + const parts = prompt.flatMap((message) => + Array.isArray(message.content) ? message.content : [], + ); + assert.equal( + parts.filter((part: any) => part.type === 'file' && part.mediaType === 'application/pdf') + .length, + 1, + ); + assert.match(parts.map((part: any) => part.text ?? '').join('\n'), /missing\.pdf.*not_found/); + }); + + test('enforces the PDF subtype budget from bytes read, not attachment metadata', async () => { + const model = completionModel(); + const backend = createTestAiSdkBackend({ + sessionId: 'session-1', + header: header(), + appendMessage: async () => {}, + connection: connection(), + apiKey: 'sk-test', + modelId: 'mock-model-id', + modelFactory: () => model, + tools: [], + newId: idGenerator(), + now: monotonicClock(), + pdfInputContract: { providerType: 'openai', wire: 'openai-responses' }, + maxProviderPdfRequestBytes: 15, + maxProviderBinaryRequestBytes: 30, + readAttachmentBytes: async () => ({ ok: true, bytes: new Uint8Array(10) }), + }); + + await drain( + backend.send({ + turnId: 'turn-current', + text: 'inspect these reports', + attachments: [pdfAttachment('first', 1), pdfAttachment('second', 1)], + context: [], + runtimeContext: [], + }), + ); + + const prompt = compactPrompt(model) as Array<{ content: unknown }>; + const parts = prompt.flatMap((message) => + Array.isArray(message.content) ? message.content : [], + ); + assert.equal( + parts.filter((part: any) => part.type === 'file' && part.mediaType === 'application/pdf') + .length, + 1, + ); + assert.match( + parts.map((part: any) => part.text ?? '').join('\n'), + /1 PDF attachment.*PDF budget/, + ); + }); + + test('enforces one combined budget across image and PDF inputs', async () => { + const model = completionModel(); + const backend = createTestAiSdkBackend({ + sessionId: 'session-1', + header: header(), + appendMessage: async () => {}, + connection: connection(), + apiKey: 'sk-test', + modelId: 'mock-model-id', + modelFactory: () => model, + tools: [], + newId: idGenerator(), + now: monotonicClock(), + supportsVision: true, + pdfInputContract: { providerType: 'openai', wire: 'openai-responses' }, + maxProviderImageRequestBytes: 20, + maxProviderPdfRequestBytes: 20, + maxProviderBinaryRequestBytes: 15, + readAttachmentBytes: async () => ({ ok: true, bytes: new Uint8Array(10) }), + }); + + await drain( + backend.send({ + turnId: 'turn-current', + text: 'inspect both inputs', + attachments: [ + { + kind: 'image', + name: 'chart.png', + mimeType: 'image/png', + bytes: 10, + ref: { kind: 'session_file', sessionId: 'session-1', relativePath: 'chart' }, + }, + pdfAttachment('report', 10), + ], + context: [], + runtimeContext: [], + }), + ); + + const prompt = compactPrompt(model) as Array<{ content: unknown }>; + const parts = prompt.flatMap((message) => + Array.isArray(message.content) ? message.content : [], + ); + assert.equal(parts.filter((part: any) => part.type === 'file').length, 1); + assert.equal((parts.find((part: any) => part.type === 'file') as any)?.mediaType, 'image/png'); + assert.match( + parts.map((part: any) => part.text ?? '').join('\n'), + /1 binary attachment.*combined image\/PDF request budget/, + ); + }); + test('reports unavailable attachment reads without consuming image budget', async () => { const model = completionModel(); const backend = createTestAiSdkBackend({ @@ -13927,7 +14265,9 @@ function archiveGatedTurnEvents(suffix: 'a' | 'b', path: string, result: unknown describe('AiSdkBackend steering durability and identity', () => { const steeringBackend = ( model: MockLanguageModelV4, - options: Partial> = {}, + options: Partial< + Pick + > = {}, ): AiSdkBackend => createTestAiSdkBackend({ sessionId: 'session-1', @@ -14004,6 +14344,7 @@ describe('AiSdkBackend steering durability and identity', () => { test('persists canonical steering content and materializes attachments for the model', async () => { const model = textCompletionModel('done'); const pngBytes = new Uint8Array([137, 80, 78, 71]); + const pdfBytes = new Uint8Array([37, 80, 68, 70, 45]); const image = { kind: 'image' as const, name: 'first.png', @@ -14015,14 +14356,18 @@ describe('AiSdkBackend steering durability and identity', () => { kind: 'pdf' as const, name: 'second.pdf', mimeType: 'application/pdf', - bytes: 12, + bytes: pdfBytes.length, ref: { kind: 'session_file' as const, sessionId: 'session-1', relativePath: 'second.pdf' }, }; const backend = steeringBackend(model, { supportsVision: true, + pdfInputContract: { providerType: 'openai', wire: 'openai-responses' }, readAttachmentBytes: async (ref) => { - assert.deepEqual(ref, image.ref); - return { ok: true, bytes: pngBytes }; + if (ref.kind === 'session_file' && ref.relativePath === image.ref.relativePath) { + return { ok: true, bytes: pngBytes }; + } + assert.deepEqual(ref, document.ref); + return { ok: true, bytes: pdfBytes }; }, }); const content = { @@ -14064,7 +14409,7 @@ describe('AiSdkBackend steering durability and identity', () => { }>; assert.deepEqual( parts.map((part) => part.type), - ['text', 'file'], + ['text', 'file', 'file'], ); assert.equal( parts[0]?.text, @@ -14074,6 +14419,9 @@ describe('AiSdkBackend steering durability and identity', () => { ); assert.equal(parts[1]?.mediaType, 'image/png'); assert.notEqual(parts[1]?.data, undefined); + assert.equal(parts[2]?.mediaType, 'application/pdf'); + assert.equal((parts[2] as { filename?: string } | undefined)?.filename, 'second.pdf'); + assert.notEqual(parts[2]?.data, undefined); assert.equal(JSON.stringify(prompt).includes('human-only command'), false); }); @@ -15515,6 +15863,16 @@ function sandboxSnapshot(): SandboxDiagnosticsSnapshot { }; } +function pdfAttachment(relativePath: string, bytes: number): AttachmentRef { + return { + kind: 'pdf', + name: `${relativePath}.pdf`, + mimeType: 'application/pdf', + bytes, + ref: { kind: 'session_file', sessionId: 'session-1', relativePath }, + }; +} + function connection(): LlmConnection { return { slug: 'anthropic-main', diff --git a/packages/runtime/src/__tests__/pdf-input-contract.test.ts b/packages/runtime/src/__tests__/pdf-input-contract.test.ts new file mode 100644 index 0000000000..bb20d656e4 --- /dev/null +++ b/packages/runtime/src/__tests__/pdf-input-contract.test.ts @@ -0,0 +1,192 @@ +import assert from 'node:assert/strict'; +import { describe, test } from 'node:test'; +import type { LlmConnection } from '@maka/core/llm-connections'; +import { getAIModel } from '../model-factory.js'; +import { resolveModelPdfInputContract } from '../model-runtime.js'; + +function connection(providerType: LlmConnection['providerType'], modelId: string): LlmConnection { + return { + slug: `${providerType}-pdf-test`, + name: `${providerType} PDF test`, + providerType, + baseUrl: 'https://provider.invalid/v1', + defaultModel: modelId, + enabled: true, + createdAt: 0, + updatedAt: 0, + }; +} + +describe('native PDF input contract', () => { + test('authorizes only PDF-capable models on verified first-party provider wires', () => { + assert.deepEqual( + resolveModelPdfInputContract(connection('anthropic', 'claude-opus-4-8'), 'claude-opus-4-8'), + { providerType: 'anthropic', wire: 'anthropic-messages' }, + ); + assert.deepEqual(resolveModelPdfInputContract(connection('openai', 'gpt-4o'), 'gpt-4o'), { + providerType: 'openai', + wire: 'openai-chat', + }); + assert.deepEqual(resolveModelPdfInputContract(connection('openai', 'gpt-5.4'), 'gpt-5.4'), { + providerType: 'openai', + wire: 'openai-responses', + }); + }); + + test('does not infer PDF support for relays, subscriptions, or unknown models', () => { + const declaredPdfModel = { + id: 'relay-pdf-model', + modalities: { input: ['text', 'pdf'] as const, output: ['text'] as const }, + }; + for (const providerType of [ + 'openai-compatible', + 'openai-codex', + 'anthropic-compatible', + 'claude-subscription', + ] as const) { + const relay = { + ...connection(providerType, declaredPdfModel.id), + models: [ + { + ...declaredPdfModel, + modalities: { + input: [...declaredPdfModel.modalities.input], + output: [...declaredPdfModel.modalities.output], + }, + }, + ], + }; + assert.equal(resolveModelPdfInputContract(relay, declaredPdfModel.id), null, providerType); + } + assert.equal( + resolveModelPdfInputContract(connection('openai', 'unknown-model'), 'unknown-model'), + null, + ); + assert.equal( + resolveModelPdfInputContract( + { + ...connection('openai', 'gpt-4o'), + models: [ + { + id: 'gpt-4o', + modalities: { input: ['text'], output: ['text'] }, + }, + ], + }, + 'gpt-4o', + ), + null, + 'an explicit provider inventory must outrank generated PDF metadata', + ); + }); +}); + +describe('AI SDK PDF wire lowering', () => { + test('lowers one generic PDF file part to each verified native request shape', async () => { + const chat = await captureRequestBody('openai', 'gpt-4o'); + assert.deepEqual(requestContentPart(chat, 'messages', 1), { + type: 'file', + file: { + filename: 'brief.pdf', + file_data: 'data:application/pdf;base64,JVBERi0=', + }, + }); + + const responses = await captureRequestBody('openai', 'gpt-5.4'); + assert.deepEqual(requestContentPart(responses, 'input', 1), { + type: 'input_file', + filename: 'brief.pdf', + file_data: 'data:application/pdf;base64,JVBERi0=', + }); + + const anthropic = await captureRequestBody('anthropic', 'claude-opus-4-8'); + assert.deepEqual(requestContentPart(anthropic, 'messages', 1), { + type: 'document', + source: { type: 'base64', media_type: 'application/pdf', data: 'JVBERi0=' }, + title: 'brief.pdf', + }); + }); +}); + +function requestContentPart( + body: Record, + field: 'messages' | 'input', + partIndex: number, +): unknown { + const messages = body[field]; + assert.ok(Array.isArray(messages), `${field} must be an array`); + const firstMessage = messages[0] as { content?: unknown } | undefined; + assert.ok( + firstMessage && Array.isArray(firstMessage.content), + `${field}[0].content must be an array`, + ); + assert.ok(partIndex in firstMessage.content, `${field}[0].content[${partIndex}] must exist`); + return firstMessage.content[partIndex]; +} + +async function captureRequestBody( + providerType: 'openai' | 'anthropic', + modelId: 'gpt-4o' | 'gpt-5.4' | 'claude-opus-4-8', +): Promise> { + let requestBody: Record | undefined; + const fetch = (async (_url: string | URL | Request, init?: RequestInit) => { + requestBody = JSON.parse(String(init?.body)) as Record; + if (providerType === 'anthropic') { + return Response.json({ + id: 'msg_pdf', + type: 'message', + role: 'assistant', + model: modelId, + content: [{ type: 'text', text: 'ok' }], + stop_reason: 'end_turn', + stop_sequence: null, + usage: { input_tokens: 1, output_tokens: 1 }, + }); + } + if (modelId === 'gpt-4o') { + return Response.json({ + id: 'chat_pdf', + object: 'chat.completion', + created: 0, + model: modelId, + choices: [ + { index: 0, message: { role: 'assistant', content: 'ok' }, finish_reason: 'stop' }, + ], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + }); + } + return Response.json({ + id: 'resp_pdf', + object: 'response', + status: 'completed', + output: [], + usage: { input_tokens: 1, output_tokens: 1 }, + }); + }) as unknown as typeof globalThis.fetch; + const model = getAIModel({ + connection: connection(providerType, modelId), + apiKey: 'test-key', + modelId, + fetch, + }); + + await model.doGenerate({ + prompt: [ + { + role: 'user', + content: [ + { type: 'text', text: 'Read the PDF.' }, + { + type: 'file', + data: { type: 'data', data: new Uint8Array([0x25, 0x50, 0x44, 0x46, 0x2d]) }, + mediaType: 'application/pdf', + filename: 'brief.pdf', + }, + ], + }, + ], + }); + + assert.ok(requestBody); + return requestBody; +} diff --git a/packages/runtime/src/ai-sdk-backend.ts b/packages/runtime/src/ai-sdk-backend.ts index 316329dbd5..65ec2ab62b 100644 --- a/packages/runtime/src/ai-sdk-backend.ts +++ b/packages/runtime/src/ai-sdk-backend.ts @@ -76,10 +76,12 @@ import { YIELD_AGENT_GRAPH_TOOL_NAME, type YieldAgentGraphToolResult, } from './stream-graph-supervisor-tools.js'; -import type { AttachmentByteReader } from '@maka/core/attachments'; import { + MAX_PROVIDER_BINARY_REQUEST_BYTES, MAX_PROVIDER_IMAGE_REQUEST_BYTES, + MAX_PROVIDER_PDF_REQUEST_BYTES, PROVIDER_IMAGE_BUDGET_EXCEEDED_MESSAGE, + type AttachmentByteReader, } from '@maka/core/attachments'; import { stripUndefinedDeep } from '@maka/core/tool-args-identity'; import { pricingModelKey } from '@maka/core/usage-stats/pricing'; @@ -155,7 +157,7 @@ import { compactionDecisionDiagnosticPatch } from './compaction-boundary.js'; import type { AutomaticMemoryCompactionDecision, AutomaticMemoryCompactionDispatch, - ProviderImageBudget, + ProviderAttachmentBudget, } from './ai-sdk-compaction.js'; import { contextDiagnosticsCompactionOf, @@ -219,7 +221,11 @@ import { type MemoryExtractionSourceSnapshot, type MemoryExtractionTrigger, } from './memory-extraction.js'; -import { modelUsesNativeOpenAiResponses, resolveModelRuntime } from './model-runtime.js'; +import { + modelUsesNativeOpenAiResponses, + resolveModelRuntime, + type ModelPdfInputContract, +} from './model-runtime.js'; import { applyPatchReplayFactText, normalizeApplyPatchReplayInput, @@ -836,9 +842,10 @@ export interface AiSdkBackendInput extends AiSdkCompactionCapabilities { */ recordToolArtifacts?: ToolArtifactRecorder; /** - * Optional attachment byte reader. When set, image attachments on the current - * user turn may be rendered as provider image parts instead of placeholder text. - * Caller wires this to the session ArtifactStore; runtime never imports storage. + * Optional attachment byte reader. When set, authorized image and PDF + * attachments may be rendered as provider file parts instead of placeholder + * text. Caller wires this to the session ArtifactStore; runtime never imports + * storage. */ readAttachmentBytes?: AttachmentByteReader; /** @@ -846,7 +853,11 @@ export interface AiSdkBackendInput extends AiSdkCompactionCapabilities { * image parts; false/unknown stay as text refs with a fallback note. */ supportsVision?: boolean; + /** Verified first-party provider/wire contract for native PDF file parts. */ + pdfInputContract?: ModelPdfInputContract; maxProviderImageRequestBytes?: number; + maxProviderPdfRequestBytes?: number; + maxProviderBinaryRequestBytes?: number; /** Host-owned bounded long-term-memory extraction. Source tools are Runtime-reserved. */ memoryExtraction?: MemoryExtractionSourceCapabilities; } @@ -979,11 +990,15 @@ class TurnScope { watchdog: StreamWatchdog | null = null; runTrace: RunTrace | null = null; /** - * Image allowance for this turn, accumulated across its provider steps. Owned - * by the scope so an overlapping turn cannot spend it, and non-null for the - * scope's whole life so no path has to decide what "no budget" means. + * Binary attachment allowance for this turn, accumulated across its provider + * steps. Owned by the scope so an overlapping turn cannot spend it, and + * non-null for the scope's whole life so no path has to decide what "no + * budget" means. */ - readonly imageBudget: ProviderImageBudget = { used: 0, decisions: new Map() }; + readonly attachmentBudget: ProviderAttachmentBudget = { + used: { image: 0, pdf: 0, total: 0 }, + decisions: new Map(), + }; /** * User messages steered into this turn, drained from the caller's queue at * step boundaries. Each entry is the canonical envelope-wrapped user @@ -1105,8 +1120,8 @@ export class AiSdkBackend implements AgentBackend { modelAdapter: this.modelAdapter, createProviderRequestTracker: (trackerInput) => this.createProviderRequestTracker(trackerInput), - materializeRuntimeReplayPlan: (plan, imageBudget, checkpoint) => - this.materializeRuntimeReplayPlan(plan, imageBudget, undefined, checkpoint), + materializeRuntimeReplayPlan: (plan, attachmentBudget, checkpoint) => + this.materializeRuntimeReplayPlan(plan, attachmentBudget, undefined, checkpoint), canReplayProviderNative: (plan) => this.canReplayProviderNative(plan), appendTurnTailPrompt: (content, turnTailPrompt) => this.appendTurnTailPrompt(content, turnTailPrompt), @@ -1326,7 +1341,12 @@ export class AiSdkBackend implements AgentBackend { orchestrationMode: identity.orchestrationMode, ...(identity.invocationId ? { invocationId: identity.invocationId } : {}), materializeDefaultToolResultOutput: ({ toolCallId, output }) => - this.materializeToolResultOutput(identity.scope().imageBudget, output, false, toolCallId), + this.materializeToolResultOutput( + identity.scope().attachmentBudget, + output, + false, + toolCallId, + ), spawnChildAgent: input.spawnChildAgent, spawnChildSession: input.spawnChildSession, prepareChildAgentResume: input.prepareChildAgentResume, @@ -1854,7 +1874,7 @@ export class AiSdkBackend implements AgentBackend { const currentUserContent = input.continuation ? undefined : await this.buildCurrentUserContent( - scope.imageBudget, + scope.attachmentBudget, input.text, input.attachments, input.quotes, @@ -1938,7 +1958,7 @@ export class AiSdkBackend implements AgentBackend { }); const currentTurnMessages = await this.materializeRuntimeReplayPlan( { ...replayPlan, items: replayItems }, - scope.imageBudget, + scope.attachmentBudget, settledModelOutputs, projectionCheckpoint, ); @@ -3432,7 +3452,7 @@ export class AiSdkBackend implements AgentBackend { if (!input.runtimeContext) { return { status: 'ready', - messages: await this.materializePriorMessages(scope.imageBudget, priorStored), + messages: await this.materializePriorMessages(scope.attachmentBudget, priorStored), gate: 'stored_message_projection', diagnostics: [], }; @@ -3441,7 +3461,7 @@ export class AiSdkBackend implements AgentBackend { (event) => event.turnId !== input.turnId, ); const projectedMessages = await this.materializePriorMessages( - scope.imageBudget, + scope.attachmentBudget, priorStored, buildSteeringSidecar(priorRuntimeContext), ); @@ -3782,7 +3802,7 @@ export class AiSdkBackend implements AgentBackend { const materializeReplayFallback = (): Promise => fallbackUsesRuntimeReplay ? this.materializeRuntimeReplayTextOnly( - scope.imageBudget, + scope.attachmentBudget, plan, projectedHistoryCompactCheckpoint, ) @@ -3818,7 +3838,7 @@ export class AiSdkBackend implements AgentBackend { status: 'ready', messages: await this.materializeRuntimeReplayPlan( plan, - scope.imageBudget, + scope.attachmentBudget, undefined, projectedHistoryCompactCheckpoint, ), @@ -3842,7 +3862,7 @@ export class AiSdkBackend implements AgentBackend { degradedPlan.items.length > 0 || hasProviderHistoryCompactCheckpoint ? await this.materializeRuntimeReplayPlan( degradedPlan, - scope.imageBudget, + scope.attachmentBudget, undefined, projectedHistoryCompactCheckpoint, ) @@ -3861,7 +3881,7 @@ export class AiSdkBackend implements AgentBackend { status: 'ready', messages: await this.materializeRuntimeReplayPlan( plan, - scope.imageBudget, + scope.attachmentBudget, undefined, projectedHistoryCompactCheckpoint, ), @@ -3933,7 +3953,7 @@ export class AiSdkBackend implements AgentBackend { */ private async materializeRuntimeReplayPlan( plan: RuntimeEventModelReplayPlan, - budget: ProviderImageBudget, + budget: ProviderAttachmentBudget, settledModelOutputs?: ReadonlyMap, historyCompactCheckpoint?: HistoryCompactCheckpoint, ): Promise { @@ -4332,7 +4352,7 @@ export class AiSdkBackend implements AgentBackend { } private async materializeRuntimeReplayTextOnly( - budget: ProviderImageBudget, + budget: ProviderAttachmentBudget, plan: RuntimeEventModelReplayPlan, historyCompactCheckpoint?: HistoryCompactCheckpoint, ): Promise { @@ -4382,7 +4402,7 @@ export class AiSdkBackend implements AgentBackend { } private async materializeRuntimeReplayItem( - budget: ProviderImageBudget, + budget: ProviderAttachmentBudget, item: Extract, ): Promise { if (item.role === 'user') { @@ -4397,7 +4417,7 @@ export class AiSdkBackend implements AgentBackend { } return { role: 'user', - content: await this.appendImageParts( + content: await this.appendAttachmentParts( budget, item.content, item.attachments, @@ -4413,7 +4433,7 @@ export class AiSdkBackend implements AgentBackend { } private async materializePriorMessages( - budget: ProviderImageBudget, + budget: ProviderAttachmentBudget, stored: readonly StoredMessage[], steeringSidecar?: ReadonlyMap, ): Promise { @@ -4429,7 +4449,7 @@ export class AiSdkBackend implements AgentBackend { out.push( steeringModelMessage( sidecar.eventId, - await this.appendImageParts( + await this.appendAttachmentParts( budget, buildSteeringEnvelope(formatTextWithInlineRefs(m.text, m)), m.attachments, @@ -4441,7 +4461,7 @@ export class AiSdkBackend implements AgentBackend { } out.push({ role: 'user', - content: await this.appendImageParts( + content: await this.appendAttachmentParts( budget, formatTextWithInlineRefs(m.text, m), m.attachments, @@ -4480,87 +4500,148 @@ export class AiSdkBackend implements AgentBackend { } /** A decision key deduplicates re-materialization; no key charges each occurrence. */ - private chargeImageBudget( - budget: ProviderImageBudget, + private chargeAttachmentBudget( + budget: ProviderAttachmentBudget, + kind: 'image' | 'pdf', bytes: number, decisionKey?: string, - ): boolean { + ): 'keep' | 'image_limit' | 'pdf_limit' | 'combined_limit' { if (decisionKey !== undefined) { const cached = budget.decisions.get(decisionKey); if (cached !== undefined) return cached; } - const keep = - budget.used + bytes <= - (this.input.maxProviderImageRequestBytes ?? MAX_PROVIDER_IMAGE_REQUEST_BYTES); - if (keep) budget.used += bytes; - if (decisionKey !== undefined) budget.decisions.set(decisionKey, keep); - return keep; + const subtypeLimit = + kind === 'image' + ? (this.input.maxProviderImageRequestBytes ?? MAX_PROVIDER_IMAGE_REQUEST_BYTES) + : (this.input.maxProviderPdfRequestBytes ?? MAX_PROVIDER_PDF_REQUEST_BYTES); + const decision = + budget.used[kind] + bytes > subtypeLimit + ? kind === 'image' + ? 'image_limit' + : 'pdf_limit' + : budget.used.total + bytes > + (this.input.maxProviderBinaryRequestBytes ?? MAX_PROVIDER_BINARY_REQUEST_BYTES) + ? 'combined_limit' + : 'keep'; + if (decision === 'keep') { + budget.used[kind] += bytes; + budget.used.total += bytes; + } + if (decisionKey !== undefined) budget.decisions.set(decisionKey, decision); + return decision; } /** * Render provider-visible content for a user message: keep the given - * (already-formatted) text, and append image attachments as provider image - * parts only for explicitly vision-capable models. Non-image attachments stay - * as placeholder refs in the text. Shared by the current turn, RuntimeEvent - * replay, and the stored-message fallback so all paths present images identically. + * (already-formatted) text, then append explicitly authorized image and PDF + * file parts. The PDF gate is a verified first-party provider/wire contract, + * not a model-name or SDK-encoding guess. Shared by the current turn, + * RuntimeEvent replay, stored-message fallback, steering, and compaction so + * every path presents one durable attachment occurrence identically. */ - private async appendImageParts( - budget: ProviderImageBudget, + private async appendAttachmentParts( + budget: ProviderAttachmentBudget, textContent: string, attachments?: AttachmentRef[], decisionKeyPrefix?: string, ): Promise { - const images = attachments?.filter((a) => a.kind === 'image') ?? []; - if (images.length === 0) { - return textContent; - } - if (this.input.supportsVision !== true) { - return appendNonVisionImageFallbackNotice(textContent); - } - if (!this.input.readAttachmentBytes) { - return textContent; - } + const binaryAttachments = + attachments?.filter( + (attachment): attachment is AttachmentRef & { kind: 'image' | 'pdf' } => + attachment.kind === 'image' || attachment.kind === 'pdf', + ) ?? []; + if (binaryAttachments.length === 0) return textContent; + const hasUnsupportedImages = + this.input.supportsVision !== true && + binaryAttachments.some((attachment) => attachment.kind === 'image'); + const fallbackText = hasUnsupportedImages + ? appendNonVisionImageFallbackNotice(textContent) + : textContent; + const eligibleAttachments = binaryAttachments.filter( + (attachment) => + (attachment.kind === 'image' && this.input.supportsVision === true) || + (attachment.kind === 'pdf' && this.input.pdfInputContract !== undefined), + ); + if (eligibleAttachments.length === 0 || !this.input.readAttachmentBytes) return fallbackText; const parts: Array< | { type: 'text'; text: string } | { type: 'file'; data: { type: 'data'; data: Uint8Array }; mediaType: string; + filename?: string; } - > = [{ type: 'text', text: textContent }]; - let omittedByBudget = 0; - for (const [index, image] of images.entries()) { - const read = await this.input.readAttachmentBytes(image.ref); + > = [{ type: 'text', text: fallbackText }]; + const omitted = { image_limit: 0, pdf_limit: 0, combined_limit: 0 }; + for (const [index, attachment] of binaryAttachments.entries()) { + const isEligible = + (attachment.kind === 'image' && this.input.supportsVision === true) || + (attachment.kind === 'pdf' && this.input.pdfInputContract !== undefined); + if (!isEligible) continue; + if (attachment.kind === 'pdf' && attachment.mimeType !== 'application/pdf') { + parts.push({ + type: 'text', + text: `PDF attachment "${attachment.name}" was omitted because its media type is not application/pdf.`, + }); + continue; + } + let read: Awaited>; + try { + read = await this.input.readAttachmentBytes(attachment.ref); + } catch { + read = { ok: false, reason: 'read_failed' }; + } if (!read.ok) { parts.push({ type: 'text', - text: `Image attachment "${image.name}" could not be loaded: ${read.reason}.`, + text: `${attachment.kind === 'pdf' ? 'PDF' : 'Image'} attachment "${attachment.name}" could not be loaded: ${read.reason}.`, }); continue; } const decisionKey = - decisionKeyPrefix === undefined ? undefined : `${decisionKeyPrefix}:image:${index}`; - if (!this.chargeImageBudget(budget, read.bytes.length, decisionKey)) { - omittedByBudget += 1; + decisionKeyPrefix === undefined + ? undefined + : `${decisionKeyPrefix}:${attachment.kind}:${index}`; + const decision = this.chargeAttachmentBudget( + budget, + attachment.kind, + read.bytes.length, + decisionKey, + ); + if (decision !== 'keep') { + omitted[decision] += 1; continue; } parts.push({ type: 'file', data: { type: 'data', data: read.bytes }, - mediaType: image.mimeType, + mediaType: attachment.mimeType, + ...(attachment.kind === 'pdf' ? { filename: attachment.name } : {}), }); } - if (omittedByBudget > 0) { + if (omitted.image_limit > 0) { parts.push({ type: 'text', - text: `[${omittedByBudget} image attachment(s) omitted: the per-request image budget was exceeded. Earlier images were sent; ask the user to send fewer or smaller images.]`, + text: `[${omitted.image_limit} image attachment(s) omitted: the per-request image budget was exceeded. Earlier images were sent; ask the user to send fewer or smaller images.]`, + }); + } + if (omitted.pdf_limit > 0) { + parts.push({ + type: 'text', + text: `[${omitted.pdf_limit} PDF attachment(s) omitted: the per-request PDF budget was exceeded. Earlier PDFs were sent; ask the user to send fewer or smaller PDFs.]`, + }); + } + if (omitted.combined_limit > 0) { + parts.push({ + type: 'text', + text: `[${omitted.combined_limit} binary attachment(s) omitted: the combined image/PDF request budget was exceeded. Earlier attachments were sent; ask the user to send fewer or smaller attachments.]`, }); } return parts; } private async materializeToolResultOutput( - budget: ProviderImageBudget, + budget: ProviderAttachmentBudget, output: unknown, isError: boolean, decisionKey: string, @@ -4572,8 +4653,9 @@ export class AiSdkBackend implements AgentBackend { if (!this.input.readAttachmentBytes) { return toolResultText('Image was read, but its stored bytes are unavailable.'); } - if (budget && budget.decisions.get(decisionKey) === false) { - return toolResultText(PROVIDER_IMAGE_BUDGET_EXCEEDED_MESSAGE); + const cachedDecision = budget.decisions.get(decisionKey); + if (cachedDecision !== undefined && cachedDecision !== 'keep') { + return toolResultText(this.imageBudgetFailureMessage(cachedDecision)); } let read: Awaited>; try { @@ -4584,8 +4666,9 @@ export class AiSdkBackend implements AgentBackend { if (!read.ok) { return toolResultText(`Image could not be loaded from artifact storage: ${read.reason}.`); } - if (!this.chargeImageBudget(budget, read.bytes.length, decisionKey)) { - return toolResultText(PROVIDER_IMAGE_BUDGET_EXCEEDED_MESSAGE); + const decision = this.chargeAttachmentBudget(budget, 'image', read.bytes.length, decisionKey); + if (decision !== 'keep') { + return toolResultText(this.imageBudgetFailureMessage(decision)); } return { type: 'content', @@ -4603,14 +4686,22 @@ export class AiSdkBackend implements AgentBackend { }; } + private imageBudgetFailureMessage( + decision: 'image_limit' | 'pdf_limit' | 'combined_limit', + ): string { + return decision === 'combined_limit' + ? `Image was read, but the combined image/PDF request budget (${MAX_PROVIDER_BINARY_REQUEST_BYTES / 1024 / 1024}MB across all binary attachments this turn) was exceeded; earlier attachments were sent and this one was omitted. Read fewer or smaller attachments.` + : PROVIDER_IMAGE_BUDGET_EXCEEDED_MESSAGE; + } + private async buildCurrentUserContent( - budget: ProviderImageBudget, + budget: ProviderAttachmentBudget, text: string, attachments?: AttachmentRef[], quotes?: QuoteRef[], runtimeEventId?: string, ): Promise { - return await this.appendImageParts( + return await this.appendAttachmentParts( budget, formatTextWithInlineRefs(text, { ...(attachments !== undefined ? { attachments } : {}), @@ -4737,8 +4828,8 @@ export class AiSdkBackend implements AgentBackend { // Materialize provider content before publishing the durable event. // After consumption there must be no fallible gap before ack/injection. const eventId = this.newId(); - const providerContent = await this.appendImageParts( - scope.imageBudget, + const providerContent = await this.appendAttachmentParts( + scope.attachmentBudget, buildSteeringEnvelope(formatTextWithInlineRefs(lease.content.text, lease.content)), lease.content.attachments, `steering:${eventId}`, diff --git a/packages/runtime/src/ai-sdk-compaction.ts b/packages/runtime/src/ai-sdk-compaction.ts index f57d90aee8..bf9db703a3 100644 --- a/packages/runtime/src/ai-sdk-compaction.ts +++ b/packages/runtime/src/ai-sdk-compaction.ts @@ -95,14 +95,16 @@ import { } from './context-budget-policy.js'; /** - * Image byte allowance for one turn, accumulated across its provider steps. + * Binary attachment allowance for one turn, accumulated across its provider + * steps. Decisions are cached by durable occurrence so rebuilding a request + * neither spends the allowance twice nor changes which attachments are sent. * * Charged while a request's content is materialized, so it belongs to the turn * issuing that request — never to the backend, which serves several turns. */ -export interface ProviderImageBudget { - used: number; - decisions: Map; +export interface ProviderAttachmentBudget { + used: { image: number; pdf: number; total: number }; + decisions: Map; } /** @@ -117,7 +119,7 @@ export interface ProviderImageBudget { */ export interface ProviderRequestOrigin { runId: string | undefined; - imageBudget: ProviderImageBudget; + attachmentBudget: ProviderAttachmentBudget; } export interface AutomaticMemoryCompactionDispatch { @@ -157,7 +159,7 @@ export interface AiSdkCompactionDeps { */ materializeRuntimeReplayPlan: ( plan: RuntimeEventModelReplayPlan, - imageBudget: ProviderImageBudget, + attachmentBudget: ProviderAttachmentBudget, checkpoint?: HistoryCompactCheckpoint, ) => Promise; canReplayProviderNative: (plan: RuntimeEventModelReplayPlan) => boolean; @@ -181,7 +183,7 @@ export class AiSdkCompaction { }) => ProviderRequestTracker | undefined; private readonly materializeRuntimeReplayPlan: ( plan: RuntimeEventModelReplayPlan, - imageBudget: ProviderImageBudget, + attachmentBudget: ProviderAttachmentBudget, checkpoint?: HistoryCompactCheckpoint, ) => Promise; private readonly canReplayProviderNative: (plan: RuntimeEventModelReplayPlan) => boolean; @@ -1685,7 +1687,7 @@ export class AiSdkCompaction { ); const replacementMessages = await this.materializeRuntimeReplayPlan( { ...replayPlan, items: replayItemsWithAnchorTail }, - input.origin.imageBudget, + input.origin.attachmentBudget, plan.checkpoint, ); // Apply the shape only when it actually shrinks the request versus the diff --git a/packages/runtime/src/model-runtime.ts b/packages/runtime/src/model-runtime.ts index 6929c8fb13..9a9cda960c 100644 --- a/packages/runtime/src/model-runtime.ts +++ b/packages/runtime/src/model-runtime.ts @@ -6,7 +6,11 @@ import { type ProviderRuntimeAdapter, type ProviderType, } from '@maka/core/llm-connections'; -import { lookupModelProviderOverride, openAiAdapterApiProtocol } from '@maka/core/model-metadata'; +import { + lookupModelProviderOverride, + openAiAdapterApiProtocol, + resolveModelPdfSupport, +} from '@maka/core/model-metadata'; import { resolveApplyPatchProfile, type ApplyPatchProfile } from './apply-patch-profile.js'; export type ModelRuntimeWire = @@ -35,6 +39,15 @@ export interface ResolvedModelRuntime { applyPatchProfile: ApplyPatchProfile | null; } +/** + * Provider/wire combinations whose native API contract has been verified to + * accept an AI SDK PDF file part. Provider identity is intentional: a relay + * using the same adapter or wire does not inherit first-party PDF support. + */ +export type ModelPdfInputContract = + | { providerType: 'anthropic'; wire: 'anthropic-messages' } + | { providerType: 'openai'; wire: 'openai-chat' | 'openai-responses' }; + export interface ModelRuntimeConnection { readonly providerType: ProviderType; readonly baseUrl?: string; @@ -100,6 +113,29 @@ export function resolveModelRuntime( }; } +/** + * Resolve native PDF materialization only when both model metadata and the + * first-party provider wire authorize it. Unknown models and compatible + * relays fail closed even if their wire happens to use an SDK file encoding. + */ +export function resolveModelPdfInputContract( + connection: ModelRuntimeConnection, + modelId: string, +): ModelPdfInputContract | null { + if (!resolveModelPdfSupport(connection.providerType, connection.models, modelId)) return null; + const { wire } = resolveModelRuntime(connection, modelId); + if (connection.providerType === 'anthropic' && wire === 'anthropic-messages') { + return { providerType: 'anthropic', wire }; + } + if ( + connection.providerType === 'openai' && + (wire === 'openai-chat' || wire === 'openai-responses') + ) { + return { providerType: 'openai', wire }; + } + return null; +} + export function modelUsesAnthropicMessages( connection: ModelRuntimeConnection, modelId: string,