diff --git a/.changeset/llm-body-timeout-request-id.md b/.changeset/llm-body-timeout-request-id.md new file mode 100644 index 000000000..b75ca3a97 --- /dev/null +++ b/.changeset/llm-body-timeout-request-id.md @@ -0,0 +1,5 @@ +--- +'@truefoundry/trueforge-core': patch +--- + +Stamp `x-tfy-request-id` on each model call and include that id with model name and gateway URL on all LLM error logs so harness logs can be correlated with AI Gateway traces. diff --git a/packages/trueforge-core/src/core/llm/VercelAILLM.ts b/packages/trueforge-core/src/core/llm/VercelAILLM.ts index a1dc948d5..64e8ca27b 100644 --- a/packages/trueforge-core/src/core/llm/VercelAILLM.ts +++ b/packages/trueforge-core/src/core/llm/VercelAILLM.ts @@ -27,6 +27,7 @@ import type { UserContent, } from 'ai'; import { APICallError, jsonSchema, Output, streamText } from 'ai'; +import { randomUUID } from 'node:crypto'; import type { ChatCompletionContentPart, ChatCompletionCreateParamsStreaming, @@ -35,19 +36,22 @@ import type { } from 'openai/resources/chat'; import type { Logger } from 'winston'; import { ModelProviderError } from '../errors'; -import { describeUnknownError, extractErrorLogFields } from '../util/errorLogFields'; +import { describeUnknownError, extractErrorLogFields, type ErrorLogFields } from '../util/errorLogFields'; import { modelSsrfFetch } from '../util/outboundFetch'; import type { ILLM, LLMCreateParams, LLMCreateParamsStreaming } from './ILLM'; import { + getEmptyUsage, type CompletionUsage, type ExtendedChatCompletionChunk, - getEmptyUsage, type RawAssistantMessage, type RawAssistantMessageWithUsage, type ThinkingBlock, } from './LLMTypes'; import { resolveMaxOutputTokens } from './usage'; +/** Correlates harness model calls with TrueFoundry AI Gateway request traces. */ +export const X_TFY_REQUEST_ID = 'x-tfy-request-id'; + // --------------------------------------------------------------------------- // Public config // --------------------------------------------------------------------------- @@ -1374,6 +1378,22 @@ export async function* mapStreamToChunks({ // VercelAILLM // --------------------------------------------------------------------------- +/** Error log fields with gateway correlation id plus model and URL when known. */ +export function llmErrorLogFields(input: { + error: unknown; + tfyRequestId: string; + model: string; + gatewayUrl: string | undefined; +}): ErrorLogFields & { tfy_request_id: string; model: string; gateway_url?: string } { + const fields = extractErrorLogFields(input.error); + return { + ...fields, + tfy_request_id: input.tfyRequestId, + model: input.model, + ...(input.gatewayUrl !== undefined ? { gateway_url: input.gatewayUrl } : {}), + }; +} + /** ILLM implementation backed by Vercel AI SDK, supporting multiple LLM providers. */ export class VercelAILLM implements ILLM { private readonly config: VercelAILLMConfig; @@ -1389,7 +1409,15 @@ export class VercelAILLM implements ILLM { async *create( body: LLMCreateParamsStreaming, ): AsyncGenerator { - const { providerConfig } = this.config; + // Per-call id on x-tfy-request-id so harness error logs can be correlated with gateway traces. + const tfyRequestId = randomUUID(); + const providerConfig: VercelAIProviderConfig = { + ...this.config.providerConfig, + headers: { + ...this.config.providerConfig.headers, + [X_TFY_REQUEST_ID]: tfyRequestId, + }, + }; const languageModel = buildLanguageModel(providerConfig); const { provider, model } = providerConfig; @@ -1447,7 +1475,15 @@ export class VercelAILLM implements ILLM { if (this.signal?.aborted) { this.logger.debug('LLM call aborted', extractErrorLogFields(streamError)); } else { - this.logger.error('Error creating streaming chat completion', extractErrorLogFields(streamError)); + this.logger.error( + 'Error creating streaming chat completion', + llmErrorLogFields({ + error: streamError, + tfyRequestId, + model: providerConfig.name, + gatewayUrl: providerConfig.baseUrl, + }), + ); } throw streamError; } @@ -1486,7 +1522,15 @@ export class VercelAILLM implements ILLM { if (this.signal?.aborted) { this.logger.debug('LLM stream aborted', extractErrorLogFields(streamError)); } else { - this.logger.error('Error reading LLM stream', extractErrorLogFields(streamError)); + this.logger.error( + 'Error reading LLM stream', + llmErrorLogFields({ + error: streamError, + tfyRequestId, + model: providerConfig.name, + gatewayUrl: providerConfig.baseUrl, + }), + ); } throw streamError; } diff --git a/packages/trueforge-core/tests/core/llm/VercelAILLM.stream.test.ts b/packages/trueforge-core/tests/core/llm/VercelAILLM.stream.test.ts index 593e2f3bb..8288acac9 100644 --- a/packages/trueforge-core/tests/core/llm/VercelAILLM.stream.test.ts +++ b/packages/trueforge-core/tests/core/llm/VercelAILLM.stream.test.ts @@ -15,6 +15,7 @@ import { ModelProviderError } from '../../../src/core/errors'; import type { ExtendedChatCompletionChunk, RawAssistantMessageWithUsage } from '../../../src/core/llm/LLMTypes'; import { describeStreamError, + llmErrorLogFields, mapFinishReason, mapStreamToChunks, normalizeUsage, @@ -685,3 +686,34 @@ describe('mapStreamToChunks', () => { expect(nonFinishChunks).toHaveLength(0); }); }); + +// ─────────── LLM error correlation ─────────── + +describe('llmErrorLogFields', () => { + it('includes tfy_request_id, model, and gateway_url for gateway correlation', () => { + const error = new Error('upstream failed'); + expect( + llmErrorLogFields({ + error, + tfyRequestId: 'req-abc', + model: 'truefoundry/acct/gpt', + gatewayUrl: 'https://gateway.example/v1', + }), + ).toMatchObject({ + error: 'upstream failed', + tfy_request_id: 'req-abc', + model: 'truefoundry/acct/gpt', + gateway_url: 'https://gateway.example/v1', + }); + }); + + it('omits gateway_url when the provider has no base URL', () => { + const fields = llmErrorLogFields({ + error: new Error('upstream failed'), + tfyRequestId: 'req-abc', + model: 'openai/gpt', + gatewayUrl: undefined, + }); + expect(fields).not.toHaveProperty('gateway_url'); + }); +});