From 472baa26749eb6e1fb66ea57d779d5fc96ba87d0 Mon Sep 17 00:00:00 2001 From: Andrew Kent Date: Wed, 5 Nov 2025 20:47:51 -0700 Subject: [PATCH] OAI completions instrumentation to match python and typescript SDKs --- .../otel/BraintrustOAISpanAttributes.java | 62 ++--- .../openai/otel/GenAiSemconvSerializer.java | 20 +- .../InstrumentedChatCompletionService.java | 18 +- ...nstrumentedChatCompletionServiceAsync.java | 24 +- .../openai/otel/OpenAITelemetryBuilder.java | 16 +- .../openai/otel/StreamListener.java | 22 +- .../openai/BraintrustOpenAITest.java | 227 +++++++++++++++++- 7 files changed, 306 insertions(+), 83 deletions(-) diff --git a/src/main/java/dev/braintrust/instrumentation/openai/otel/BraintrustOAISpanAttributes.java b/src/main/java/dev/braintrust/instrumentation/openai/otel/BraintrustOAISpanAttributes.java index d48635c8..0586cc5a 100644 --- a/src/main/java/dev/braintrust/instrumentation/openai/otel/BraintrustOAISpanAttributes.java +++ b/src/main/java/dev/braintrust/instrumentation/openai/otel/BraintrustOAISpanAttributes.java @@ -7,9 +7,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.openai.models.chat.completions.ChatCompletion; -import com.openai.models.chat.completions.ChatCompletionMessage; import io.opentelemetry.api.trace.Span; -import java.util.List; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; @@ -25,54 +23,34 @@ final class BraintrustOAISpanAttributes { private BraintrustOAISpanAttributes() {} - /** - * Sets the gen_ai.input.messages attribute with the serialized input messages. This captures - * the user's prompt and system messages before sending to OpenAI. - */ @SneakyThrows - public static void setInputMessages(Span span, List messages) { - String semconvJson = - GenAiSemconvSerializer.serializeInputMessages( - (List) - messages); + static void setRequestAttributes( + Span span, com.openai.models.chat.completions.ChatCompletionCreateParams request) { + // Set input messages + String semconvJson = GenAiSemconvSerializer.serializeInputMessages(request.messages()); span.setAttribute("gen_ai.input.messages", semconvJson); - } - /** - * Sets the gen_ai.output.messages attribute with the serialized output message. This captures - * the assistant's response from OpenAI for a single choice. - */ - @SneakyThrows - public static void setOutputMessages( - Span span, ChatCompletionMessage message, String finishReason) { - String outputJson = GenAiSemconvSerializer.serializeOutputMessage(message, finishReason); - span.setAttribute("gen_ai.output.messages", outputJson); - } + // Set Braintrust metadata + span.setAttribute("braintrust.metadata.provider", SYSTEM_OPENAI); - /** - * Sets the gen_ai.output.messages attribute for the primary choice in a completion. Logs a - * debug message if there are no choices or multiple choices. - */ - public static void setOutputMessagesFromCompletion(Span span, ChatCompletion completion) { - if (completion.choices().isEmpty()) { - log.debug("no choices in OAI response"); - } else if (completion.choices().size() > 1) { - log.debug("multiple choices in OAI response: {}", completion.choices().size()); - } else { - // Set gen_ai.output.messages attribute for single choice (most common case) - ChatCompletion.Choice choice = completion.choices().get(0); - setOutputMessages(span, choice.message(), choice.finishReason().toString()); + // Set model in metadata if present + try { + var model = request.model(); + span.setAttribute("braintrust.metadata.model", model.toString()); + } catch (Exception e) { + // If model() throws or returns null, just skip setting it + log.debug("Could not get model from request", e); } } - /** - * Sets the braintrust.output_json attribute with a single message. This is used for streaming - * responses to capture output in Braintrust format. - */ @SneakyThrows - public static void setBraintrustOutputJson(Span span, ChatCompletionMessage message) { + static void setOutputMessagesFromCompletion(Span span, ChatCompletion completion) { span.setAttribute( - "braintrust.output_json", - JSON_MAPPER.writeValueAsString(new ChatCompletionMessage[] {message})); + "gen_ai.output.messages", + GenAiSemconvSerializer.serializeOutputMessages(completion.choices())); + } + + static void setTimeToFirstToken(Span span, double timeInSeconds) { + span.setAttribute("braintrust.metrics.time_to_first_token", timeInSeconds); } } diff --git a/src/main/java/dev/braintrust/instrumentation/openai/otel/GenAiSemconvSerializer.java b/src/main/java/dev/braintrust/instrumentation/openai/otel/GenAiSemconvSerializer.java index ce5aa84e..1a9b6c6f 100644 --- a/src/main/java/dev/braintrust/instrumentation/openai/otel/GenAiSemconvSerializer.java +++ b/src/main/java/dev/braintrust/instrumentation/openai/otel/GenAiSemconvSerializer.java @@ -7,16 +7,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializerProvider; import com.fasterxml.jackson.databind.module.SimpleModule; -import com.openai.models.chat.completions.ChatCompletionAssistantMessageParam; -import com.openai.models.chat.completions.ChatCompletionContentPartImage; -import com.openai.models.chat.completions.ChatCompletionContentPartText; -import com.openai.models.chat.completions.ChatCompletionDeveloperMessageParam; -import com.openai.models.chat.completions.ChatCompletionMessage; -import com.openai.models.chat.completions.ChatCompletionMessageParam; -import com.openai.models.chat.completions.ChatCompletionMessageToolCall; -import com.openai.models.chat.completions.ChatCompletionSystemMessageParam; -import com.openai.models.chat.completions.ChatCompletionToolMessageParam; -import com.openai.models.chat.completions.ChatCompletionUserMessageParam; +import com.openai.models.chat.completions.*; import dev.braintrust.trace.Base64Attachment; import java.io.IOException; import java.lang.invoke.MethodHandle; @@ -441,9 +432,12 @@ static String serializeInputMessages(List messages) } @SneakyThrows - static String serializeOutputMessage(ChatCompletionMessage message, String finishReason) { - SemconvOutputChatMessage outputMessage = transformOutputMessage(message, finishReason); - return JSON_MAPPER.writeValueAsString(new SemconvOutputChatMessage[] {outputMessage}); + static String serializeOutputMessages(List choices) { + var semConvMessages = + choices.stream() + .map(c -> transformOutputMessage(c.message(), c.finishReason().toString())) + .toList(); + return JSON_MAPPER.writeValueAsString(semConvMessages); } @Nullable diff --git a/src/main/java/dev/braintrust/instrumentation/openai/otel/InstrumentedChatCompletionService.java b/src/main/java/dev/braintrust/instrumentation/openai/otel/InstrumentedChatCompletionService.java index 8361cf67..20ac1f57 100644 --- a/src/main/java/dev/braintrust/instrumentation/openai/otel/InstrumentedChatCompletionService.java +++ b/src/main/java/dev/braintrust/instrumentation/openai/otel/InstrumentedChatCompletionService.java @@ -100,9 +100,15 @@ private ChatCompletion createWithAttributes( Context context, ChatCompletionCreateParams chatCompletionCreateParams, RequestOptions requestOptions) { - BraintrustOAISpanAttributes.setInputMessages( - Span.current(), chatCompletionCreateParams.messages()); + BraintrustOAISpanAttributes.setRequestAttributes( + Span.current(), chatCompletionCreateParams); + + long startTimeNanos = System.nanoTime(); ChatCompletion result = delegate.create(chatCompletionCreateParams, requestOptions); + long elapsedNanos = System.nanoTime() - startTimeNanos; + double timeToFirstTokenSeconds = elapsedNanos / 1_000_000_000.0; + + BraintrustOAISpanAttributes.setTimeToFirstToken(Span.current(), timeToFirstTokenSeconds); BraintrustOAISpanAttributes.setOutputMessagesFromCompletion(Span.current(), result); return result; } @@ -130,8 +136,9 @@ private StreamResponse createStreamingWithAttributes( ChatCompletionCreateParams chatCompletionCreateParams, RequestOptions requestOptions, boolean newSpan) { - BraintrustOAISpanAttributes.setInputMessages( - Span.current(), chatCompletionCreateParams.messages()); + BraintrustOAISpanAttributes.setRequestAttributes( + Span.current(), chatCompletionCreateParams); + long startTimeNanos = System.nanoTime(); StreamResponse result = delegate.createStreaming(chatCompletionCreateParams, requestOptions); return new TracingStreamedResponse( @@ -141,6 +148,7 @@ private StreamResponse createStreamingWithAttributes( chatCompletionCreateParams, instrumenter, captureMessageContent, - newSpan)); + newSpan, + startTimeNanos)); } } diff --git a/src/main/java/dev/braintrust/instrumentation/openai/otel/InstrumentedChatCompletionServiceAsync.java b/src/main/java/dev/braintrust/instrumentation/openai/otel/InstrumentedChatCompletionServiceAsync.java index 2885b80e..b66715f3 100644 --- a/src/main/java/dev/braintrust/instrumentation/openai/otel/InstrumentedChatCompletionServiceAsync.java +++ b/src/main/java/dev/braintrust/instrumentation/openai/otel/InstrumentedChatCompletionServiceAsync.java @@ -103,14 +103,20 @@ private CompletableFuture createWithAttributes( Context context, ChatCompletionCreateParams chatCompletionCreateParams, RequestOptions requestOptions) { - BraintrustOAISpanAttributes.setInputMessages( - Span.current(), chatCompletionCreateParams.messages()); + BraintrustOAISpanAttributes.setRequestAttributes( + Span.current(), chatCompletionCreateParams); + + long startTimeNanos = System.nanoTime(); CompletableFuture future = delegate.create(chatCompletionCreateParams, requestOptions); future.thenAccept( - r -> - BraintrustOAISpanAttributes.setOutputMessagesFromCompletion( - Span.current(), r)); + r -> { + long elapsedNanos = System.nanoTime() - startTimeNanos; + double timeToFirstTokenSeconds = elapsedNanos / 1_000_000_000.0; + BraintrustOAISpanAttributes.setTimeToFirstToken( + Span.current(), timeToFirstTokenSeconds); + BraintrustOAISpanAttributes.setOutputMessagesFromCompletion(Span.current(), r); + }); return future; } @@ -137,8 +143,9 @@ private AsyncStreamResponse createStreamingWithAttributes( ChatCompletionCreateParams chatCompletionCreateParams, RequestOptions requestOptions, boolean newSpan) { - BraintrustOAISpanAttributes.setInputMessages( - Span.current(), chatCompletionCreateParams.messages()); + BraintrustOAISpanAttributes.setRequestAttributes( + Span.current(), chatCompletionCreateParams); + long startTimeNanos = System.nanoTime(); AsyncStreamResponse result = delegate.createStreaming(chatCompletionCreateParams, requestOptions); return new TracingAsyncStreamedResponse( @@ -148,6 +155,7 @@ private AsyncStreamResponse createStreamingWithAttributes( chatCompletionCreateParams, instrumenter, captureMessageContent, - newSpan)); + newSpan, + startTimeNanos)); } } diff --git a/src/main/java/dev/braintrust/instrumentation/openai/otel/OpenAITelemetryBuilder.java b/src/main/java/dev/braintrust/instrumentation/openai/otel/OpenAITelemetryBuilder.java index 077b604a..3b4fd4c0 100644 --- a/src/main/java/dev/braintrust/instrumentation/openai/otel/OpenAITelemetryBuilder.java +++ b/src/main/java/dev/braintrust/instrumentation/openai/otel/OpenAITelemetryBuilder.java @@ -13,9 +13,9 @@ import io.opentelemetry.api.OpenTelemetry; import io.opentelemetry.instrumentation.api.incubator.semconv.genai.GenAiAttributesExtractor; import io.opentelemetry.instrumentation.api.incubator.semconv.genai.GenAiClientMetrics; -import io.opentelemetry.instrumentation.api.incubator.semconv.genai.GenAiSpanNameExtractor; import io.opentelemetry.instrumentation.api.instrumenter.Instrumenter; import io.opentelemetry.instrumentation.api.instrumenter.SpanKindExtractor; +import io.opentelemetry.instrumentation.api.instrumenter.SpanNameExtractor; /** A builder of {@link OpenAITelemetry}. */ @SuppressWarnings("IdentifierName") // Want to match library's convention @@ -47,11 +47,15 @@ public OpenAITelemetryBuilder setCaptureMessageContent(boolean captureMessageCon * OpenAITelemetryBuilder}. */ public OpenAITelemetry build() { + // Use hardcoded span names to match Python/TypeScript SDKs + SpanNameExtractor chatSpanNameExtractor = + request -> "Chat Completion"; + SpanNameExtractor embeddingSpanNameExtractor = + request -> "Embedding"; + Instrumenter chatInstrumenter = Instrumenter.builder( - openTelemetry, - INSTRUMENTATION_NAME, - GenAiSpanNameExtractor.create(ChatAttributesGetter.INSTANCE)) + openTelemetry, INSTRUMENTATION_NAME, chatSpanNameExtractor) .addAttributesExtractor( GenAiAttributesExtractor.create(ChatAttributesGetter.INSTANCE)) .addOperationMetrics(GenAiClientMetrics.get()) @@ -59,9 +63,7 @@ public OpenAITelemetry build() { Instrumenter embeddingsInstrumenter = Instrumenter.builder( - openTelemetry, - INSTRUMENTATION_NAME, - GenAiSpanNameExtractor.create(EmbeddingAttributesGetter.INSTANCE)) + openTelemetry, INSTRUMENTATION_NAME, embeddingSpanNameExtractor) .addAttributesExtractor( GenAiAttributesExtractor.create(EmbeddingAttributesGetter.INSTANCE)) .addOperationMetrics(GenAiClientMetrics.get()) diff --git a/src/main/java/dev/braintrust/instrumentation/openai/otel/StreamListener.java b/src/main/java/dev/braintrust/instrumentation/openai/otel/StreamListener.java index 16b3828d..b0d5f92a 100644 --- a/src/main/java/dev/braintrust/instrumentation/openai/otel/StreamListener.java +++ b/src/main/java/dev/braintrust/instrumentation/openai/otel/StreamListener.java @@ -28,6 +28,8 @@ final class StreamListener { private final boolean captureMessageContent; private final boolean newSpan; private final AtomicBoolean hasEnded; + private final AtomicBoolean firstChunkReceived; + private final long startTimeNanos; @Nullable private CompletionUsage usage; @Nullable private String model; @@ -38,18 +40,29 @@ final class StreamListener { ChatCompletionCreateParams request, Instrumenter instrumenter, boolean captureMessageContent, - boolean newSpan) { + boolean newSpan, + long startTimeNanos) { this.context = context; this.request = request; this.instrumenter = instrumenter; this.captureMessageContent = captureMessageContent; this.newSpan = newSpan; + this.startTimeNanos = startTimeNanos; choiceBuffers = new ArrayList<>(); hasEnded = new AtomicBoolean(); + firstChunkReceived = new AtomicBoolean(); } @SneakyThrows void onChunk(ChatCompletionChunk chunk) { + // Calculate time to first token on the first chunk + if (firstChunkReceived.compareAndSet(false, true)) { + long elapsedNanos = System.nanoTime() - startTimeNanos; + double timeToFirstTokenSeconds = elapsedNanos / 1_000_000_000.0; + BraintrustOAISpanAttributes.setTimeToFirstToken( + Span.fromContext(context), timeToFirstTokenSeconds); + } + model = chunk.model(); responseId = chunk.id(); chunk.usage().ifPresent(u -> usage = u); @@ -66,8 +79,6 @@ void onChunk(ChatCompletionChunk chunk) { buffer.append(choice.delta()); if (choice.finishReason().isPresent()) { buffer.finishReason = choice.finishReason().get().toString(); - BraintrustOAISpanAttributes.setBraintrustOutputJson( - Span.fromContext(context), buffer.toChoice().message()); } } } @@ -102,7 +113,10 @@ void endSpan(@Nullable Throwable error) { } if (newSpan) { - instrumenter.end(context, request, result.build(), error); + ChatCompletion completion = result.build(); + BraintrustOAISpanAttributes.setOutputMessagesFromCompletion( + Span.fromContext(context), completion); + instrumenter.end(context, request, completion, error); } } } diff --git a/src/test/java/dev/braintrust/instrumentation/openai/BraintrustOpenAITest.java b/src/test/java/dev/braintrust/instrumentation/openai/BraintrustOpenAITest.java index ab80bc65..e1312050 100644 --- a/src/test/java/dev/braintrust/instrumentation/openai/BraintrustOpenAITest.java +++ b/src/test/java/dev/braintrust/instrumentation/openai/BraintrustOpenAITest.java @@ -8,7 +8,10 @@ import com.github.tomakehurst.wiremock.junit5.WireMockExtension; import com.openai.client.OpenAIClient; import com.openai.client.okhttp.OpenAIOkHttpClient; +import com.openai.core.JsonValue; import com.openai.models.ChatModel; +import com.openai.models.FunctionDefinition; +import com.openai.models.FunctionParameters; import com.openai.models.chat.completions.*; import dev.braintrust.TestHarness; import dev.braintrust.api.BraintrustApiClient; @@ -108,6 +111,9 @@ void testWrapOpenAi() { assertEquals(1, spans.size()); var span = spans.get(0); + // Verify span name matches other SDKs + assertEquals("Chat Completion", span.getName()); + assertEquals("openai", span.getAttributes().get(AttributeKey.stringKey("gen_ai.system"))); assertEquals( "gpt-4o-mini", @@ -142,13 +148,43 @@ void testWrapOpenAi() { 0.0, span.getAttributes().get(AttributeKey.doubleKey("gen_ai.request.temperature"))); + // Verify time to first token metric was captured + Double timeToFirstToken = + span.getAttributes() + .get(AttributeKey.doubleKey("braintrust.metrics.time_to_first_token")); + assertNotNull(timeToFirstToken, "time_to_first_token should be set"); + assertTrue(timeToFirstToken >= 0.0, "time_to_first_token should be non-negative"); + + // Verify Braintrust metadata was captured + assertEquals( + "openai", + span.getAttributes().get(AttributeKey.stringKey("braintrust.metadata.provider"))); + assertEquals( + "gpt-4o-mini", + span.getAttributes().get(AttributeKey.stringKey("braintrust.metadata.model"))); + + // Verify gen_ai.output.messages conforms to OTel spec with finish_reason String outputJson = span.getAttributes().get(AttributeKey.stringKey("gen_ai.output.messages")); + assertNotNull(outputJson); assertEquals( "[{\"role\":\"assistant\",\"parts\":[{\"type\":\"text\",\"content\":\"The capital" + " of France is Paris.\"}],\"finish_reason\":\"stop\"}]", outputJson); + // Verify the structure includes finish_reason and correct message content + var outputMessages = JSON_MAPPER.readTree(outputJson); + assertEquals(1, outputMessages.size()); + var outputMessage = outputMessages.get(0); + assertEquals("assistant", outputMessage.get("role").asText()); + assertEquals("stop", outputMessage.get("finish_reason").asText()); + assertTrue(outputMessage.has("parts")); + var parts = outputMessage.get("parts"); + assertEquals(1, parts.size()); + var textPart = parts.get(0); + assertEquals("text", textPart.get("type").asText()); + assertEquals("The capital of France is Paris.", textPart.get("content").asText()); + assertEquals( "chatcmpl-test123", span.getAttributes().get(AttributeKey.stringKey("gen_ai.response.id"))); @@ -237,6 +273,9 @@ void testWrapOpenAiStreaming() { assertEquals(1, spans.size()); var span = spans.get(0); + // Verify span name matches other SDKs + assertEquals("Chat Completion", span.getName()); + // Verify span attributes assertEquals("openai", span.getAttributes().get(AttributeKey.stringKey("gen_ai.system"))); assertEquals( @@ -262,14 +301,38 @@ void testWrapOpenAiStreaming() { assertEquals( 8L, span.getAttributes().get(AttributeKey.longKey("gen_ai.usage.output_tokens"))); - // Verify output JSON was captured + // Verify time to first token metric was captured + Double timeToFirstToken = + span.getAttributes() + .get(AttributeKey.doubleKey("braintrust.metrics.time_to_first_token")); + assertNotNull(timeToFirstToken, "time_to_first_token should be set"); + assertTrue(timeToFirstToken >= 0.0, "time_to_first_token should be non-negative"); + + // Verify Braintrust metadata was captured + assertEquals( + "openai", + span.getAttributes().get(AttributeKey.stringKey("braintrust.metadata.provider"))); + assertEquals( + "gpt-4o-mini", + span.getAttributes().get(AttributeKey.stringKey("braintrust.metadata.model"))); + + // Verify gen_ai.output.messages conforms to OTel spec with finish_reason String outputJson = - span.getAttributes().get(AttributeKey.stringKey("braintrust.output_json")); + span.getAttributes().get(AttributeKey.stringKey("gen_ai.output.messages")); assertNotNull(outputJson); + + // Verify the structure includes finish_reason and correct message content var outputMessages = JSON_MAPPER.readTree(outputJson); assertEquals(1, outputMessages.size()); - var messageZero = outputMessages.get(0); - assertEquals("The capital of France is Paris.", messageZero.get("content").asText()); + var outputMessage = outputMessages.get(0); + assertEquals("assistant", outputMessage.get("role").asText()); + assertEquals("stop", outputMessage.get("finish_reason").asText()); + assertTrue(outputMessage.has("parts")); + var parts = outputMessage.get("parts"); + assertEquals(1, parts.size()); + var textPart = parts.get(0); + assertEquals("text", textPart.get("type").asText()); + assertEquals("The capital of France is Paris.", textPart.get("content").asText()); } @Test @@ -472,4 +535,160 @@ void testBuildChatCompletionsPrompt() { .build(), renderedParams); } + + @Test + @SneakyThrows + void testWrapOpenAiWithToolCalls() { + // Mock the OpenAI API response with tool calls + wireMock.stubFor( + post(urlEqualTo("/chat/completions")) + .willReturn( + aResponse() + .withStatus(200) + .withHeader("Content-Type", "application/json") + .withBody( + """ + { + "id": "chatcmpl-test-tool", + "object": "chat.completion", + "created": 1234567890, + "model": "gpt-4o-2024-08-06", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": null, + "tool_calls": [ + { + "id": "call_abc123", + "type": "function", + "function": { + "name": "get_weather", + "arguments": "{\\"location\\":\\"Paris, France\\"}" + } + } + ] + }, + "finish_reason": "tool_calls" + } + ], + "usage": { + "prompt_tokens": 20, + "completion_tokens": 15, + "total_tokens": 35 + } + } + """))); + + OpenAIClient client = + BraintrustOpenAI.wrapOpenAI( + testHarness.openTelemetry(), + OpenAIOkHttpClient.builder() + .baseUrl(wireMock.baseUrl()) + .apiKey("test-api-key") + .build()); + + // Make a request with tools (we'll just verify the SDK handles tool call responses + // correctly) + + ChatCompletionCreateParams params = + ChatCompletionCreateParams.builder() + .model(ChatModel.GPT_4O) + .maxTokens(500) + .temperature(0.0) + .tools( + List.of( + ChatCompletionTool.builder() + .type(JsonValue.from("function")) + .function( + FunctionDefinition.builder() + .name("get_weather") + .description( + "Get the current weather" + + " for a location") + .parameters( + FunctionParameters.builder() + .putAllAdditionalProperties( + Map.of( + "location", + JsonValue + .from( + Map + .of( + "type", + "string", + "description", + "The city" + + " and state," + + " e.g." + + " San Francisco," + + " CA")), + "unit", + JsonValue + .from( + Map + .of( + "type", + "string", + "enum", + List + .of( + "celsius", + "fahrenheit"), + "description", + "The unit" + + " of temperature")))) + .build()) + .build()) + .build())) + .addUserMessage("What is the weather like in Paris, France?") + .build(); + + ChatCompletion response = client.chat().completions().create(params); + + // Verify the response has tool calls + assertEquals(1, response.choices().size()); + var choice = response.choices().get(0); + assertEquals("tool_calls", choice.finishReason().asString()); + assertTrue(choice.message().toolCalls().isPresent()); + assertEquals(1, choice.message().toolCalls().get().size()); + var toolCall = choice.message().toolCalls().get().get(0); + assertEquals("call_abc123", toolCall.id()); + assertEquals("get_weather", toolCall.function().name()); + + // Verify spans were exported + var spans = testHarness.awaitExportedSpans(); + assertEquals(1, spans.size()); + var span = spans.get(0); + + // Verify span name + assertEquals("Chat Completion", span.getName()); + + // Verify gen_ai.output.messages conforms to OTel spec with tool calls and finish_reason + String outputJson = + span.getAttributes().get(AttributeKey.stringKey("gen_ai.output.messages")); + assertNotNull(outputJson); + + // Parse and verify the structure + var outputMessages = JSON_MAPPER.readTree(outputJson); + assertEquals(1, outputMessages.size()); + var outputMessage = outputMessages.get(0); + + // Verify role and finish_reason + assertEquals("assistant", outputMessage.get("role").asText()); + assertEquals("tool_calls", outputMessage.get("finish_reason").asText()); + + // Verify parts contain tool_call + assertTrue(outputMessage.has("parts")); + var parts = outputMessage.get("parts"); + assertEquals(1, parts.size()); + + // Verify tool call part matches OTel spec for ToolCallRequestPart + var toolCallPart = parts.get(0); + assertEquals("tool_call", toolCallPart.get("type").asText()); + assertEquals("call_abc123", toolCallPart.get("id").asText()); + assertEquals("get_weather", toolCallPart.get("name").asText()); + assertTrue(toolCallPart.get("arguments").asText().contains("Paris")); + } }