Skip to content

enhance anthropic instrumentation #12

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Andrew Kent (realark) merged 1 commit into from
Nov 12, 2025
Merged

enhance anthropic instrumentation #12

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions src/main/java/dev/braintrust/instrumentation/anthropic/otel/AnthropicTelemetryBuilder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
import io.opentelemetry.api.OpenTelemetry;
import io.opentelemetry.instrumentation.api.incubator.semconv.genai.GenAiAttributesExtractor;
import io.opentelemetry.instrumentation.api.incubator.semconv.genai.GenAiClientMetrics;
import io.opentelemetry.instrumentation.api.incubator.semconv.genai.GenAiSpanNameExtractor;
import io.opentelemetry.instrumentation.api.instrumenter.Instrumenter;
import io.opentelemetry.instrumentation.api.instrumenter.SpanKindExtractor;

Expand Down Expand Up @@ -39,11 +38,14 @@ public AnthropicTelemetryBuilder setCaptureMessageContent(boolean captureMessage
* AnthropicTelemetryBuilder}.
*/
public AnthropicTelemetry build() {
// Use a custom span name extractor that returns just the operation name
// without appending the model name (unlike the default GenAiSpanNameExtractor)
Instrumenter<MessageCreateParams, Message> messageInstrumenter =
Instrumenter.<MessageCreateParams, Message>builder(
openTelemetry,
INSTRUMENTATION_NAME,
GenAiSpanNameExtractor.create(MessageAttributesGetter.INSTANCE))
request ->
MessageAttributesGetter.INSTANCE.getOperationName(request))
.addAttributesExtractor(
GenAiAttributesExtractor.create(MessageAttributesGetter.INSTANCE))
.addOperationMetrics(GenAiClientMetrics.get())
Expand Down
3 changes: 1 addition & 2 deletions ...java/dev/braintrust/instrumentation/anthropic/otel/BraintrustAnthropicSpanAttributes.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,7 @@ public static void setInputMessages(Span span, List<MessageParam> messages) {
*/
@SneakyThrows
public static void setOutputMessage(Span span, Message message) {
span.setAttribute(
"braintrust.output_json", JSON_MAPPER.writeValueAsString(new Message[] {message}));
span.setAttribute("braintrust.output_json", JSON_MAPPER.writeValueAsString(message));
}

/**
Expand Down
19 changes: 12 additions & 7 deletions src/main/java/dev/braintrust/instrumentation/anthropic/otel/InstrumentedMessageService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,17 +81,20 @@ private Message create(MessageCreateParams inputMessage, RequestOptions requestO
Context context = instrumenter.start(parentContext, inputMessage);
Message outputMessage;
try (Scope ignored = context.makeCurrent()) {
Span currentSpan = Span.current();
// Set provider metadata
currentSpan.setAttribute("provider", "anthropic");

List<MessageParam> inputMessages = new ArrayList<>(inputMessage.messages());
// Put system in the input message so the backend will pick it up in the LLM display
// Append system message to the end so the backend will pick it up in the LLM display
if (inputMessage.system().isPresent()) {
inputMessages.add(
0,
MessageParam.builder()
.role(MessageParam.Role.of("system"))
.content(inputMessage.system().get().asString())
.build());
}
BraintrustAnthropicSpanAttributes.setInputMessages(Span.current(), inputMessages);
BraintrustAnthropicSpanAttributes.setInputMessages(currentSpan, inputMessages);
outputMessage = delegate.create(inputMessage, requestOptions);
BraintrustAnthropicSpanAttributes.setOutputMessage(Span.current(), outputMessage);
} catch (Throwable t) {
Expand Down Expand Up @@ -125,18 +128,20 @@ private StreamResponse<RawMessageStreamEvent> createStreamingWithAttributes(
MessageCreateParams inputMessage,
RequestOptions requestOptions,
boolean newSpan) {
Span span = Span.fromContext(context);
// Set provider metadata
span.setAttribute("provider", "anthropic");

List<MessageParam> inputMessages = new ArrayList<>(inputMessage.messages());
// Put system in the input message so the backend will pick it up in the LLM display
// Append system message to the end so the backend will pick it up in the LLM display
if (inputMessage.system().isPresent()) {
inputMessages.add(
0,
MessageParam.builder()
.role(MessageParam.Role.of("system"))
.content(inputMessage.system().get().asString())
.build());
}
BraintrustAnthropicSpanAttributes.setInputMessages(
Span.fromContext(context), inputMessages);
BraintrustAnthropicSpanAttributes.setInputMessages(span, inputMessages);

StreamResponse<RawMessageStreamEvent> result =
delegate.createStreaming(inputMessage, requestOptions);
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/dev/braintrust/instrumentation/anthropic/otel/MessageAttributesGetter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ enum MessageAttributesGetter implements GenAiAttributesGetter<MessageCreateParam

@Override
public String getOperationName(MessageCreateParams request) {
return BraintrustAnthropicSpanAttributes.OPERATION_CHAT;
return "anthropic.messages.create";
}

@Override
Expand Down
28 changes: 14 additions & 14 deletions src/test/java/dev/braintrust/instrumentation/anthropic/BraintrustAnthropicTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,8 @@ void testWrapAnthropic() {
.get(AttributeKey.stringArrayKey("gen_ai.response.finish_reasons"))
.toString());
assertEquals(
"chat", span.getAttributes().get(AttributeKey.stringKey("gen_ai.operation.name")));
"anthropic.messages.create",
span.getAttributes().get(AttributeKey.stringKey("gen_ai.operation.name")));
assertEquals(
"msg_test123",
span.getAttributes().get(AttributeKey.stringKey("gen_ai.response.id")));
Expand All @@ -132,27 +133,25 @@ void testWrapAnthropic() {

// Verify Braintrust-specific attributes
assertEquals(
"[{\"content\":\"You are a helpful"
+ " assistant\",\"role\":\"system\",\"valid\":false},{\"content\":\"What is the"
+ " capital of France?\",\"role\":\"user\",\"valid\":true}]",
"[{\"content\":\"What is the capital of"
+ " France?\",\"role\":\"user\",\"valid\":true},{\"content\":\"You are a"
+ " helpful assistant\",\"role\":\"system\",\"valid\":false}]",
span.getAttributes().get(AttributeKey.stringKey("braintrust.input_json")));

// Verify output JSON
String outputJson =
span.getAttributes().get(AttributeKey.stringKey("braintrust.output_json"));
assertNotNull(outputJson);

var outputMessages = JSON_MAPPER.readTree(outputJson);
assertEquals(1, outputMessages.size());
var messageZero = outputMessages.get(0);
assertEquals("msg_test123", messageZero.get("id").asText());
assertEquals("message", messageZero.get("type").asText());
assertEquals("assistant", messageZero.get("role").asText());
var outputMessage = JSON_MAPPER.readTree(outputJson);
assertEquals("msg_test123", outputMessage.get("id").asText());
assertEquals("message", outputMessage.get("type").asText());
assertEquals("assistant", outputMessage.get("role").asText());
assertEquals(
"The capital of France is Paris.",
messageZero.get("content").get(0).get("text").asText());
assertEquals(8, messageZero.get("usage").get("output_tokens").asInt());
assertEquals(20, messageZero.get("usage").get("input_tokens").asInt());
outputMessage.get("content").get(0).get("text").asText());
assertEquals(8, outputMessage.get("usage").get("output_tokens").asInt());
assertEquals(20, outputMessage.get("usage").get("input_tokens").asInt());
}

@Test
Expand Down Expand Up @@ -260,7 +259,8 @@ void testWrapAnthropicStreaming() {
"claude-3-5-haiku-20241022",
span.getAttributes().get(AttributeKey.stringKey("gen_ai.response.model")));
assertEquals(
"chat", span.getAttributes().get(AttributeKey.stringKey("gen_ai.operation.name")));
"anthropic.messages.create",
span.getAttributes().get(AttributeKey.stringKey("gen_ai.operation.name")));
assertEquals(
"msg_test123",
span.getAttributes().get(AttributeKey.stringKey("gen_ai.response.id")));
Expand Down