From e1e7f1a01bb4b84047d8450dd152dd4813e25c7a Mon Sep 17 00:00:00 2001 From: yahya-mouman Date: Thu, 27 Aug 2026 13:36:50 +0200 Subject: [PATCH 1/7] feat(llmobs): add AgentManifest builder to LLMObs API - Add AgentManifest immutable value class with Builder pattern in LLMObs.java - Add AgentTool immutable value class in LLMObs.java - Add AGENT_MANIFEST constant to LLMObsTags - Add annotateAgentManifest() default method to LLMObsSpan interface - Add comprehensive builder tests for AgentManifest and AgentTool Co-Authored-By: Claude Sonnet 4.6 --- .../java/datadog/trace/api/llmobs/LLMObs.java | 135 ++++++++++++++++++ .../datadog/trace/api/llmobs/LLMObsSpan.java | 9 ++ .../datadog/trace/api/llmobs/LLMObsTags.java | 1 + .../datadog/trace/api/llmobs/LLMObsTest.java | 67 +++++++++ 4 files changed, 212 insertions(+) diff --git a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObs.java b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObs.java index fcac3de284c..abac0b8d961 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObs.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObs.java @@ -1109,4 +1109,139 @@ public Double getScore() { return score; } } + + /** A tool declared in an agent manifest. */ + public static final class AgentTool { + private final String name; + private final String description; + private final Map parameters; + + public static AgentTool from(String name) { + return new AgentTool(name, null, null); + } + + public static AgentTool from( + String name, @Nullable String description, @Nullable Map parameters) { + return new AgentTool(name, description, parameters); + } + + private AgentTool(String name, String description, Map parameters) { + this.name = name; + this.description = description; + this.parameters = + parameters == null ? null : Collections.unmodifiableMap(new LinkedHashMap<>(parameters)); + } + + public String getName() { + return name; + } + + @Nullable + public String getDescription() { + return description; + } + + @Nullable + public Map getParameters() { + return parameters; + } + } + + /** + * Declares the configuration of an agent span: what model it calls, what instructions it runs + * with, and which tools it has available. + * + *

Build via {@link AgentManifest#builder()} and pass to {@link + * LLMObsSpan#annotateAgentManifest(AgentManifest)}. Only applied on agent spans; ignored on other + * span kinds. A subsequent call on the same span overwrites the previous manifest. + */ + public static final class AgentManifest { + private final String name; + private final String instructions; + private final String model; + private final Map modelSettings; + private final List tools; + + public static Builder builder() { + return new Builder(); + } + + private AgentManifest(Builder builder) { + this.name = builder.name; + this.instructions = builder.instructions; + this.model = builder.model; + this.modelSettings = + builder.modelSettings == null + ? null + : Collections.unmodifiableMap(new LinkedHashMap<>(builder.modelSettings)); + this.tools = + builder.tools == null + ? null + : Collections.unmodifiableList(new ArrayList<>(builder.tools)); + } + + @Nullable + public String getName() { + return name; + } + + @Nullable + public String getInstructions() { + return instructions; + } + + @Nullable + public String getModel() { + return model; + } + + @Nullable + public Map getModelSettings() { + return modelSettings; + } + + @Nullable + public List getTools() { + return tools; + } + + public static final class Builder { + private String name; + private String instructions; + private String model; + private Map modelSettings; + private List tools; + + private Builder() {} + + public Builder name(String name) { + this.name = name; + return this; + } + + public Builder instructions(String instructions) { + this.instructions = instructions; + return this; + } + + public Builder model(String model) { + this.model = model; + return this; + } + + public Builder modelSettings(Map modelSettings) { + this.modelSettings = modelSettings; + return this; + } + + public Builder tools(List tools) { + this.tools = tools; + return this; + } + + public AgentManifest build() { + return new AgentManifest(this); + } + } + } } diff --git a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObsSpan.java b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObsSpan.java index efaf13123b7..8145037637f 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObsSpan.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObsSpan.java @@ -59,6 +59,15 @@ default void annotatePrompt(LLMObs.Prompt prompt) {} */ default void setToolDefinitions(List toolDefinitions) {} + /** + * Annotate an agent span with its manifest configuration. + * + *

This annotation is ignored for non-agent spans. + * + * @param agentManifest The agent manifest configuration + */ + default void annotateAgentManifest(LLMObs.AgentManifest agentManifest) {} + /** * Annotate the span with metadata * diff --git a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObsTags.java b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObsTags.java index 130cf610dc0..6be02e5af08 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObsTags.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObsTags.java @@ -13,4 +13,5 @@ public class LLMObsTags { public static final String MODEL_VERSION = "model_version"; public static final String MODEL_PROVIDER = "model_provider"; public static final String TOOL_DEFINITIONS = "tool_definitions"; + public static final String AGENT_MANIFEST = "agent_manifest"; } diff --git a/dd-trace-api/src/test/java/datadog/trace/api/llmobs/LLMObsTest.java b/dd-trace-api/src/test/java/datadog/trace/api/llmobs/LLMObsTest.java index 584895070a5..286a5e5c071 100644 --- a/dd-trace-api/src/test/java/datadog/trace/api/llmobs/LLMObsTest.java +++ b/dd-trace-api/src/test/java/datadog/trace/api/llmobs/LLMObsTest.java @@ -208,6 +208,73 @@ void testPromptBuilderWithChatTemplate() { assertNotSame(chatTemplate, prompt.getChatTemplate()); } + @Test + void testAnnotateAgentManifestIsCompatibilityPreservingDefaultMethod() throws Exception { + assertTrue( + LLMObsSpan.class + .getMethod("annotateAgentManifest", LLMObs.AgentManifest.class) + .isDefault()); + } + + @Test + void testAgentManifestBuilderWithAllFields() { + Map modelSettings = new HashMap<>(); + modelSettings.put("temperature", 0.7); + modelSettings.put("max_tokens", 1024); + + List tools = + Arrays.asList( + LLMObs.AgentTool.from( + "get_weather", + "Look up the weather", + Collections.singletonMap("city", Collections.singletonMap("type", "string")))); + + LLMObs.AgentManifest manifest = + LLMObs.AgentManifest.builder() + .name("travel_desk") + .instructions("Book travel for the user.") + .model("gpt-4o") + .modelSettings(modelSettings) + .tools(tools) + .build(); + + assertEquals("travel_desk", manifest.getName()); + assertEquals("Book travel for the user.", manifest.getInstructions()); + assertEquals("gpt-4o", manifest.getModel()); + assertEquals(modelSettings, manifest.getModelSettings()); + assertNotSame(modelSettings, manifest.getModelSettings()); + assertEquals(1, manifest.getTools().size()); + assertNotSame(tools, manifest.getTools()); + assertEquals("get_weather", manifest.getTools().get(0).getName()); + assertEquals("Look up the weather", manifest.getTools().get(0).getDescription()); + } + + @Test + void testAgentManifestBuilderMinimal() { + LLMObs.AgentManifest manifest = LLMObs.AgentManifest.builder().build(); + assertNull(manifest.getName()); + assertNull(manifest.getInstructions()); + assertNull(manifest.getModel()); + assertNull(manifest.getModelSettings()); + assertNull(manifest.getTools()); + } + + @Test + void testAgentToolCreation() { + LLMObs.AgentTool tool = LLMObs.AgentTool.from("search"); + assertEquals("search", tool.getName()); + assertNull(tool.getDescription()); + assertNull(tool.getParameters()); + + Map params = new HashMap<>(); + params.put("query", Collections.singletonMap("type", "string")); + LLMObs.AgentTool fullTool = LLMObs.AgentTool.from("search", "Web search", params); + assertEquals("search", fullTool.getName()); + assertEquals("Web search", fullTool.getDescription()); + assertEquals(params, fullTool.getParameters()); + assertNotSame(params, fullTool.getParameters()); + } + @Test void testLLMMessageCreationWithToolCalls() { Map args = new HashMap<>(); From ff0f77cce44ed685957b53808b8db4fd8490653f Mon Sep 17 00:00:00 2001 From: yahya-mouman Date: Thu, 27 Aug 2026 13:42:17 +0200 Subject: [PATCH 2/7] feat(llmobs): add annotateAgentManifest default method to LLMObsSpan --- .../java/datadog/trace/api/llmobs/noop/NoOpLLMObsSpan.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/noop/NoOpLLMObsSpan.java b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/noop/NoOpLLMObsSpan.java index bcde38ccfae..ec393ab6844 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/noop/NoOpLLMObsSpan.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/noop/NoOpLLMObsSpan.java @@ -18,6 +18,9 @@ public void annotateIO(String inputData, String outputData) {} @Override public void annotatePrompt(LLMObs.Prompt prompt) {} + @Override + public void annotateAgentManifest(LLMObs.AgentManifest manifest) {} + @Override public void setToolDefinitions(List toolDefinitions) {} From 0985ede34c9da2b3a5bdf665d615153880ee039f Mon Sep 17 00:00:00 2001 From: yahya-mouman Date: Thu, 27 Aug 2026 13:52:28 +0200 Subject: [PATCH 3/7] feat(llmobs): implement annotateAgentManifest in DDLLMObsSpan Stores agent manifest fields (name, instructions, model, model_settings, tools) as an internal tag `_ml_obs_tag.agent_manifest`. Only applies to agent spans; warns and no-ops on other span kinds. Null manifest is silently ignored. Tools with null/empty names are skipped with a warning. A second call overwrites the previous manifest. Framework field "AgentObs SDK" is always added when any manifest fields are present. Co-Authored-By: Claude Sonnet 4.6 --- .../trace/llmobs/domain/DDLLMObsSpan.java | 60 +++++++ .../llmobs/domain/DDLLMObsSpanTest.groovy | 154 ++++++++++++++++++ 2 files changed, 214 insertions(+) diff --git a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java index b5283b73dad..04a7c4e958d 100644 --- a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java +++ b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java @@ -41,6 +41,8 @@ public class DDLLMObsSpan implements LLMObsSpan { private static final String SPAN_KIND = LLMOBS_TAG_PREFIX + Tags.SPAN_KIND; private static final String METADATA = LLMOBS_TAG_PREFIX + LLMObsTags.METADATA; private static final String TOOL_DEFINITIONS = LLMOBS_TAG_PREFIX + LLMObsTags.TOOL_DEFINITIONS; + private static final String AGENT_MANIFEST = LLMOBS_TAG_PREFIX + LLMObsTags.AGENT_MANIFEST; + private static final String MANUAL_FRAMEWORK = "AgentObs SDK"; private static final String PROMPT_TRACKING_INSTRUMENTATION_METHOD = LLMOBS_TAG_PREFIX + "prompt_tracking_instrumentation_method"; private static final String INSTRUMENTATION_METHOD_ANNOTATED = "annotated"; @@ -291,6 +293,64 @@ public void annotatePrompt(LLMObs.Prompt prompt) { span.setTag(PROMPT_TRACKING_INSTRUMENTATION_METHOD, INSTRUMENTATION_METHOD_ANNOTATED); } + @Override + public void annotateAgentManifest(LLMObs.AgentManifest manifest) { + if (finished || manifest == null) { + return; + } + if (!Tags.LLMOBS_AGENT_SPAN_KIND.equals(spanKind)) { + LOGGER.warn( + "dropping agent manifest on non-agent span kind; annotateAgentManifest is only supported for agent spans"); + return; + } + Map manifestMap = buildManifestMap(manifest); + if (!manifestMap.isEmpty()) { + manifestMap.put("framework", MANUAL_FRAMEWORK); + span.setTag(AGENT_MANIFEST, manifestMap); + } + } + + private Map buildManifestMap(LLMObs.AgentManifest manifest) { + Map map = new LinkedHashMap<>(); + CharSequence sn = span.getSpanName(); + String name = + manifest.getName() != null ? manifest.getName() : (sn != null ? sn.toString() : null); + if (name != null && !name.isEmpty()) { + map.put("name", name); + } + if (manifest.getInstructions() != null && !manifest.getInstructions().isEmpty()) { + map.put("instructions", manifest.getInstructions()); + } + if (manifest.getModel() != null && !manifest.getModel().isEmpty()) { + map.put("model", manifest.getModel()); + } + if (manifest.getModelSettings() != null && !manifest.getModelSettings().isEmpty()) { + map.put("model_settings", new LinkedHashMap<>(manifest.getModelSettings())); + } + if (manifest.getTools() != null && !manifest.getTools().isEmpty()) { + List> toolList = new ArrayList<>(); + for (LLMObs.AgentTool tool : manifest.getTools()) { + if (tool == null || tool.getName() == null || tool.getName().isEmpty()) { + LOGGER.warn("agent manifest tool missing required name; skipping"); + continue; + } + Map toolMap = new LinkedHashMap<>(); + toolMap.put("name", tool.getName()); + if (tool.getDescription() != null) { + toolMap.put("description", tool.getDescription()); + } + if (tool.getParameters() != null && !tool.getParameters().isEmpty()) { + toolMap.put("parameters", new LinkedHashMap<>(tool.getParameters())); + } + toolList.add(toolMap); + } + if (!toolList.isEmpty()) { + map.put("tools", toolList); + } + } + return map; + } + private static Map copyStringKeyedMap(Map source) { Map copy = new LinkedHashMap<>(); for (Map.Entry entry : source.entrySet()) { diff --git a/dd-java-agent/agent-llmobs/src/test/groovy/datadog/trace/llmobs/domain/DDLLMObsSpanTest.groovy b/dd-java-agent/agent-llmobs/src/test/groovy/datadog/trace/llmobs/domain/DDLLMObsSpanTest.groovy index bef904409a2..0c933fa7566 100644 --- a/dd-java-agent/agent-llmobs/src/test/groovy/datadog/trace/llmobs/domain/DDLLMObsSpanTest.groovy +++ b/dd-java-agent/agent-llmobs/src/test/groovy/datadog/trace/llmobs/domain/DDLLMObsSpanTest.groovy @@ -63,6 +63,7 @@ class DDLLMObsSpanTest extends DDSpecification{ private static final String OUTPUT = LLMOBS_TAG_PREFIX + "output" private static final String METADATA = LLMOBS_TAG_PREFIX + LLMObsTags.METADATA private static final String TOOL_DEFINITIONS = LLMOBS_TAG_PREFIX + LLMObsTags.TOOL_DEFINITIONS + private static final String AGENT_MANIFEST = LLMOBS_TAG_PREFIX + "agent_manifest" private static final String PROMPT_TRACKING_INSTRUMENTATION_METHOD = LLMOBS_TAG_PREFIX + "prompt_tracking_instrumentation_method" @@ -791,6 +792,159 @@ class DDLLMObsSpanTest extends DDSpecification{ innerSpan.getTag(LLMOBS_TAG_PREFIX + "owner") == "ml-platform" } + def "agent manifest full annotation sets correct tag"() { + setup: + def test = llmObsSpan(Tags.LLMOBS_AGENT_SPAN_KIND, "my-agent") + def settings = [temperature: 0.7, max_tokens: 1024] + def params = [city: [type: "string"]] + def tools = [LLMObs.AgentTool.from("get_weather", "Look up weather", params)] + def manifest = LLMObs.AgentManifest.builder() + .name("travel_desk") + .instructions("Book travel.") + .model("gpt-4o") + .modelSettings(settings) + .tools(tools) + .build() + + when: + test.annotateAgentManifest(manifest) + + then: + def innerSpan = (AgentSpan) test.span + def stored = (Map) innerSpan.getTag(AGENT_MANIFEST) + stored["name"] == "travel_desk" + stored["instructions"] == "Book travel." + stored["model"] == "gpt-4o" + stored["framework"] == "AgentObs SDK" + def ms = (Map) stored["model_settings"] + ms["temperature"] == 0.7 + ms["max_tokens"] == 1024 + def toolList = (List) stored["tools"] + toolList.size() == 1 + toolList[0]["name"] == "get_weather" + toolList[0]["description"] == "Look up weather" + toolList[0]["parameters"] == [city: [type: "string"]] + + cleanup: + test.finish() + } + + def "agent manifest name defaults to span name when not provided"() { + setup: + def test = llmObsSpan(Tags.LLMOBS_AGENT_SPAN_KIND, "my-agent") + def manifest = LLMObs.AgentManifest.builder().instructions("Do something.").build() + + when: + test.annotateAgentManifest(manifest) + + then: + def innerSpan = (AgentSpan) test.span + def stored = (Map) innerSpan.getTag(AGENT_MANIFEST) + stored["name"] == "my-agent" + stored["instructions"] == "Do something." + stored["framework"] == "AgentObs SDK" + + cleanup: + test.finish() + } + + def "agent manifest drops tool with null name"() { + setup: + def test = llmObsSpan(Tags.LLMOBS_AGENT_SPAN_KIND, "my-agent") + def validTool = LLMObs.AgentTool.from("valid-tool") + def badTool = LLMObs.AgentTool.from(null) + def manifest = LLMObs.AgentManifest.builder() + .tools([badTool, validTool]) + .build() + + when: + test.annotateAgentManifest(manifest) + + then: + def innerSpan = (AgentSpan) test.span + def stored = (Map) innerSpan.getTag(AGENT_MANIFEST) + def toolList = (List) stored["tools"] + toolList.size() == 1 + toolList[0]["name"] == "valid-tool" + + cleanup: + test.finish() + } + + def "agent manifest on non-agent span is silently dropped"() { + setup: + def test = llmObsSpan(Tags.LLMOBS_LLM_SPAN_KIND, "llm-span") + def manifest = LLMObs.AgentManifest.builder().name("agent").build() + + when: + test.annotateAgentManifest(manifest) + + then: + def innerSpan = (AgentSpan) test.span + innerSpan.getTag(AGENT_MANIFEST) == null + + cleanup: + test.finish() + } + + def "second annotateAgentManifest call overwrites the first"() { + setup: + def test = llmObsSpan(Tags.LLMOBS_AGENT_SPAN_KIND, "my-agent") + def first = LLMObs.AgentManifest.builder().name("first").instructions("v1").build() + def second = LLMObs.AgentManifest.builder().name("second").model("gpt-4o").build() + + when: + test.annotateAgentManifest(first) + test.annotateAgentManifest(second) + + then: + def innerSpan = (AgentSpan) test.span + def stored = (Map) innerSpan.getTag(AGENT_MANIFEST) + stored["name"] == "second" + stored["model"] == "gpt-4o" + !stored.containsKey("instructions") + + cleanup: + test.finish() + } + + def "agent manifest model_settings forwarded as-is"() { + setup: + def test = llmObsSpan(Tags.LLMOBS_AGENT_SPAN_KIND, "my-agent") + def manifest = LLMObs.AgentManifest.builder() + .name("agent") + .modelSettings([temperature: 0.5, custom_key: "custom_val"]) + .build() + + when: + test.annotateAgentManifest(manifest) + + then: + def innerSpan = (AgentSpan) test.span + def stored = (Map) innerSpan.getTag(AGENT_MANIFEST) + def ms = (Map) stored["model_settings"] + ms["temperature"] == 0.5 + ms["custom_key"] == "custom_val" + + cleanup: + test.finish() + } + + def "annotateAgentManifest null manifest is ignored"() { + setup: + def test = llmObsSpan(Tags.LLMOBS_AGENT_SPAN_KIND, "my-agent") + + when: + test.annotateAgentManifest(null) + + then: + def innerSpan = (AgentSpan) test.span + innerSpan.getTag(AGENT_MANIFEST) == null + + cleanup: + test.finish() + } + private LLMObsSpan llmObsSpan(String kind, name) { llmObsSpan(kind, name, null) } From b5f56f11d6faae32415beed528634b506281667a Mon Sep 17 00:00:00 2001 From: yahya-mouman Date: Thu, 27 Aug 2026 13:58:52 +0200 Subject: [PATCH 4/7] feat(llmobs): serialize agent_manifest to meta in LLMObsSpanMapper Add AGENT_MANIFEST_KEY byte constant, include the tag in TAGS_FOR_REMAPPING, and handle it in the meta serialization loop as a msgpack map. Add two tests: one verifying all manifest fields appear in meta, one verifying the tag does not leak into the tags list. Co-Authored-By: Claude Sonnet 4.6 --- .../writer/ddintake/LLMObsSpanMapper.java | 13 +++- .../writer/ddintake/LLMObsSpanMapperTest.java | 71 +++++++++++++++++++ 2 files changed, 83 insertions(+), 1 deletion(-) diff --git a/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java b/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java index e5484d79432..42b7222e7d7 100644 --- a/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java +++ b/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java @@ -80,6 +80,8 @@ public class LLMObsSpanMapper implements RemoteMapper { private static final byte[] META = "meta".getBytes(StandardCharsets.UTF_8); private static final byte[] METADATA = "metadata".getBytes(StandardCharsets.UTF_8); + private static final byte[] AGENT_MANIFEST_KEY = + "agent_manifest".getBytes(StandardCharsets.UTF_8); private static final byte[] PROMPT = "prompt".getBytes(StandardCharsets.UTF_8); private static final byte[] SPAN_KIND = "span.kind".getBytes(StandardCharsets.UTF_8); private static final byte[] SPANS = "spans".getBytes(StandardCharsets.UTF_8); @@ -345,7 +347,8 @@ private static final class MetaWriter implements MetadataConsumer { LLMOBS_TAG_PREFIX + LLMObsTags.MODEL_PROVIDER, LLMOBS_TAG_PREFIX + LLMObsTags.MODEL_VERSION, LLMOBS_TAG_PREFIX + LLMObsTags.TOOL_DEFINITIONS, - LLMOBS_TAG_PREFIX + LLMObsTags.METADATA))); + LLMOBS_TAG_PREFIX + LLMObsTags.METADATA, + LLMOBS_TAG_PREFIX + LLMObsTags.AGENT_MANIFEST))); MetaWriter withWritable(Writable writable, Map errorInfo) { this.writable = writable; @@ -527,6 +530,14 @@ public void accept(Metadata metadata) { writable.writeString(entry.getKey(), null); writable.writeObject(entry.getValue(), null); } + } else if (key.equals(LLMObsTags.AGENT_MANIFEST) && val instanceof Map) { + Map manifestMap = (Map) val; + writable.writeUTF8(AGENT_MANIFEST_KEY); + writable.startMap(manifestMap.size()); + for (Map.Entry entry : manifestMap.entrySet()) { + writable.writeString(String.valueOf(entry.getKey()), null); + writable.writeObject(entry.getValue(), null); + } } else { writable.writeString(key, null); writable.writeObject(val, null); diff --git a/dd-trace-core/src/test/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.java b/dd-trace-core/src/test/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.java index 4ec1172525c..3924c31aaf4 100644 --- a/dd-trace-core/src/test/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.java @@ -816,6 +816,77 @@ void testLLMObsSpanProcessorInputAndOutputRejectNull() { tracer.close(); } + @Test + void testLLMObsSpanMapperSerializesAgentManifest() throws Exception { + LLMObsSpanMapper mapper = new LLMObsSpanMapper(); + CoreTracer tracer = tracerBuilder().writer(new ListWriter()).build(); + + Map manifest = new LinkedHashMap<>(); + manifest.put("name", "travel_desk"); + manifest.put("instructions", "Book travel."); + manifest.put("model", "gpt-4o"); + manifest.put("framework", "AgentObs SDK"); + + Map modelSettings = new LinkedHashMap<>(); + modelSettings.put("temperature", 0.7); + manifest.put("model_settings", modelSettings); + + List> tools = new ArrayList<>(); + Map tool = new LinkedHashMap<>(); + tool.put("name", "get_weather"); + tool.put("description", "Look up the weather."); + tools.add(tool); + manifest.put("tools", tools); + + AgentSpan agentSpan = + tracer + .buildSpan("datadog", "my-agent") + .withTag("_ml_obs_tag.span.kind", "agent") + .withTag("_ml_obs_tag.agent_manifest", manifest) + .start(); + agentSpan.setSpanType(InternalSpanTypes.LLMOBS); + agentSpan.finish(); + + Map spanData = serializeSingleSpan(mapper, agentSpan); + Map meta = (Map) spanData.get("meta"); + + assertTrue(meta.containsKey("agent_manifest")); + Map gotManifest = (Map) meta.get("agent_manifest"); + assertEquals("travel_desk", gotManifest.get("name")); + assertEquals("Book travel.", gotManifest.get("instructions")); + assertEquals("gpt-4o", gotManifest.get("model")); + assertEquals("AgentObs SDK", gotManifest.get("framework")); + assertEquals(modelSettings, gotManifest.get("model_settings")); + assertEquals(tools, gotManifest.get("tools")); + + tracer.close(); + } + + @Test + void testAgentManifestDoesNotAppearInTags() throws Exception { + LLMObsSpanMapper mapper = new LLMObsSpanMapper(); + CoreTracer tracer = tracerBuilder().writer(new ListWriter()).build(); + + Map manifest = new LinkedHashMap<>(); + manifest.put("name", "my-agent"); + manifest.put("framework", "AgentObs SDK"); + + AgentSpan agentSpan = + tracer + .buildSpan("datadog", "my-agent") + .withTag("_ml_obs_tag.span.kind", "agent") + .withTag("_ml_obs_tag.agent_manifest", manifest) + .start(); + agentSpan.setSpanType(InternalSpanTypes.LLMOBS); + agentSpan.finish(); + + Map spanData = serializeSingleSpan(mapper, agentSpan); + List tags = (List) spanData.get("tags"); + assertFalse(tags.stream().anyMatch(t -> t.startsWith("agent_manifest:"))); + + tracer.close(); + } + private static AgentSpan newLlmObsSpan(CoreTracer tracer, String name, boolean drop) { AgentSpan span = tracer From 71284a38aebeef9fa2b2e4493c1ef159efd10b7c Mon Sep 17 00:00:00 2001 From: yahya-mouman Date: Fri, 28 Aug 2026 15:25:09 +0200 Subject: [PATCH 5/7] fix(llmobs): merge semantics, empty-name fallback, framework=manual, test hardening - Change annotateAgentManifest to merge successive calls instead of overwriting: name/instructions/model use new-value-wins logic with fallbacks; model_settings shallow-merges; tools replace only when caller provides a non-empty list. - Change MANUAL_FRAMEWORK constant from "AgentObs SDK" to "manual". - Update all test assertions that expected "AgentObs SDK" to "manual". - Replace overwrite test with merge-semantics test; add model_settings merge test. - Tests for post-finish no-op and empty-tools-list were already present. Co-Authored-By: Claude Sonnet 4.6 --- .../trace/llmobs/domain/DDLLMObsSpan.java | 55 ++++++++---- .../llmobs/domain/DDLLMObsSpanTest.groovy | 86 +++++++++++++++++-- .../java/datadog/trace/api/llmobs/LLMObs.java | 9 ++ .../datadog/trace/api/llmobs/LLMObsSpan.java | 2 + .../writer/ddintake/LLMObsSpanMapperTest.java | 11 ++- 5 files changed, 132 insertions(+), 31 deletions(-) diff --git a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java index 04a7c4e958d..fef711c85df 100644 --- a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java +++ b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java @@ -42,7 +42,7 @@ public class DDLLMObsSpan implements LLMObsSpan { private static final String METADATA = LLMOBS_TAG_PREFIX + LLMObsTags.METADATA; private static final String TOOL_DEFINITIONS = LLMOBS_TAG_PREFIX + LLMObsTags.TOOL_DEFINITIONS; private static final String AGENT_MANIFEST = LLMOBS_TAG_PREFIX + LLMObsTags.AGENT_MANIFEST; - private static final String MANUAL_FRAMEWORK = "AgentObs SDK"; + private static final String MANUAL_FRAMEWORK = "manual"; private static final String PROMPT_TRACKING_INSTRUMENTATION_METHOD = LLMOBS_TAG_PREFIX + "prompt_tracking_instrumentation_method"; private static final String INSTRUMENTATION_METHOD_ANNOTATED = "annotated"; @@ -303,30 +303,48 @@ public void annotateAgentManifest(LLMObs.AgentManifest manifest) { "dropping agent manifest on non-agent span kind; annotateAgentManifest is only supported for agent spans"); return; } - Map manifestMap = buildManifestMap(manifest); - if (!manifestMap.isEmpty()) { - manifestMap.put("framework", MANUAL_FRAMEWORK); - span.setTag(AGENT_MANIFEST, manifestMap); - } + // Read existing manifest (may be null on first call) + Object existing = span.getTag(AGENT_MANIFEST); + @SuppressWarnings("unchecked") + Map base = + (existing instanceof Map) + ? new LinkedHashMap<>((Map) existing) + : new LinkedHashMap<>(); + mergeManifest(base, manifest); + base.put("framework", MANUAL_FRAMEWORK); + span.setTag(AGENT_MANIFEST, base); } - private Map buildManifestMap(LLMObs.AgentManifest manifest) { - Map map = new LinkedHashMap<>(); - CharSequence sn = span.getSpanName(); - String name = - manifest.getName() != null ? manifest.getName() : (sn != null ? sn.toString() : null); - if (name != null && !name.isEmpty()) { - map.put("name", name); + private void mergeManifest(Map base, LLMObs.AgentManifest manifest) { + // name: new non-empty wins, else keep existing, else span name + String manifestName = manifest.getName(); + if (manifestName != null && !manifestName.isEmpty()) { + base.put("name", manifestName); + } else if (!base.containsKey("name")) { + CharSequence sn = span.getSpanName(); + if (sn != null && sn.length() > 0) { + base.put("name", sn.toString()); + } } + // instructions if (manifest.getInstructions() != null && !manifest.getInstructions().isEmpty()) { - map.put("instructions", manifest.getInstructions()); + base.put("instructions", manifest.getInstructions()); } + // model if (manifest.getModel() != null && !manifest.getModel().isEmpty()) { - map.put("model", manifest.getModel()); + base.put("model", manifest.getModel()); } + // model_settings: shallow merge if (manifest.getModelSettings() != null && !manifest.getModelSettings().isEmpty()) { - map.put("model_settings", new LinkedHashMap<>(manifest.getModelSettings())); - } + @SuppressWarnings("unchecked") + Map existingSettings = + (base.get("model_settings") instanceof Map) + ? new LinkedHashMap<>((Map) base.get("model_settings")) + : new LinkedHashMap<>(); + existingSettings.putAll(manifest.getModelSettings()); + base.put("model_settings", existingSettings); + } + // tools: replace if non-null non-empty if (manifest.getTools() != null && !manifest.getTools().isEmpty()) { List> toolList = new ArrayList<>(); for (LLMObs.AgentTool tool : manifest.getTools()) { @@ -345,10 +363,9 @@ private Map buildManifestMap(LLMObs.AgentManifest manifest) { toolList.add(toolMap); } if (!toolList.isEmpty()) { - map.put("tools", toolList); + base.put("tools", toolList); } } - return map; } private static Map copyStringKeyedMap(Map source) { diff --git a/dd-java-agent/agent-llmobs/src/test/groovy/datadog/trace/llmobs/domain/DDLLMObsSpanTest.groovy b/dd-java-agent/agent-llmobs/src/test/groovy/datadog/trace/llmobs/domain/DDLLMObsSpanTest.groovy index 0c933fa7566..bd5ce69ab57 100644 --- a/dd-java-agent/agent-llmobs/src/test/groovy/datadog/trace/llmobs/domain/DDLLMObsSpanTest.groovy +++ b/dd-java-agent/agent-llmobs/src/test/groovy/datadog/trace/llmobs/domain/DDLLMObsSpanTest.groovy @@ -815,7 +815,7 @@ class DDLLMObsSpanTest extends DDSpecification{ stored["name"] == "travel_desk" stored["instructions"] == "Book travel." stored["model"] == "gpt-4o" - stored["framework"] == "AgentObs SDK" + stored["framework"] == "manual" def ms = (Map) stored["model_settings"] ms["temperature"] == 0.7 ms["max_tokens"] == 1024 @@ -842,7 +842,7 @@ class DDLLMObsSpanTest extends DDSpecification{ def stored = (Map) innerSpan.getTag(AGENT_MANIFEST) stored["name"] == "my-agent" stored["instructions"] == "Do something." - stored["framework"] == "AgentObs SDK" + stored["framework"] == "manual" cleanup: test.finish() @@ -871,6 +871,26 @@ class DDLLMObsSpanTest extends DDSpecification{ test.finish() } + def "agent manifest with empty tools list omits tools key"() { + setup: + def test = llmObsSpan(Tags.LLMOBS_AGENT_SPAN_KIND, "my-agent") + def manifest = LLMObs.AgentManifest.builder() + .name("agent") + .tools([]) + .build() + + when: + test.annotateAgentManifest(manifest) + + then: + def innerSpan = (AgentSpan) test.span + def stored = (Map) innerSpan.getTag(AGENT_MANIFEST) + !stored.containsKey("tools") + + cleanup: + test.finish() + } + def "agent manifest on non-agent span is silently dropped"() { setup: def test = llmObsSpan(Tags.LLMOBS_LLM_SPAN_KIND, "llm-span") @@ -887,11 +907,18 @@ class DDLLMObsSpanTest extends DDSpecification{ test.finish() } - def "second annotateAgentManifest call overwrites the first"() { + def "second annotateAgentManifest call merges with the first"() { setup: def test = llmObsSpan(Tags.LLMOBS_AGENT_SPAN_KIND, "my-agent") - def first = LLMObs.AgentManifest.builder().name("first").instructions("v1").build() - def second = LLMObs.AgentManifest.builder().name("second").model("gpt-4o").build() + def first = LLMObs.AgentManifest.builder() + .name("first") + .instructions("v1 instructions") + .model("gpt-3.5") + .build() + def second = LLMObs.AgentManifest.builder() + .name("second") + .model("gpt-4o") + .build() when: test.annotateAgentManifest(first) @@ -900,9 +927,37 @@ class DDLLMObsSpanTest extends DDSpecification{ then: def innerSpan = (AgentSpan) test.span def stored = (Map) innerSpan.getTag(AGENT_MANIFEST) - stored["name"] == "second" - stored["model"] == "gpt-4o" - !stored.containsKey("instructions") + stored["name"] == "second" // second call wins on name + stored["model"] == "gpt-4o" // second call wins on model + stored["instructions"] == "v1 instructions" // first call's instructions preserved + stored["framework"] == "manual" + + cleanup: + test.finish() + } + + def "second annotateAgentManifest call merges model_settings"() { + setup: + def test = llmObsSpan(Tags.LLMOBS_AGENT_SPAN_KIND, "my-agent") + def first = LLMObs.AgentManifest.builder() + .name("agent") + .modelSettings([temperature: 0.5, max_tokens: 512]) + .build() + def second = LLMObs.AgentManifest.builder() + .modelSettings([temperature: 0.9, top_p: 0.95]) + .build() + + when: + test.annotateAgentManifest(first) + test.annotateAgentManifest(second) + + then: + def innerSpan = (AgentSpan) test.span + def stored = (Map) innerSpan.getTag(AGENT_MANIFEST) + def ms = (Map) stored["model_settings"] + ms["temperature"] == 0.9 // second wins + ms["max_tokens"] == 512 // first preserved + ms["top_p"] == 0.95 // second adds cleanup: test.finish() @@ -945,6 +1000,21 @@ class DDLLMObsSpanTest extends DDSpecification{ test.finish() } + def "annotateAgentManifest after finish is silently ignored"() { + setup: + def test = llmObsSpan(Tags.LLMOBS_AGENT_SPAN_KIND, "my-agent") + def manifest = LLMObs.AgentManifest.builder().name("agent").model("gpt-4o").build() + + when: + test.finish() + test.annotateAgentManifest(manifest) + + then: + noExceptionThrown() + def innerSpan = (AgentSpan) test.span + innerSpan.getTag(AGENT_MANIFEST) == null + } + private LLMObsSpan llmObsSpan(String kind, name) { llmObsSpan(kind, name, null) } diff --git a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObs.java b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObs.java index abac0b8d961..f9870c2675e 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObs.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObs.java @@ -1120,6 +1120,14 @@ public static AgentTool from(String name) { return new AgentTool(name, null, null); } + /** + * Creates an agent tool with a name, description, and parameter schema. + * + * @param name the tool name + * @param description an optional description of what the tool does + * @param parameters optional parameter schema; the map is shallow-copied — callers must not + * mutate nested values after construction + */ public static AgentTool from( String name, @Nullable String description, @Nullable Map parameters) { return new AgentTool(name, description, parameters); @@ -1132,6 +1140,7 @@ private AgentTool(String name, String description, Map parameter parameters == null ? null : Collections.unmodifiableMap(new LinkedHashMap<>(parameters)); } + @Nullable public String getName() { return name; } diff --git a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObsSpan.java b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObsSpan.java index 8145037637f..c5f447a0b4c 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObsSpan.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObsSpan.java @@ -64,6 +64,8 @@ default void setToolDefinitions(List toolDefinitions) {} * *

This annotation is ignored for non-agent spans. * + *

A fully-empty manifest (no fields set) still writes the {@code framework} key. + * * @param agentManifest The agent manifest configuration */ default void annotateAgentManifest(LLMObs.AgentManifest agentManifest) {} diff --git a/dd-trace-core/src/test/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.java b/dd-trace-core/src/test/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.java index 3924c31aaf4..425666c3fda 100644 --- a/dd-trace-core/src/test/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.java @@ -2,6 +2,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -825,7 +826,7 @@ void testLLMObsSpanMapperSerializesAgentManifest() throws Exception { manifest.put("name", "travel_desk"); manifest.put("instructions", "Book travel."); manifest.put("model", "gpt-4o"); - manifest.put("framework", "AgentObs SDK"); + manifest.put("framework", "manual"); Map modelSettings = new LinkedHashMap<>(); modelSettings.put("temperature", 0.7); @@ -855,8 +856,10 @@ void testLLMObsSpanMapperSerializesAgentManifest() throws Exception { assertEquals("travel_desk", gotManifest.get("name")); assertEquals("Book travel.", gotManifest.get("instructions")); assertEquals("gpt-4o", gotManifest.get("model")); - assertEquals("AgentObs SDK", gotManifest.get("framework")); + assertEquals("manual", gotManifest.get("framework")); assertEquals(modelSettings, gotManifest.get("model_settings")); + assertInstanceOf( + Double.class, ((Map) gotManifest.get("model_settings")).get("temperature")); assertEquals(tools, gotManifest.get("tools")); tracer.close(); @@ -869,7 +872,7 @@ void testAgentManifestDoesNotAppearInTags() throws Exception { Map manifest = new LinkedHashMap<>(); manifest.put("name", "my-agent"); - manifest.put("framework", "AgentObs SDK"); + manifest.put("framework", "manual"); AgentSpan agentSpan = tracer @@ -882,7 +885,7 @@ void testAgentManifestDoesNotAppearInTags() throws Exception { Map spanData = serializeSingleSpan(mapper, agentSpan); List tags = (List) spanData.get("tags"); - assertFalse(tags.stream().anyMatch(t -> t.startsWith("agent_manifest:"))); + assertFalse(tags.stream().anyMatch(t -> t.contains("agent_manifest"))); tracer.close(); } From a0e864cf0f43a6cb2fc947b10cc06e9dab7e2149 Mon Sep 17 00:00:00 2001 From: Yahya Mouman <103438582+yahya-mouman@users.noreply.github.com> Date: Fri, 28 Aug 2026 19:43:21 +0200 Subject: [PATCH 6/7] Update dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java Co-authored-by: Sam Brenner <106700075+sabrenner@users.noreply.github.com> --- .../src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java index fef711c85df..2e05ff6d679 100644 --- a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java +++ b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java @@ -321,7 +321,7 @@ private void mergeManifest(Map base, LLMObs.AgentManifest manife if (manifestName != null && !manifestName.isEmpty()) { base.put("name", manifestName); } else if (!base.containsKey("name")) { - CharSequence sn = span.getSpanName(); + CharSequence spanName = span.getSpanName(); if (sn != null && sn.length() > 0) { base.put("name", sn.toString()); } From df4a55983d452dd95b9c94e927b801a4ef5ffce9 Mon Sep 17 00:00:00 2001 From: yahya-mouman Date: Fri, 28 Aug 2026 20:00:05 +0200 Subject: [PATCH 7/7] Extract manifest getter calls to local variables; fix spanName reference; update docstring Co-Authored-By: Claude Sonnet 4.6 --- .../trace/llmobs/domain/DDLLMObsSpan.java | 29 +++++++++++-------- .../java/datadog/trace/api/llmobs/LLMObs.java | 2 +- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java index 2e05ff6d679..6147ec2424a 100644 --- a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java +++ b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java @@ -322,38 +322,43 @@ private void mergeManifest(Map base, LLMObs.AgentManifest manife base.put("name", manifestName); } else if (!base.containsKey("name")) { CharSequence spanName = span.getSpanName(); - if (sn != null && sn.length() > 0) { - base.put("name", sn.toString()); + if (spanName != null && spanName.length() > 0) { + base.put("name", spanName.toString()); } } // instructions - if (manifest.getInstructions() != null && !manifest.getInstructions().isEmpty()) { - base.put("instructions", manifest.getInstructions()); + String instructions = manifest.getInstructions(); + if (instructions != null && !instructions.isEmpty()) { + base.put("instructions", instructions); } // model - if (manifest.getModel() != null && !manifest.getModel().isEmpty()) { - base.put("model", manifest.getModel()); + String model = manifest.getModel(); + if (model != null && !model.isEmpty()) { + base.put("model", model); } // model_settings: shallow merge - if (manifest.getModelSettings() != null && !manifest.getModelSettings().isEmpty()) { + Map modelSettings = manifest.getModelSettings(); + if (modelSettings != null && !modelSettings.isEmpty()) { @SuppressWarnings("unchecked") Map existingSettings = (base.get("model_settings") instanceof Map) ? new LinkedHashMap<>((Map) base.get("model_settings")) : new LinkedHashMap<>(); - existingSettings.putAll(manifest.getModelSettings()); + existingSettings.putAll(modelSettings); base.put("model_settings", existingSettings); } // tools: replace if non-null non-empty - if (manifest.getTools() != null && !manifest.getTools().isEmpty()) { + List tools = manifest.getTools(); + if (tools != null && !tools.isEmpty()) { List> toolList = new ArrayList<>(); - for (LLMObs.AgentTool tool : manifest.getTools()) { - if (tool == null || tool.getName() == null || tool.getName().isEmpty()) { + for (LLMObs.AgentTool tool : tools) { + String toolName = tool == null ? null : tool.getName(); + if (toolName == null || toolName.isEmpty()) { LOGGER.warn("agent manifest tool missing required name; skipping"); continue; } Map toolMap = new LinkedHashMap<>(); - toolMap.put("name", tool.getName()); + toolMap.put("name", toolName); if (tool.getDescription() != null) { toolMap.put("description", tool.getDescription()); } diff --git a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObs.java b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObs.java index f9870c2675e..225f703ad77 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObs.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObs.java @@ -1162,7 +1162,7 @@ public Map getParameters() { * *

Build via {@link AgentManifest#builder()} and pass to {@link * LLMObsSpan#annotateAgentManifest(AgentManifest)}. Only applied on agent spans; ignored on other - * span kinds. A subsequent call on the same span overwrites the previous manifest. + * span kinds. A subsequent call on the same span merges with the previous manifest. */ public static final class AgentManifest { private final String name;