From 78d65771a4b1969360fd3cd0ac5b1f54e28c4c4a Mon Sep 17 00:00:00 2001 From: "benjamin.a" Date: Tue, 15 Sep 2026 22:01:13 +0900 Subject: [PATCH 1/5] Retry retransformation without the TaskWrapper interface UnwrappingVisitor adds the TaskWrapper interface to the classes it instruments. Adding an interface is a structural change, which the JVM rejects on retransformation, so this only works because those classes are normally still unloaded when the agent installs. A JDK 24+ AOT cache (JEP 483) breaks that assumption: the cached classes are materialized before premain runs, so the retransformation batch that contains them fails. There is no BatchAllocator configured, so that is the single batch holding every class, and the failure aborts the whole install - the tracer reports itself as healthy but no instrumentation is applied. Clear the flag on failure and retry the batch once, so queueing-time profiling is dropped instead of all instrumentation. Co-Authored-By: Claude Opus 5 --- .../trace/agent/tooling/AgentInstaller.java | 3 +- ...skWrapperRedefinitionStrategyListener.java | 36 +++++++++++++++++++ .../profiling/UnwrappingVisitor.java | 14 +++++++- 3 files changed, 51 insertions(+), 2 deletions(-) create mode 100644 dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/bytebuddy/profiling/TaskWrapperRedefinitionStrategyListener.java diff --git a/dd-java-agent/agent-installer/src/main/java/datadog/trace/agent/tooling/AgentInstaller.java b/dd-java-agent/agent-installer/src/main/java/datadog/trace/agent/tooling/AgentInstaller.java index 321b726d63b..efce460e83f 100644 --- a/dd-java-agent/agent-installer/src/main/java/datadog/trace/agent/tooling/AgentInstaller.java +++ b/dd-java-agent/agent-installer/src/main/java/datadog/trace/agent/tooling/AgentInstaller.java @@ -12,6 +12,7 @@ import datadog.trace.agent.tooling.bytebuddy.matcher.DDElementMatchers; import datadog.trace.agent.tooling.bytebuddy.memoize.MemoizedMatchers; import datadog.trace.agent.tooling.bytebuddy.outline.TypePoolFacade; +import datadog.trace.agent.tooling.bytebuddy.profiling.TaskWrapperRedefinitionStrategyListener; import datadog.trace.agent.tooling.usm.UsmExtractorImpl; import datadog.trace.agent.tooling.usm.UsmMessageFactoryImpl; import datadog.trace.api.InstrumenterConfig; @@ -403,7 +404,7 @@ private static AgentBuilder.RedefinitionStrategy.Listener redefinitionStrategyLi if (enabledSystems.contains(InstrumenterModule.TargetSystem.IAST)) { return TaintableRedefinitionStrategyListener.INSTANCE; } else { - return AgentBuilder.RedefinitionStrategy.Listener.NoOp.INSTANCE; + return TaskWrapperRedefinitionStrategyListener.INSTANCE; } } diff --git a/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/bytebuddy/profiling/TaskWrapperRedefinitionStrategyListener.java b/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/bytebuddy/profiling/TaskWrapperRedefinitionStrategyListener.java new file mode 100644 index 00000000000..8eac3141572 --- /dev/null +++ b/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/bytebuddy/profiling/TaskWrapperRedefinitionStrategyListener.java @@ -0,0 +1,36 @@ +package datadog.trace.agent.tooling.bytebuddy.profiling; + +import java.util.Collections; +import java.util.List; +import javax.annotation.Nonnull; +import net.bytebuddy.agent.builder.AgentBuilder; + +/** + * {@link UnwrappingVisitor} redefines the structure of a class by adding an interface, meaning that + * it cannot be applied to already loaded classes. + * + *

This listener disables the visitor and retries the batch, so that unretransformable classes do + * not abort the whole instrumentation install. + */ +public final class TaskWrapperRedefinitionStrategyListener + extends AgentBuilder.RedefinitionStrategy.Listener.Adapter { + + public static final TaskWrapperRedefinitionStrategyListener INSTANCE = + new TaskWrapperRedefinitionStrategyListener(); + + private TaskWrapperRedefinitionStrategyListener() {} + + @Override + @Nonnull + public Iterable>> onError( + final int index, + @Nonnull final List> batch, + @Nonnull final Throwable throwable, + @Nonnull final List> types) { + if (UnwrappingVisitor.ENABLED) { + UnwrappingVisitor.ENABLED = false; + return Collections.singletonList(batch); + } + return Collections.emptyList(); + } +} diff --git a/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/bytebuddy/profiling/UnwrappingVisitor.java b/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/bytebuddy/profiling/UnwrappingVisitor.java index 1c8e07ecaf4..3146634f984 100644 --- a/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/bytebuddy/profiling/UnwrappingVisitor.java +++ b/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/bytebuddy/profiling/UnwrappingVisitor.java @@ -18,6 +18,18 @@ public class UnwrappingVisitor implements AsmVisitorWrapper { + /** + * Adding the {@code TaskWrapper} interface changes the structure of a class, which the JVM + * rejects for classes that are already loaded ("attempted to change superclass or interfaces"). + * That normally does not happen because the agent installs before those classes load, but a JDK + * 24+ AOT cache (JEP 483) materializes them up-front, so the whole retransform batch fails and no + * further instrumentation is installed. + * + *

When that happens {@link TaskWrapperRedefinitionStrategyListener} clears this flag and + * retries the batch, trading queueing-time profiling for a working install. + */ + public static volatile boolean ENABLED = true; + private final Map classNameToDelegateFieldNames; public UnwrappingVisitor(String... classAndDelegateFieldNames) { @@ -81,7 +93,7 @@ public void visit( String signature, String superName, String[] interfaces) { - if (interfaces == null || !Arrays.asList(interfaces).contains(TASK_WRAPPER)) { + if (ENABLED && (interfaces == null || !Arrays.asList(interfaces).contains(TASK_WRAPPER))) { interfaces = append(interfaces, TASK_WRAPPER); if (signature != null) { signature += 'L' + TASK_WRAPPER + ';'; From 8736e87807df7a38b5da67071ec5df4ff75c7150 Mon Sep 17 00:00:00 2001 From: "benjamin.a" Date: Tue, 15 Sep 2026 22:54:52 +0900 Subject: [PATCH 2/5] Align the listener with the IAST one and cover it with a test Mirror TaintableRedefinitionStrategyListener: same debug logging and the same onComplete hook, so both structural visitors behave the same way. Chain the two listeners when IAST is enabled. Both visitors change the structure of a class, so both need a chance to back off; previously only the IAST one ran and an AOT cache still took the whole install down. Add a test asserting that neither the interface nor the generated unwrap method is emitted once the visitor is disabled. Co-Authored-By: Claude Opus 5 --- .../trace/agent/tooling/AgentInstaller.java | 5 ++- ...skWrapperRedefinitionStrategyListener.java | 32 +++++++++++++-- .../profiling/UnwrappingVisitorTest.groovy | 39 +++++++++++++++++++ 3 files changed, 72 insertions(+), 4 deletions(-) diff --git a/dd-java-agent/agent-installer/src/main/java/datadog/trace/agent/tooling/AgentInstaller.java b/dd-java-agent/agent-installer/src/main/java/datadog/trace/agent/tooling/AgentInstaller.java index efce460e83f..72ebdeea455 100644 --- a/dd-java-agent/agent-installer/src/main/java/datadog/trace/agent/tooling/AgentInstaller.java +++ b/dd-java-agent/agent-installer/src/main/java/datadog/trace/agent/tooling/AgentInstaller.java @@ -402,7 +402,10 @@ private static void temporaryOverride(String key, String value, BooleanSupplier private static AgentBuilder.RedefinitionStrategy.Listener redefinitionStrategyListener( final Set enabledSystems) { if (enabledSystems.contains(InstrumenterModule.TargetSystem.IAST)) { - return TaintableRedefinitionStrategyListener.INSTANCE; + // both visitors change the structure of a class, so both need a chance to back off + return new AgentBuilder.RedefinitionStrategy.Listener.Compound( + TaintableRedefinitionStrategyListener.INSTANCE, + TaskWrapperRedefinitionStrategyListener.INSTANCE); } else { return TaskWrapperRedefinitionStrategyListener.INSTANCE; } diff --git a/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/bytebuddy/profiling/TaskWrapperRedefinitionStrategyListener.java b/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/bytebuddy/profiling/TaskWrapperRedefinitionStrategyListener.java index 8eac3141572..9f8346584c4 100644 --- a/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/bytebuddy/profiling/TaskWrapperRedefinitionStrategyListener.java +++ b/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/bytebuddy/profiling/TaskWrapperRedefinitionStrategyListener.java @@ -2,19 +2,25 @@ import java.util.Collections; import java.util.List; +import java.util.Map; import javax.annotation.Nonnull; import net.bytebuddy.agent.builder.AgentBuilder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * {@link UnwrappingVisitor} redefines the structure of a class by adding an interface, meaning that * it cannot be applied to already loaded classes. * - *

This listener disables the visitor and retries the batch, so that unretransformable classes do - * not abort the whole instrumentation install. + *

This listener will disable the visitor to prevent a failure with the whole redefinition batch. */ public final class TaskWrapperRedefinitionStrategyListener extends AgentBuilder.RedefinitionStrategy.Listener.Adapter { + private static final Logger LOGGER = + LoggerFactory.getLogger(TaskWrapperRedefinitionStrategyListener.class); + private static final boolean DEBUG = LOGGER.isDebugEnabled(); + public static final TaskWrapperRedefinitionStrategyListener INSTANCE = new TaskWrapperRedefinitionStrategyListener(); @@ -28,9 +34,29 @@ public Iterable>> onError( @Nonnull final Throwable throwable, @Nonnull final List> types) { if (UnwrappingVisitor.ENABLED) { + if (DEBUG) { + LOGGER.debug( + "Exception while retransforming with the visitor in batch {}, disabling it", index); + } UnwrappingVisitor.ENABLED = false; return Collections.singletonList(batch); + } else { + if (DEBUG) { + LOGGER.debug( + "Exception while retransforming after disabling the visitor in batch {}, queueing time profiling is disabled", + index); + } + return Collections.emptyList(); + } + } + + @Override + public void onComplete( + final int amount, final List> types, final Map>, Throwable> failures) { + if (DEBUG) { + if (!UnwrappingVisitor.ENABLED) { + LOGGER.debug("Retransforming succeeded with a disabled visitor"); + } } - return Collections.emptyList(); } } diff --git a/dd-java-agent/agent-tooling/src/test/groovy/datadog/trace/agent/tooling/bytebuddy/profiling/UnwrappingVisitorTest.groovy b/dd-java-agent/agent-tooling/src/test/groovy/datadog/trace/agent/tooling/bytebuddy/profiling/UnwrappingVisitorTest.groovy index 2b46c89a6cc..2866ed52278 100644 --- a/dd-java-agent/agent-tooling/src/test/groovy/datadog/trace/agent/tooling/bytebuddy/profiling/UnwrappingVisitorTest.groovy +++ b/dd-java-agent/agent-tooling/src/test/groovy/datadog/trace/agent/tooling/bytebuddy/profiling/UnwrappingVisitorTest.groovy @@ -6,6 +6,7 @@ import java.util.concurrent.FutureTask import net.bytebuddy.jar.asm.ClassReader import net.bytebuddy.jar.asm.ClassVisitor import net.bytebuddy.jar.asm.ClassWriter +import net.bytebuddy.jar.asm.MethodVisitor import net.bytebuddy.jar.asm.signature.SignatureReader import net.bytebuddy.jar.asm.signature.SignatureVisitor import org.apache.commons.io.IOUtils @@ -57,6 +58,44 @@ class UnwrappingVisitorTest extends Specification { genericInterfaces.last() == 'datadog/trace/bootstrap/instrumentation/api/TaskWrapper' } + void 'test the visitor backs off when disabled'(){ + setup: + def classFile = readClassBytes(FutureTask) + def classReader = new ClassReader(classFile) + def classWriter = new ClassWriter(classReader, 0) + + def declaredInterfaces = [] + def methodNames = [] + + def classVisitor = new ClassVisitor(ASM_API, classWriter) { + @Override + void visit(int version, int access, String name, String signature, String superName, String[] interfaces) { + declaredInterfaces = interfaces + super.visit(version, access, name, signature, superName, interfaces) + } + + @Override + MethodVisitor visitMethod(int access, String name, String descriptor, String signature, String[] exceptions) { + methodNames += name + return super.visitMethod(access, name, descriptor, signature, exceptions) + } + } + + def taskVisitor = new UnwrappingVisitor.ImplementTaskWrapperClassVisitor( + classVisitor, 'java.util.concurrent.FutureTask', 'callable') + + when: + UnwrappingVisitor.ENABLED = false + classReader.accept(taskVisitor, 0) + + then: 'neither the interface nor the unwrap method is added' + !declaredInterfaces.contains('datadog/trace/bootstrap/instrumentation/api/TaskWrapper') + !methodNames.contains('$$DD$$__unwrap') + + cleanup: + UnwrappingVisitor.ENABLED = true + } + static byte [] readClassBytes(Class clazz){ final String classResourceName = '/' + clazz.getName().replace('.', '/') + '.class' try (InputStream is = clazz.getResourceAsStream(classResourceName)) { From 205064e378b03e3e7cf11bef7032f36400e969f1 Mon Sep 17 00:00:00 2001 From: gibKim Date: Thu, 17 Sep 2026 07:04:17 +0900 Subject: [PATCH 3/5] Log once at info when queueing time profiling is disabled Silent feature loss was the reported problem, so the one-time recovery notice is logged at info instead of debug. The flag is one-way, so the message cannot repeat; the per-batch retry and completion logs stay at debug to keep startup logs quiet. --- .../TaskWrapperRedefinitionStrategyListener.java | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/bytebuddy/profiling/TaskWrapperRedefinitionStrategyListener.java b/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/bytebuddy/profiling/TaskWrapperRedefinitionStrategyListener.java index 9f8346584c4..e2ec27e2e21 100644 --- a/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/bytebuddy/profiling/TaskWrapperRedefinitionStrategyListener.java +++ b/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/bytebuddy/profiling/TaskWrapperRedefinitionStrategyListener.java @@ -34,10 +34,13 @@ public Iterable>> onError( @Nonnull final Throwable throwable, @Nonnull final List> types) { if (UnwrappingVisitor.ENABLED) { - if (DEBUG) { - LOGGER.debug( - "Exception while retransforming with the visitor in batch {}, disabling it", index); - } + // logged once: the flag is one-way, so this branch cannot be re-entered + LOGGER.info( + "Disabling queueing time profiling: retransformation failed for a batch of {} classes" + + " because adding the TaskWrapper interface is a structural change the JVM rejects" + + " for already loaded classes (e.g. materialized from a JDK 24+ AOT cache)." + + " Retrying the batch without it; all other instrumentation is preserved.", + batch.size()); UnwrappingVisitor.ENABLED = false; return Collections.singletonList(batch); } else { From d1be92e037ee0dc3ae24126860daf43dd87f66e7 Mon Sep 17 00:00:00 2001 From: "benjamin.a" Date: Fri, 18 Sep 2026 09:26:42 +0900 Subject: [PATCH 4/5] Say task unwrapping, not queueing time, in the log and javadoc Clearing the flag does not turn queueing-time profiling off. TaskWrapper is only read by QueueTimeEvent.setTask(), and TaskWrapper.getUnwrappedType is guarded by instanceof, so without the interface it returns the object's own class. The event is still recorded with its duration, scheduler, queue type, queue length and span ids -- only the reported task type loses its resolution. Measured with -Ddd.profiling.debug.dump_path and -Ddd.profiling.queueing.time.threshold.millis=0, 500 requests: AOT off 12 datadog.QueueTime, real task types AOT on (flag cleared on retry) 7 datadog.QueueTime, wrapper type queueing.time.enabled=false 0 datadog.QueueTime Co-Authored-By: Claude Opus 5 --- .../TaskWrapperRedefinitionStrategyListener.java | 10 ++++++---- .../tooling/bytebuddy/profiling/UnwrappingVisitor.java | 5 ++++- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/bytebuddy/profiling/TaskWrapperRedefinitionStrategyListener.java b/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/bytebuddy/profiling/TaskWrapperRedefinitionStrategyListener.java index e2ec27e2e21..49d914562fc 100644 --- a/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/bytebuddy/profiling/TaskWrapperRedefinitionStrategyListener.java +++ b/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/bytebuddy/profiling/TaskWrapperRedefinitionStrategyListener.java @@ -36,10 +36,12 @@ public Iterable>> onError( if (UnwrappingVisitor.ENABLED) { // logged once: the flag is one-way, so this branch cannot be re-entered LOGGER.info( - "Disabling queueing time profiling: retransformation failed for a batch of {} classes" - + " because adding the TaskWrapper interface is a structural change the JVM rejects" - + " for already loaded classes (e.g. materialized from a JDK 24+ AOT cache)." - + " Retrying the batch without it; all other instrumentation is preserved.", + "Disabling task unwrapping for queueing time profiling: retransformation failed for a" + + " batch of {} classes because adding the TaskWrapper interface is a structural" + + " change the JVM rejects for already loaded classes (e.g. materialized from a" + + " JDK 24+ AOT cache). Retrying the batch without it. Queueing time is still" + + " recorded, but the task type is reported as the wrapper class. All other" + + " instrumentation is preserved.", batch.size()); UnwrappingVisitor.ENABLED = false; return Collections.singletonList(batch); diff --git a/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/bytebuddy/profiling/UnwrappingVisitor.java b/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/bytebuddy/profiling/UnwrappingVisitor.java index 3146634f984..97b20b6d90f 100644 --- a/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/bytebuddy/profiling/UnwrappingVisitor.java +++ b/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/bytebuddy/profiling/UnwrappingVisitor.java @@ -26,7 +26,10 @@ public class UnwrappingVisitor implements AsmVisitorWrapper { * further instrumentation is installed. * *

When that happens {@link TaskWrapperRedefinitionStrategyListener} clears this flag and - * retries the batch, trading queueing-time profiling for a working install. + * retries the batch, trading task unwrapping for a working install. Queueing time itself keeps + * working, since {@code TaskWrapper} is only read by {@code QueueTimeEvent.setTask()} and {@code + * TaskWrapper.getUnwrappedType} falls back to the object's own class; only the reported task type + * loses its resolution. */ public static volatile boolean ENABLED = true; From d0c90c89c701dbe12761c55e8351bd55eab2720c Mon Sep 17 00:00:00 2001 From: "benjamin.a" Date: Wed, 23 Sep 2026 09:07:22 +0900 Subject: [PATCH 5/5] Say task unwrapping in the retry debug log too The previous commit renamed the info message and javadoc, but the debug message on the second failure still said queueing time profiling is disabled. It is not: only task unwrapping is. --- .../profiling/TaskWrapperRedefinitionStrategyListener.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/bytebuddy/profiling/TaskWrapperRedefinitionStrategyListener.java b/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/bytebuddy/profiling/TaskWrapperRedefinitionStrategyListener.java index 49d914562fc..8835cccdbf6 100644 --- a/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/bytebuddy/profiling/TaskWrapperRedefinitionStrategyListener.java +++ b/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/bytebuddy/profiling/TaskWrapperRedefinitionStrategyListener.java @@ -48,7 +48,7 @@ public Iterable>> onError( } else { if (DEBUG) { LOGGER.debug( - "Exception while retransforming after disabling the visitor in batch {}, queueing time profiling is disabled", + "Exception while retransforming after disabling the visitor in batch {}, task unwrapping is disabled", index); } return Collections.emptyList();