From cb8ddef063bf7b99e289d4b87d6b4268e9ffe955 Mon Sep 17 00:00:00 2001 From: Andrea Marziali Date: Thu, 11 Jun 2026 08:45:32 +0200 Subject: [PATCH 01/35] Introduce systematic continuation leakage diagnostics --- .agents/skills/apm-integrations/SKILL.md | 7 +- .../skills/fix-continuation-leakage/SKILL.md | 56 +++ .../skills/fix-continuation-leakage/SKILL.md | 23 ++ .../test/InstrumentationSpecification.groovy | 78 +++- .../test/AbstractInstrumentationTest.java | 4 +- .../scopediag/ContinuableScopeAdvice.java | 39 ++ .../test/scopediag/ContinuationAdvice.java | 76 ++++ .../test/scopediag/ContinuationRecord.java | 240 +++++++++++ .../test/scopediag/ContinuationStatus.java | 18 + .../trace/agent/test/scopediag/Failure.java | 21 + .../test/scopediag/PendingTraceAdvice.java | 33 ++ .../scopediag/ScopeContinuationProbe.java | 319 +++++++++++++++ .../ScopeContinuationTransformer.java | 65 +++ .../test/scopediag/ScopeDiagnostics.java | 359 +++++++++++++++++ .../scopediag/ScopeDiagnosticsExtension.java | 62 +++ .../scopediag/ScopeDiagnosticsReport.java | 381 ++++++++++++++++++ .../agent/test/scopediag/ScopeEvent.java | 44 ++ .../agent/test/scopediag/ScopeRecord.java | 134 ++++++ .../agent/test/scopediag/ScopeSources.java | 25 ++ .../agent/test/scopediag/StackFilter.java | 75 ++++ .../scopediag/TrackScopeContinuations.java | 25 ++ .../scopediag/ScopeContinuationProbeTest.java | 99 +++++ .../ScopeDiagnosticsConfigurationTest.java | 36 ++ .../ScopeDiagnosticsIntegrationTest.java | 175 ++++++++ .../scopediag/ScopeDiagnosticsReportTest.java | 179 ++++++++ .../agent/test/scopediag/ScopeRecordTest.java | 86 ++++ .../agent/test/scopediag/StackFilterTest.java | 53 +++ 27 files changed, 2696 insertions(+), 16 deletions(-) create mode 100644 .agents/skills/fix-continuation-leakage/SKILL.md create mode 100644 .claude/skills/fix-continuation-leakage/SKILL.md create mode 100644 dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ContinuableScopeAdvice.java create mode 100644 dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ContinuationAdvice.java create mode 100644 dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ContinuationRecord.java create mode 100644 dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ContinuationStatus.java create mode 100644 dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/Failure.java create mode 100644 dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/PendingTraceAdvice.java create mode 100644 dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeContinuationProbe.java create mode 100644 dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeContinuationTransformer.java create mode 100644 dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeDiagnostics.java create mode 100644 dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsExtension.java create mode 100644 dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsReport.java create mode 100644 dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeEvent.java create mode 100644 dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeRecord.java create mode 100644 dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeSources.java create mode 100644 dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/StackFilter.java create mode 100644 dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/TrackScopeContinuations.java create mode 100644 dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeContinuationProbeTest.java create mode 100644 dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsConfigurationTest.java create mode 100644 dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsIntegrationTest.java create mode 100644 dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsReportTest.java create mode 100644 dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeRecordTest.java create mode 100644 dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/StackFilterTest.java diff --git a/.agents/skills/apm-integrations/SKILL.md b/.agents/skills/apm-integrations/SKILL.md index 6553df66b1c..ebfcd4920a0 100644 --- a/.agents/skills/apm-integrations/SKILL.md +++ b/.agents/skills/apm-integrations/SKILL.md @@ -150,8 +150,11 @@ After `updateAgentJarIntegrationsGoldenFile` runs, commit the updated `metadata/ **If `checkDecoratorAnalyticsConfigurations` fails:** a name returned by the decorator's `instrumentationNames()` is missing `DD_TRACE__ANALYTICS_ENABLED` / `DD_TRACE__ANALYTICS_SAMPLE_RATE` entries in `metadata/supported-configurations.json` — add them per [Supported Configurations](references/supported-configurations.md). -**If tests fail:** verify span lifecycle order (start → activate → error → close → finish), helper registration, -and `contextStore()` map entries match actual usage. +**If tests fail:** verify span lifecycle order (start → activate → error → close → finish), helper +registration, and `contextStore()` map entries match actual usage. If the output contains +`Scope/continuation timeline`, read and follow +`.agents/skills/fix-continuation-leakage/SKILL.md`; fix the broken ownership edge rather than +adding `strictTraceWrites(false)` or disabling the diagnostic. ## Step 11 – Checklist before finishing diff --git a/.agents/skills/fix-continuation-leakage/SKILL.md b/.agents/skills/fix-continuation-leakage/SKILL.md new file mode 100644 index 00000000000..961f3d88eb4 --- /dev/null +++ b/.agents/skills/fix-continuation-leakage/SKILL.md @@ -0,0 +1,56 @@ +--- +name: fix-continuation-leakage +description: > + Diagnose and fix scope or continuation lifecycle failures in dd-trace-java instrumentation + tests. Use when a test reports a continuation leak, double resolution, activation after resolve, + or an unclosed scope, or when strictTraceWrites(false) appears to hide one. Reads the automatic + diagnostic timeline, finds the broken lifecycle edge, fixes it, and explains it with a compact + Mermaid diagram. +user-invocable: true +context: fork +allowed-tools: + - Bash + - Read + - Edit + - Glob + - Grep + - AskUserQuestion +--- + +# Fix continuation leakage + +Instrumentation tests run the diagnostic automatically. A failure includes the capture, resume, +resolution, scope, thread, timing, and callsite data needed to find the missing lifecycle edge. + +## Work the failure + +1. Run the smallest failing test with full output: + +```bash +./gradlew :dd-java-agent:instrumentation:-:test --tests '' --info 2>&1 | tee /tmp/scopediag-run.txt +``` + +2. Find `Scope/continuation timeline` in the output. If Gradle hides it, inspect the test XML's + `` under the module's `build/test-results` directory. +3. Follow the failing record from its first event: + - `LEAKED` / `NEVER_CLOSED`: find the success, error, cancellation, and rejection exits that + skipped `release()` or `close()`. + - `DOUBLE_FINISH`: find two owners of the same cleanup. + - `ACTIVATE_AFTER_RESOLVE`: find work scheduled after ownership ended. + - `LATE_FINISH` / `CLOSE_WRONG_THREAD`: advisory evidence; verify whether ordering is valid. +4. Fix ownership where it breaks. Prefer one owner and `try/finally` cleanup across every exit. +5. Rerun the failing test, then its module. + +Do not make the test green with `strictTraceWrites(false)` or +`@TrackScopeContinuations(enabled=false, reason="...")`. Those hide evidence. The opt-out requires +a reason and is only for a proven diagnostic incompatibility. If the failure is genuinely +intermittent, treat that as a flaky-test finding, keep diagnostics enabled, and link the `@Flaky` +annotation to a tracked issue. + +## Explain it to a human + +Lead with one sentence: what was captured, which cleanup edge was missing, and where. Cite the +timeline callsites. Then include a small Mermaid `flowchart LR`; use green for healthy edges, red +for the broken edge, and label thread handoffs. Use a Gantt only when timing itself caused the bug. + +End with the code fix and the exact tests that passed. diff --git a/.claude/skills/fix-continuation-leakage/SKILL.md b/.claude/skills/fix-continuation-leakage/SKILL.md new file mode 100644 index 00000000000..fa436670a4b --- /dev/null +++ b/.claude/skills/fix-continuation-leakage/SKILL.md @@ -0,0 +1,23 @@ +--- +name: fix-continuation-leakage +description: > + Diagnose and fix scope or continuation lifecycle failures in dd-trace-java instrumentation + tests. Use when a test reports a continuation leak, double resolution, activation after resolve, + or an unclosed scope, or when strictTraceWrites(false) appears to hide one. Reads the automatic + diagnostic timeline, finds the broken lifecycle edge, fixes it, and explains it with a compact + Mermaid diagram. +user-invocable: true +context: fork +allowed-tools: + - Bash + - Read + - Edit + - Glob + - Grep + - AskUserQuestion +--- + +# Fix continuation leakage + +Read `.agents/skills/fix-continuation-leakage/SKILL.md` in full and follow it. That file is the +shared playbook for repository agents. diff --git a/dd-java-agent/instrumentation-testing/src/main/groovy/datadog/trace/agent/test/InstrumentationSpecification.groovy b/dd-java-agent/instrumentation-testing/src/main/groovy/datadog/trace/agent/test/InstrumentationSpecification.groovy index d1a53d79762..0ec8227a0a8 100644 --- a/dd-java-agent/instrumentation-testing/src/main/groovy/datadog/trace/agent/test/InstrumentationSpecification.groovy +++ b/dd-java-agent/instrumentation-testing/src/main/groovy/datadog/trace/agent/test/InstrumentationSpecification.groovy @@ -33,6 +33,8 @@ import datadog.metrics.impl.DDSketchHistograms import datadog.metrics.impl.MonitoringImpl import datadog.trace.agent.test.asserts.ListWriterAssert import datadog.trace.agent.test.asserts.TagsAssert +import datadog.trace.agent.test.scopediag.ScopeDiagnostics +import datadog.trace.agent.test.scopediag.TrackScopeContinuations import datadog.trace.agent.test.datastreams.MockFeaturesDiscovery import datadog.trace.agent.test.datastreams.RecordingDatastreamsPayloadWriter import datadog.trace.agent.tooling.AgentInstaller @@ -467,6 +469,9 @@ abstract class InstrumentationSpecification extends DDSpecification implements A } TEST_WRITER.start() + if (scopeDiagnosticsEnabled()) { + ScopeDiagnostics.startRecording() + } TEST_DATA_STREAMS_WRITER.clear() TEST_DATA_STREAMS_MONITORING.clear() @@ -500,27 +505,74 @@ abstract class InstrumentationSpecification extends DDSpecification implements A } TEST_TRACER.flush() - def util = new MockUtil() - util.detachMock(STATS_D_CLIENT) + def scopeDiagnosticsFailure = reportScopeDiagnostics() + + try { + def util = new MockUtil() + util.detachMock(STATS_D_CLIENT) + + ActiveSubsystems.APPSEC_ACTIVE = originalAppSecRuntimeValue + + if (Config.get().isDebuggerCodeOriginEnabled()) { + injectSysConfig(CODE_ORIGIN_FOR_SPANS_ENABLED, "false", true) + rebuildConfig() + } - ActiveSubsystems.APPSEC_ACTIVE = originalAppSecRuntimeValue + try { + if (enabledFinishTimingChecks()) { + doCheckRepeatedFinish() + } + } finally { + spanFinishLocations.clear() + originalToTrackingSpan.clear() + } - if (Config.get().isDebuggerCodeOriginEnabled()) { - injectSysConfig(CODE_ORIGIN_FOR_SPANS_ENABLED, "false", true) - rebuildConfig() + // check for instrumentation issues while running each test + assert InstrumentationErrors.noErrors(): InstrumentationErrors.describeErrors() + } catch (Throwable cleanupFailure) { + if (scopeDiagnosticsFailure != null) { + cleanupFailure.addSuppressed(scopeDiagnosticsFailure) + } + throw cleanupFailure + } + if (scopeDiagnosticsFailure != null) { + throw scopeDiagnosticsFailure } + } + + /** Resolves the {@link TrackScopeContinuations} annotation from the feature method or spec class. */ + private TrackScopeContinuations scopeDiagConfig() { + def method = specificationContext?.currentFeature?.featureMethod?.reflection + def ann = method?.getAnnotation(TrackScopeContinuations) + if (ann == null) { + ann = this.class.getAnnotation(TrackScopeContinuations) + } + return ann + } + + private boolean scopeDiagnosticsEnabled() { + return ScopeDiagnostics.isEnabled(scopeDiagConfig()) + } + /** Captures the diagnostic failure so the rest of Spock cleanup always runs first. */ + private Throwable reportScopeDiagnostics() { + def config = scopeDiagConfig() + if (!ScopeDiagnostics.isEnabled(config)) { + return null + } try { - if (enabledFinishTimingChecks()) { - doCheckRepeatedFinish() + ScopeDiagnostics.stop() + def report = ScopeDiagnostics.report() + if (report.hasFindings()) { + println(report.renderTimeline()) } + ScopeDiagnostics.assertNoLeaks(report) + return null + } catch (Throwable failure) { + return failure } finally { - spanFinishLocations.clear() - originalToTrackingSpan.clear() + ScopeDiagnostics.reset() } - - // check for instrumentation issues while running each test - assert InstrumentationErrors.noErrors(): InstrumentationErrors.describeErrors() } private void doCheckRepeatedFinish() { diff --git a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/AbstractInstrumentationTest.java b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/AbstractInstrumentationTest.java index a6a2d83ebc0..2efee9eccbe 100644 --- a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/AbstractInstrumentationTest.java +++ b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/AbstractInstrumentationTest.java @@ -7,6 +7,7 @@ import datadog.instrument.classinject.ClassInjector; import datadog.trace.agent.test.assertions.TraceAssertions; import datadog.trace.agent.test.assertions.TraceMatcher; +import datadog.trace.agent.test.scopediag.ScopeDiagnosticsExtension; import datadog.trace.agent.tooling.AgentInstaller; import datadog.trace.agent.tooling.InstrumenterModule; import datadog.trace.agent.tooling.TracerInstaller; @@ -58,7 +59,8 @@ @ExtendWith({ TestClassShadowingExtension.class, AllowContextTestingExtension.class, - LegacyContextTestingExtension.class + LegacyContextTestingExtension.class, + ScopeDiagnosticsExtension.class }) public abstract class AbstractInstrumentationTest { static final Instrumentation INSTRUMENTATION = ByteBuddyAgent.getInstrumentation(); diff --git a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ContinuableScopeAdvice.java b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ContinuableScopeAdvice.java new file mode 100644 index 00000000000..3dab0453128 --- /dev/null +++ b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ContinuableScopeAdvice.java @@ -0,0 +1,39 @@ +package datadog.trace.agent.test.scopediag; + +import net.bytebuddy.asm.Advice; + +/** + * Test-only ByteBuddy advice woven into {@code datadog.trace.core.scopemanager.ContinuableScope} + * (and, by inheritance, {@code ContinuingScope}) to track the scope activation lifecycle. + * + *

The target type is package-private, so {@code this} is typed as {@link Object} and re-cast + * inside {@link ScopeContinuationProbe}. {@code afterActivated} is the open point (first call per + * scope identity), {@code onProperClose} the pop, and {@code close} the wrong-thread check. + */ +public final class ContinuableScopeAdvice { + private ContinuableScopeAdvice() {} + + /** {@code afterActivated()} — the scope became active. */ + public static final class AfterActivated { + @Advice.OnMethodExit(suppress = Throwable.class) + public static void exit(@Advice.This Object scope) { + ScopeContinuationProbe.onScopeOpen(scope); + } + } + + /** {@code onProperClose()} — the scope was popped from its thread's stack. */ + public static final class OnProperClose { + @Advice.OnMethodExit(suppress = Throwable.class) + public static void exit(@Advice.This Object scope) { + ScopeContinuationProbe.onScopeClose(scope); + } + } + + /** {@code close()} — check for an out-of-order / wrong-thread close. */ + public static final class Close { + @Advice.OnMethodEnter(suppress = Throwable.class) + public static void enter(@Advice.This Object scope) { + ScopeContinuationProbe.onScopeClosing(scope); + } + } +} diff --git a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ContinuationAdvice.java b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ContinuationAdvice.java new file mode 100644 index 00000000000..8264d3c87da --- /dev/null +++ b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ContinuationAdvice.java @@ -0,0 +1,76 @@ +package datadog.trace.agent.test.scopediag; + +import net.bytebuddy.asm.Advice; + +/** + * Test-only ByteBuddy advice woven into {@code datadog.trace.core.scopemanager.ScopeContinuation}. + * + *

The target type is package-private and cannot be named here, so {@code this} is typed as + * {@link Object} and re-cast to the public {@code ContextContinuation} supertype inside {@link + * ScopeContinuationProbe}. {@link Advice.FieldValue} reads the private {@code count} field — legal + * because the advice is inlined into the field's own class. + */ +public final class ContinuationAdvice { + private ContinuationAdvice() {} + + /** {@code register()} — the continuation was captured. */ + public static final class Register { + @Advice.OnMethodExit(suppress = Throwable.class) + public static void exit(@Advice.This Object self) { + ScopeContinuationProbe.onCapture(self); + } + } + + /** + * {@code resume()} — a (possibly noop) activation; the probe filters the rollback branch. + * + *

The activation timestamp is captured at method entry, not exit: the same-span reuse + * optimization ({@code ContinuableScopeManager.continueSpan}) cancels the continuation from + * inside {@code resume()} before it returns, so timestamping the resume at exit would + * order it after that internal resolution and spuriously flag {@code ACTIVATE_AFTER_RESOLVE}. + */ + public static final class Activate { + @Advice.OnMethodEnter + public static long enter() { + return System.nanoTime(); + } + + @Advice.OnMethodExit(suppress = Throwable.class) + public static void exit( + @Advice.This Object self, @Advice.Enter long ddActivateNanos, @Advice.Return Object scope) { + ScopeContinuationProbe.onActivate(self, scope, ddActivateNanos); + } + } + + /** + * Resolution detected via the {@code count} transition. Applied to both {@code release()} and + * {@code cancelFromContinuedScopeClose()} — they need identical before/after observation. The + * originating method name ({@code #m}) distinguishes an explicit cancel from a normal + * finish-on-scope-close. + * + *

The resolve timestamp is captured at method entry (the {@code ddResolveNanos} + * local), not at exit: the body itself may call {@code removeContinuation() -> + * PendingTrace.write()}, which is exactly where the root-written timestamp is taken. Timestamping + * at exit would place the resolution after the root write it triggered, producing a spurious + * late-finish. + */ + public static final class Cancel { + @Advice.OnMethodEnter + public static int enter( + @Advice.FieldValue("count") int count, + @Advice.Local("ddResolveNanos") long ddResolveNanos) { + ddResolveNanos = System.nanoTime(); + return count; + } + + @Advice.OnMethodExit(suppress = Throwable.class) + public static void exit( + @Advice.This Object self, + @Advice.Origin("#m") String method, + @Advice.Enter int countBefore, + @Advice.Local("ddResolveNanos") long ddResolveNanos, + @Advice.FieldValue("count") int countAfter) { + ScopeContinuationProbe.onResolve(self, method, countBefore, countAfter, ddResolveNanos); + } + } +} diff --git a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ContinuationRecord.java b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ContinuationRecord.java new file mode 100644 index 00000000000..d272eb90f83 --- /dev/null +++ b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ContinuationRecord.java @@ -0,0 +1,240 @@ +package datadog.trace.agent.test.scopediag; + +import datadog.trace.api.DDTraceId; +import java.util.ArrayList; +import java.util.EnumSet; +import java.util.List; + +/** + * The correlated continuation lifecycle: its capture, every resume (activation), any + * failed activation, and its terminal resolution (finish/cancel). Scope activation lifetimes are + * modelled separately by {@link ScopeRecord}; the scopes a continuation spawned are linked here by + * their seq ids ({@link #scopeRecordSeqs()}). + * + *

Built incrementally as events arrive on different threads, so all mutating access is + * synchronized on the instance. + */ +public final class ContinuationRecord { + public final long seq; + public final DDTraceId traceId; + public final long spanId; + public final String spanName; + public final byte source; + + /** {@code true} when a resume/resolution was seen without a preceding capture in this window. */ + public final boolean orphan; + + private final ScopeEvent capture; + private final List resumes = new ArrayList<>(1); + private final List failedActivations = new ArrayList<>(0); + private ScopeEvent terminal; + private final List extraTerminals = new ArrayList<>(0); + private final List scopeRecordSeqs = new ArrayList<>(1); + + ContinuationRecord( + long seq, + DDTraceId traceId, + long spanId, + String spanName, + byte source, + boolean orphan, + ScopeEvent capture) { + this.seq = seq; + this.traceId = traceId; + this.spanId = spanId; + this.spanName = spanName; + this.source = source; + this.orphan = orphan; + this.capture = capture; + } + + // ---- mutation ------------------------------------------------------------ + + synchronized void addResume(ScopeEvent event) { + resumes.add(event); + } + + synchronized void addFailedActivation(ScopeEvent event) { + failedActivations.add(event); + } + + /** First terminal sets {@link #terminal}; any subsequent terminal is a double-finish signal. */ + synchronized void setTerminalOrExtra(ScopeEvent event) { + if (terminal == null) { + terminal = event; + } else { + extraTerminals.add(event); + } + } + + synchronized void linkScope(long scopeSeq) { + scopeRecordSeqs.add(scopeSeq); + } + + synchronized ContinuationRecord snapshot() { + ContinuationRecord copy = + new ContinuationRecord( + seq, + traceId, + spanId, + spanName, + source, + orphan, + capture == null ? null : capture.snapshot()); + for (ScopeEvent event : resumes) { + copy.resumes.add(event.snapshot()); + } + for (ScopeEvent event : failedActivations) { + copy.failedActivations.add(event.snapshot()); + } + copy.terminal = terminal == null ? null : terminal.snapshot(); + for (ScopeEvent event : extraTerminals) { + copy.extraTerminals.add(event.snapshot()); + } + copy.scopeRecordSeqs.addAll(scopeRecordSeqs); + return copy; + } + + // ---- accessors ----------------------------------------------------------- + + public synchronized ScopeEvent capture() { + return capture; + } + + public synchronized List resumes() { + return new ArrayList<>(resumes); + } + + public synchronized List failedActivations() { + return new ArrayList<>(failedActivations); + } + + public synchronized ScopeEvent terminal() { + return terminal; + } + + public synchronized List extraTerminals() { + return new ArrayList<>(extraTerminals); + } + + public synchronized List scopeRecordSeqs() { + return new ArrayList<>(scopeRecordSeqs); + } + + public synchronized boolean isResumed() { + return !resumes.isEmpty(); + } + + public synchronized boolean isResolved() { + return terminal != null; + } + + // ---- derived ------------------------------------------------------------- + + public synchronized ContinuationStatus status() { + if (terminal != null) { + return terminal.type == ScopeEvent.Type.RESOLVE_CANCEL + ? ContinuationStatus.CANCELLED + : ContinuationStatus.FINISHED; + } + return ContinuationStatus.LEAKED; + } + + /** + * Derives the failure set for this continuation. {@code rootWrittenNanos} may be {@code null}. + */ + public synchronized EnumSet failures(Long rootWrittenNanos) { + EnumSet failures = EnumSet.noneOf(Failure.class); + if (terminal == null) { + failures.add(Failure.LEAKED); + } + if (!extraTerminals.isEmpty()) { + failures.add(Failure.DOUBLE_FINISH); + } + if (!failedActivations.isEmpty() || resumedAfterTerminal()) { + failures.add(Failure.ACTIVATE_AFTER_RESOLVE); + } + if (rootWrittenNanos != null + && (laterThan(terminal, rootWrittenNanos) || laterThan(resumes, rootWrittenNanos))) { + failures.add(Failure.LATE_FINISH); + } + return failures; + } + + private boolean resumedAfterTerminal() { + if (terminal == null) { + return false; + } + for (ScopeEvent r : resumes) { + if (r.nanos > terminal.nanos) { + return true; + } + } + return false; + } + + /** {@code true} when capture and any resume/terminal happened on different threads. */ + public synchronized boolean threadHandoff() { + if (capture == null) { + return false; + } + String captureThread = capture.threadName; + for (ScopeEvent r : resumes) { + if (!captureThread.equals(r.threadName)) { + return true; + } + } + return terminal != null && !captureThread.equals(terminal.threadName); + } + + /** Nanos between capture and the first resume, or {@code null} if not both observed. */ + public synchronized Long captureToFirstResumeNanos() { + if (capture == null || resumes.isEmpty()) { + return null; + } + return resumes.get(0).nanos - capture.nanos; + } + + /** Nanos between capture and the terminal resolution, or {@code null} if not both observed. */ + public synchronized Long ageAtTerminalNanos() { + if (capture == null || terminal == null) { + return null; + } + return terminal.nanos - capture.nanos; + } + + /** Earliest known event time for ordering the timeline. */ + public synchronized long firstNanos() { + long min = capture != null ? capture.nanos : Long.MAX_VALUE; + for (ScopeEvent e : resumes) { + min = Math.min(min, e.nanos); + } + for (ScopeEvent e : failedActivations) { + min = Math.min(min, e.nanos); + } + if (terminal != null) { + min = Math.min(min, terminal.nanos); + } + for (ScopeEvent e : extraTerminals) { + min = Math.min(min, e.nanos); + } + return min; + } + + public String sourceName() { + return ScopeSources.name(source); + } + + private static boolean laterThan(ScopeEvent event, long nanos) { + return event != null && event.nanos > nanos; + } + + private static boolean laterThan(List events, long nanos) { + for (ScopeEvent e : events) { + if (e.nanos > nanos) { + return true; + } + } + return false; + } +} diff --git a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ContinuationStatus.java b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ContinuationStatus.java new file mode 100644 index 00000000000..213a57a62a8 --- /dev/null +++ b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ContinuationStatus.java @@ -0,0 +1,18 @@ +package datadog.trace.agent.test.scopediag; + +/** + * Derived lifecycle state of a continuation, for rendering. The authoritative bug signal is the + * {@link Failure} set, not this status. + */ +public enum ContinuationStatus { + /** Captured but not yet resumed or resolved. */ + CAPTURED, + /** Resumed at least once but not yet resolved. */ + RESUMED, + /** Resolved normally (all activations closed or a clean cancel with no outstanding work). */ + FINISHED, + /** Resolved via the cancel-with-outstanding-work path. */ + CANCELLED, + /** Captured (and possibly resumed) but never resolved within the recording window. */ + LEAKED +} diff --git a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/Failure.java b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/Failure.java new file mode 100644 index 00000000000..6632609584d --- /dev/null +++ b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/Failure.java @@ -0,0 +1,21 @@ +package datadog.trace.agent.test.scopediag; + +/** + * A derived failure classification. Shared by {@link ContinuationRecord} (continuation-lifetime + * failures) and {@link ScopeRecord} (scope-lifetime failures). See {@link + * ScopeDiagnosticsReport#hasProblems()} for which of these fail a test versus are report-only. + */ +public enum Failure { + /** Continuation captured but never resolved within the window. */ + LEAKED, + /** Continuation resolved/resumed after the root span of its trace was already written. */ + LATE_FINISH, + /** Continuation resolved more than once. */ + DOUBLE_FINISH, + /** Continuation activated after it had already been resolved. */ + ACTIVATE_AFTER_RESOLVE, + /** Scope closed while not on top of its thread's stack (closed on the wrong thread / order). */ + CLOSE_WRONG_THREAD, + /** Scope opened but never closed within the window. */ + NEVER_CLOSED +} diff --git a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/PendingTraceAdvice.java b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/PendingTraceAdvice.java new file mode 100644 index 00000000000..8d583cf0618 --- /dev/null +++ b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/PendingTraceAdvice.java @@ -0,0 +1,33 @@ +package datadog.trace.agent.test.scopediag; + +import net.bytebuddy.asm.Advice; + +/** + * Test-only ByteBuddy advice woven into {@code datadog.trace.core.PendingTrace}. Fires the + * root-written signal after {@code write(boolean)} actually changes {@code rootSpanWritten} from + * false to true. Observing the completed transition avoids treating an empty write as a root write. + * The timestamp is conservative: a resolution racing inside {@code write} may be omitted from the + * late category, but it cannot be falsely classified as late. + */ +public final class PendingTraceAdvice { + private PendingTraceAdvice() {} + + public static final class Write { + @Advice.OnMethodEnter(suppress = Throwable.class) + public static boolean enter( + @Advice.Argument(0) boolean isPartial, + @Advice.FieldValue("rootSpanWritten") boolean alreadyWritten) { + return !isPartial && !alreadyWritten; + } + + @Advice.OnMethodExit(suppress = Throwable.class) + public static void exit( + @Advice.Enter boolean candidate, + @Advice.FieldValue("rootSpanWritten") boolean written, + @Advice.FieldValue("traceId") Object traceId) { + if (candidate && written) { + ScopeContinuationProbe.onRootWritten(traceId); + } + } + } +} diff --git a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeContinuationProbe.java b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeContinuationProbe.java new file mode 100644 index 00000000000..f4240759077 --- /dev/null +++ b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeContinuationProbe.java @@ -0,0 +1,319 @@ +package datadog.trace.agent.test.scopediag; + +import datadog.context.ContextContinuation; +import datadog.trace.api.DDTraceId; +import datadog.trace.bootstrap.instrumentation.api.AgentScope; +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.bootstrap.instrumentation.api.NoopScope; +import java.lang.reflect.Field; +import java.lang.reflect.Method; + +/** + * Recorder hook that the test-only ByteBuddy advice ({@link ContinuationAdvice}, {@link + * PendingTraceAdvice}) funnels scope-continuation lifecycle events into. It replaces the former + * production {@code ContinuationDiagnostics} seam: the advice is woven into {@code + * datadog.trace.core.scopemanager.ScopeContinuation} and {@code datadog.trace.core.PendingTrace} at + * test time only, so production tracer code carries no diagnostic footprint at all. + * + *

Inlined advice runs in the same app classloader as this class at test time, so it can call + * these statics directly. Every entry point first checks the {@link #recording} flag and is fully + * wrapped so a diagnostic failure can never propagate back into the tracer. + */ +public final class ScopeContinuationProbe { + /** + * Mirrors {@code ScopeContinuation.CANCELLED} (see {@code + * dd-trace-core/.../scopemanager/ScopeContinuation.java}). A continuation is resolved exactly + * when its {@code count} field transitions to this sentinel during a cancel call. Kept in sync by + * {@code ScopeContinuationProbeTest}. + */ + static final int CANCELLED = Integer.MIN_VALUE >> 1; + + private static volatile boolean recording = false; + + /** Cached reflective handle to the package-private {@code ScopeContinuation.source} field. */ + private static volatile Field sourceField; + + // cached reflective handles for scope-lifecycle reads (set-once, best-effort) + private static volatile Field scopeSourceField; // ContinuableScope.source + private static volatile Field continuationField; // ContinuingScope.continuation + private static volatile Field scopeManagerField; // ContinuableScope.scopeManager + private static volatile Method scopeStackMethod; // ContinuableScopeManager.scopeStack() + private static volatile Method checkTopMethod; // ScopeStack.checkTop(ContinuableScope) + + private ScopeContinuationProbe() {} + + /** Installs the transformer (once per JVM) and opens the cheap advice-side recording gate. */ + static synchronized void enable() { + ScopeContinuationTransformer.install(); + recording = true; + } + + /** Stops recording. The transformer stays installed (inert while not recording). */ + static void disable() { + recording = false; + } + + // ---- advice entry points (public so inlined advice can reference them) ------------------- + + /** {@code ScopeContinuation.register()} exit: the continuation was captured. */ + public static void onCapture(Object self) { + if (!recording) { + return; + } + try { + ContextContinuation continuation = (ContextContinuation) self; + AgentSpan span = AgentSpan.fromContext(continuation.context()); + if (span != null) { + ScopeDiagnostics.recordCapture( + continuation, span.getTraceId(), span.getSpanId(), spanName(span), sourceOf(self)); + } + } catch (Throwable ignored) { + // diagnostics must never disturb the tracer + } + } + + /** + * {@code ScopeContinuation.resume()} exit: a real activation happened. The rollback branch + * returns the {@link NoopScope#INSTANCE noop scope} singleton, so a returned noop scope is + * skipped — this exactly reproduces the original "success branch only" semantics. + */ + public static void onActivate(Object self, Object returnedScope, long activateNanos) { + if (!recording) { + return; + } + try { + ContextContinuation continuation = (ContextContinuation) self; + if (returnedScope == NoopScope.INSTANCE) { + // activate() returned the noop scope: the continuation was already resolved. This is the + // activate-after-resolve signal — the engine records it only if a terminal was seen. + ScopeDiagnostics.recordActivateFailed(continuation); + return; + } + AgentSpan span = AgentSpan.fromContext(continuation.context()); + if (span != null) { + ScopeDiagnostics.recordActivate( + continuation, + span.getTraceId(), + span.getSpanId(), + spanName(span), + sourceOf(self), + activateNanos); + } + } catch (Throwable ignored) { + } + } + + /** + * {@code ScopeContinuation.release()} / {@code cancelFromContinuedScopeClose()} exit. The + * original production seam fired only from inside the clean resolution branch; here we detect + * that branch by observing the {@code count} field transition to {@link #CANCELLED} during this + * call. A cancel with outstanding activations leaves {@code count} unchanged (not a resolution). + */ + public static void onResolve( + Object self, String method, int countBefore, int countAfter, long resolveNanos) { + if (!recording) { + return; + } + if (countAfter != CANCELLED) { + return; // not a resolution + } + // An explicit cancel() is a discard; cancelFromContinuedScopeClose() is a normal finish once + // the continued scope closes. (Caveat: the rare cancelFromContinuedScopeClose slow path + // delegates to cancel(), so a multi-activation finish is recorded as a cancel.) + boolean cancelled = "release".equals(method); + try { + ContextContinuation continuation = (ContextContinuation) self; + // countBefore == CANCELLED is a genuine second finish/cancel. First transitions are deduped + // by the engine under the same lock as stop/reset/report, including nested advice frames. + ScopeDiagnostics.recordResolve( + continuation, cancelled, resolveNanos, countBefore == CANCELLED); + } catch (Throwable ignored) { + } + } + + /** {@code PendingTrace.write()} root-written site. */ + public static void onRootWritten(Object traceId) { + if (!recording) { + return; + } + try { + ScopeDiagnostics.recordRootWritten((DDTraceId) traceId); + } catch (Throwable ignored) { + } + } + + /** + * {@code ContinuableScope.afterActivated()} exit: a scope became active. Re-activations (parent + * restored after a child closes) reach here too; the engine keeps only the first per scope + * identity. Links to the spawning continuation when the scope is a {@code ContinuingScope}. + */ + public static void onScopeOpen(Object scope) { + if (!recording) { + return; + } + try { + AgentSpan span = ((AgentScope) scope).span(); + DDTraceId traceId = span != null ? span.getTraceId() : DDTraceId.ZERO; + long spanId = span != null ? span.getSpanId() : 0L; + String name = span != null ? spanName(span) : null; + ScopeDiagnostics.recordScopeOpen( + scope, traceId, spanId, name, scopeSourceOf(scope), continuationOf(scope)); + } catch (Throwable ignored) { + } + } + + /** + * {@code ContinuableScope.onProperClose()} exit: the scope was popped from its thread's stack. + */ + public static void onScopeClose(Object scope) { + if (!recording) { + return; + } + try { + ScopeDiagnostics.recordScopeClose(scope); + } catch (Throwable ignored) { + } + } + + /** + * {@code ContinuableScope.close()} entry: if the scope is not on top of its thread's stack, this + * is an out-of-order / wrong-thread close. Best-effort — silently does nothing if the internal + * stack check cannot be reached reflectively. + */ + public static void onScopeClosing(Object scope) { + if (!recording) { + return; + } + try { + if (isNotOnTop(scope)) { + ScopeDiagnostics.recordScopeCloseWrongThread(scope); + } + } catch (Throwable ignored) { + } + } + + /** Snapshots the span name as a String (the CharSequence may mutate later), or {@code null}. */ + private static String spanName(AgentSpan span) { + try { + CharSequence name = span.getSpanName(); + return name == null ? null : name.toString(); + } catch (Throwable ignored) { + return null; + } + } + + /** + * Reads the package-private {@code source} byte field, falling back to the {@code -1} sentinel. + */ + private static byte sourceOf(Object self) { + try { + Field field = sourceField; + if (field == null) { + field = self.getClass().getDeclaredField("source"); + field.setAccessible(true); + sourceField = field; + } + return field.getByte(self); + } catch (Throwable ignored) { + return (byte) -1; + } + } + + /** Reads the {@code source} byte of a scope (declared on {@code ContinuableScope}). */ + private static byte scopeSourceOf(Object scope) { + try { + Field field = scopeSourceField; + if (field == null) { + field = findField(scope.getClass(), "source"); + scopeSourceField = field; + } + return field != null ? field.getByte(scope) : (byte) -1; + } catch (Throwable ignored) { + return (byte) -1; + } + } + + /** + * The continuation that spawned a scope, read from {@code ContinuingScope.continuation}; {@code + * null} for a plain (non-continuation) scope. + */ + private static ContextContinuation continuationOf(Object scope) { + try { + Field field = continuationField; + if (field == null) { + field = findField(scope.getClass(), "continuation"); + continuationField = field; + } + if (field == null || !field.getDeclaringClass().isInstance(scope)) { + return null; // not a ContinuingScope + } + Object value = field.get(scope); + return value instanceof ContextContinuation ? (ContextContinuation) value : null; + } catch (Throwable ignored) { + return null; + } + } + + /** Best-effort: {@code true} when the scope is not on top of its thread's scope stack. */ + private static boolean isNotOnTop(Object scope) { + try { + Field managerField = scopeManagerField; + if (managerField == null) { + managerField = findField(scope.getClass(), "scopeManager"); + scopeManagerField = managerField; + } + Object manager = managerField != null ? managerField.get(scope) : null; + if (manager == null) { + return false; + } + Method stackMethod = scopeStackMethod; + if (stackMethod == null) { + stackMethod = findMethod(manager.getClass(), "scopeStack", 0); + scopeStackMethod = stackMethod; + } + Object stack = stackMethod != null ? stackMethod.invoke(manager) : null; + if (stack == null) { + return false; + } + Method check = checkTopMethod; + if (check == null) { + check = findMethod(stack.getClass(), "checkTop", 1); + checkTopMethod = check; + } + if (check == null) { + return false; + } + Object onTop = check.invoke(stack, scope); + return onTop instanceof Boolean && !((Boolean) onTop); + } catch (Throwable ignored) { + return false; + } + } + + /** Finds a named field declared on a class or any superclass, made accessible. */ + private static Field findField(Class cls, String name) { + for (Class c = cls; c != null; c = c.getSuperclass()) { + try { + Field f = c.getDeclaredField(name); + f.setAccessible(true); + return f; + } catch (NoSuchFieldException ignored) { + // keep walking up + } + } + return null; + } + + /** Finds a named method with the given parameter count on a class or superclass, accessible. */ + private static Method findMethod(Class cls, String name, int paramCount) { + for (Class c = cls; c != null; c = c.getSuperclass()) { + for (Method m : c.getDeclaredMethods()) { + if (m.getName().equals(name) && m.getParameterCount() == paramCount) { + m.setAccessible(true); + return m; + } + } + } + return null; + } +} diff --git a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeContinuationTransformer.java b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeContinuationTransformer.java new file mode 100644 index 00000000000..c364729de22 --- /dev/null +++ b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeContinuationTransformer.java @@ -0,0 +1,65 @@ +package datadog.trace.agent.test.scopediag; + +import static net.bytebuddy.matcher.ElementMatchers.named; +import static net.bytebuddy.matcher.ElementMatchers.takesArguments; + +import java.lang.instrument.Instrumentation; +import net.bytebuddy.agent.ByteBuddyAgent; +import net.bytebuddy.agent.builder.AgentBuilder; +import net.bytebuddy.agent.builder.ResettableClassFileTransformer; +import net.bytebuddy.asm.Advice; + +/** + * Installs the test-only diagnostic advice into the tracer's own {@code + * datadog.trace.core.scopemanager.ScopeContinuation} and {@code datadog.trace.core.PendingTrace}. + * + *

These classes sit under {@code datadog.trace.core.*}, which the tracer's own {@code + * AgentBuilder} hard-ignores. We therefore install a separate {@link AgentBuilder} on the + * raw {@link Instrumentation} with no global-ignore filter, using retransformation so the + * already-loaded classes are rewoven on install. The advice is schema-preserving, so {@code + * disableClassFormatChanges()} + {@code REDEFINE} keep it retransform-safe. Installed once per JVM. + */ +final class ScopeContinuationTransformer { + private static volatile ResettableClassFileTransformer transformer; + + private ScopeContinuationTransformer() {} + + static synchronized void install() { + if (transformer != null) { + return; + } + Instrumentation instrumentation = ByteBuddyAgent.getInstrumentation(); + transformer = + new AgentBuilder.Default() + .disableClassFormatChanges() + .with(AgentBuilder.RedefinitionStrategy.RETRANSFORMATION) + .with(AgentBuilder.TypeStrategy.Default.REDEFINE) + .type(named("datadog.trace.core.scopemanager.ScopeContinuation")) + .transform( + (builder, type, classLoader, module, pd) -> + builder + .visit(Advice.to(ContinuationAdvice.Register.class).on(named("register"))) + .visit(Advice.to(ContinuationAdvice.Activate.class).on(named("resume"))) + .visit( + Advice.to(ContinuationAdvice.Cancel.class) + .on(named("release").or(named("cancelFromContinuedScopeClose"))))) + .type(named("datadog.trace.core.PendingTrace")) + .transform( + (builder, type, classLoader, module, pd) -> + builder.visit( + Advice.to(PendingTraceAdvice.Write.class) + .on(named("write").and(takesArguments(boolean.class))))) + .type(named("datadog.trace.core.scopemanager.ContinuableScope")) + .transform( + (builder, type, classLoader, module, pd) -> + builder + .visit( + Advice.to(ContinuableScopeAdvice.AfterActivated.class) + .on(named("afterActivated"))) + .visit( + Advice.to(ContinuableScopeAdvice.OnProperClose.class) + .on(named("onProperClose"))) + .visit(Advice.to(ContinuableScopeAdvice.Close.class).on(named("close")))) + .installOn(instrumentation); + } +} diff --git a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeDiagnostics.java b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeDiagnostics.java new file mode 100644 index 00000000000..d06777fa6ac --- /dev/null +++ b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeDiagnostics.java @@ -0,0 +1,359 @@ +package datadog.trace.agent.test.scopediag; + +import datadog.context.ContextContinuation; +import datadog.trace.api.DDTraceId; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.IdentityHashMap; +import java.util.Map; +import java.util.Set; + +/** + * Test-time engine that records scope/continuation lifecycle events and renders leak reports. + * + *

It models two correlated lifecycles separately: continuations ({@link ContinuationRecord} — + * captured/resumed/finished) and scopes ({@link ScopeRecord} — opened/closed). While recording, + * {@link ScopeContinuationProbe} (test-only bytecode advice) feeds it events, which it correlates + * by identity (an {@link IdentityHashMap}, never {@code equals}/{@code hashCode}) — continuations + * by their {@code ContextContinuation} instance, scopes by their scope instance. It assumes a + * single test runs at a time per JVM (true for instrumentation tests); {@link #reset()} isolates + * one test from the next. + * + *

Usage: + * + * 

+ *   ScopeDiagnostics.startRecording();
+ *   ... exercise code under test ...
+ *   System.out.println(ScopeDiagnostics.report().renderSummary());
+ *   ScopeDiagnostics.assertNoLeaks();   // optional
+ *   ScopeDiagnostics.stop();
+ *
+ */ +public final class ScopeDiagnostics { + private static final int DEFAULT_MAX_FRAMES = 6; + + private static final ScopeDiagnostics INSTANCE = new ScopeDiagnostics(); + + /** Linearizes event admission, stop/reset, and report snapshots. */ + private final Object lifecycleLock = new Object(); + + private final Map records = new IdentityHashMap<>(); + private final Map scopeRecords = new IdentityHashMap<>(); + private final Map rootWrittenNanos = new HashMap<>(); + private final Set resolved = + Collections.newSetFromMap(new IdentityHashMap()); + private long seq; + private long scopeSeq; + private boolean recording; + private StackFilter stackFilter = new StackFilter(DEFAULT_MAX_FRAMES); + + private final Listener listener = new Listener(); + + private ScopeDiagnostics() {} + + // ---- public static facade ------------------------------------------------ + + /** Clears any prior data and starts recording with the default stack depth. */ + public static void startRecording() { + startRecording(DEFAULT_MAX_FRAMES); + } + + /** Clears any prior data and starts recording, keeping up to {@code maxFrames} per stack. */ + public static void startRecording(int maxFrames) { + ScopeContinuationProbe.disable(); + synchronized (INSTANCE.lifecycleLock) { + INSTANCE.recording = false; + INSTANCE.clear(); + INSTANCE.stackFilter = new StackFilter(maxFrames); + } + ScopeContinuationProbe.enable(); + synchronized (INSTANCE.lifecycleLock) { + INSTANCE.recording = true; + } + } + + /** Stops recording (the probe goes inert). Recorded data remains queryable until reset. */ + public static void stop() { + ScopeContinuationProbe.disable(); + synchronized (INSTANCE.lifecycleLock) { + INSTANCE.recording = false; + } + } + + /** Discards all recorded data. */ + public static void reset() { + ScopeContinuationProbe.disable(); + synchronized (INSTANCE.lifecycleLock) { + INSTANCE.recording = false; + INSTANCE.clear(); + } + } + + /** Builds an immutable snapshot report of everything recorded so far. */ + public static ScopeDiagnosticsReport report() { + synchronized (INSTANCE.lifecycleLock) { + return new ScopeDiagnosticsReport( + new ArrayList<>(INSTANCE.records.values()), + new ArrayList<>(INSTANCE.scopeRecords.values()), + new HashMap<>(INSTANCE.rootWrittenNanos)); + } + } + + /** + * Fails with an {@link AssertionError} (carrying the problem summary) if the report flags a + * genuine bug (see {@link ScopeDiagnosticsReport#hasProblems()}). Report-only signals such as + * late-after-root and close-on-wrong-thread do not fail. + */ + public static void assertNoLeaks() { + assertNoLeaks(report()); + } + + /** Fails using the supplied snapshot, so rendering and assertion examine the same events. */ + public static void assertNoLeaks(ScopeDiagnosticsReport report) { + if (report.hasProblems()) { + throw new AssertionError("Scope continuation problems detected:\n" + report.renderSummary()); + } + } + + /** Resolves the default-on policy and rejects undocumented opt-outs. */ + public static boolean isEnabled(TrackScopeContinuations config) { + if (config == null || config.enabled()) { + return true; + } + if (config.reason().trim().isEmpty()) { + throw new IllegalArgumentException( + "@TrackScopeContinuations(enabled = false) requires a reason"); + } + return false; + } + + private void clear() { + records.clear(); + scopeRecords.clear(); + rootWrittenNanos.clear(); + resolved.clear(); + seq = 0; + scopeSeq = 0; + } + + // ---- listener implementation --------------------------------------------- + + private static final StackTraceElement[] NO_STACK = new StackTraceElement[0]; + + private ScopeEvent event(ScopeEvent.Type type) { + return event(type, System.nanoTime()); + } + + /** Builds an event with an explicit timestamp (thread and stack are still captured now). */ + private ScopeEvent event(ScopeEvent.Type type, long nanos) { + // Capturing a stack per event is the dominant cost and perturbs the very timings we record; + // skip it entirely when callsites are disabled (maxFrames <= 0) rather than walking then + // trimming. + StackFilter filter = stackFilter; + StackTraceElement[] stack = + filter.maxFrames() <= 0 ? NO_STACK : filter.filter(new Throwable().getStackTrace()); + return new ScopeEvent(type, Thread.currentThread().getName(), nanos, stack); + } + + // ---- static forwarders called by ScopeContinuationProbe ------------------ + + static void recordCapture( + ContextContinuation id, DDTraceId traceId, long spanId, String spanName, byte source) { + synchronized (INSTANCE.lifecycleLock) { + if (INSTANCE.recording) { + INSTANCE.listener.onCapture(id, traceId, spanId, spanName, source); + } + } + } + + static void recordActivate( + ContextContinuation id, + DDTraceId traceId, + long spanId, + String spanName, + byte source, + long nanos) { + synchronized (INSTANCE.lifecycleLock) { + if (INSTANCE.recording) { + INSTANCE.listener.onActivate(id, traceId, spanId, spanName, source, nanos); + } + } + } + + static void recordActivateFailed(ContextContinuation id) { + synchronized (INSTANCE.lifecycleLock) { + if (INSTANCE.recording) { + INSTANCE.listener.onActivateFailed(id); + } + } + } + + static void recordResolve( + ContextContinuation id, boolean cancelled, long resolveNanos, boolean alreadyResolved) { + synchronized (INSTANCE.lifecycleLock) { + if (INSTANCE.recording && (alreadyResolved || INSTANCE.resolved.add(id))) { + INSTANCE.listener.onResolve(id, cancelled, resolveNanos); + } + } + } + + static void recordRootWritten(DDTraceId traceId) { + synchronized (INSTANCE.lifecycleLock) { + if (INSTANCE.recording) { + INSTANCE.listener.onRootWritten(traceId); + } + } + } + + static void recordScopeOpen( + Object scope, + DDTraceId traceId, + long spanId, + String spanName, + byte source, + ContextContinuation continuation) { + synchronized (INSTANCE.lifecycleLock) { + if (INSTANCE.recording) { + INSTANCE.listener.onScopeOpen(scope, traceId, spanId, spanName, source, continuation); + } + } + } + + static void recordScopeClose(Object scope) { + synchronized (INSTANCE.lifecycleLock) { + if (INSTANCE.recording) { + INSTANCE.listener.onScopeClose(scope); + } + } + } + + static void recordScopeCloseWrongThread(Object scope) { + synchronized (INSTANCE.lifecycleLock) { + if (INSTANCE.recording) { + INSTANCE.listener.onScopeCloseWrongThread(scope); + } + } + } + + private final class Listener { + void onCapture( + ContextContinuation id, DDTraceId traceId, long spanId, String spanName, byte source) { + try { + ContinuationRecord record = + new ContinuationRecord( + seq++, traceId, spanId, spanName, source, false, event(ScopeEvent.Type.CAPTURE)); + records.put(id, record); + } catch (Throwable ignored) { + // diagnostics must never disturb the tracer + } + } + + void onActivate( + ContextContinuation id, + DDTraceId traceId, + long spanId, + String spanName, + byte source, + long nanos) { + try { + recordFor(id, traceId, spanId, spanName, source) + .addResume(event(ScopeEvent.Type.ACTIVATE, nanos)); + } catch (Throwable ignored) { + } + } + + void onActivateFailed(ContextContinuation id) { + try { + ContinuationRecord record = records.get(id); + // only an activation of an already-resolved continuation is a real failure; a plain + // rollback (e.g. cancelled before any capture was recorded) is benign and ignored + if (record != null && record.isResolved()) { + record.addFailedActivation(event(ScopeEvent.Type.ACTIVATE_FAILED)); + } + } catch (Throwable ignored) { + } + } + + void onResolve(ContextContinuation id, boolean cancelled, long resolveNanos) { + try { + ScopeEvent.Type type = + cancelled ? ScopeEvent.Type.RESOLVE_CANCEL : ScopeEvent.Type.RESOLVE_FINISH; + recordFor(id, DDTraceId.ZERO, 0, null, (byte) -1) + .setTerminalOrExtra(event(type, resolveNanos)); + } catch (Throwable ignored) { + } + } + + void onRootWritten(DDTraceId traceId) { + try { + rootWrittenNanos.putIfAbsent(traceId, System.nanoTime()); + } catch (Throwable ignored) { + } + } + + void onScopeOpen( + Object scope, + DDTraceId traceId, + long spanId, + String spanName, + byte source, + ContextContinuation continuation) { + try { + if (scopeRecords.containsKey(scope)) { + return; // re-activation of an already-open scope, not a new open + } + ContinuationRecord owner = continuation != null ? records.get(continuation) : null; + Long continuationSeq = owner != null ? owner.seq : null; + long s = scopeSeq++; + scopeRecords.put( + scope, + new ScopeRecord( + s, + traceId, + spanId, + spanName, + source, + continuationSeq, + event(ScopeEvent.Type.SCOPE_OPEN))); + if (owner != null) { + owner.linkScope(s); + } + } catch (Throwable ignored) { + } + } + + void onScopeClose(Object scope) { + try { + ScopeRecord record = scopeRecords.get(scope); + if (record != null) { + record.setClose(event(ScopeEvent.Type.SCOPE_CLOSE)); + } + } catch (Throwable ignored) { + } + } + + void onScopeCloseWrongThread(Object scope) { + try { + ScopeRecord record = scopeRecords.get(scope); + if (record != null) { + record.addWrongThreadClose(event(ScopeEvent.Type.SCOPE_CLOSE_WRONG_THREAD)); + } + } catch (Throwable ignored) { + } + } + + /** Returns the record for an id, creating an orphan record if capture was not observed. */ + private ContinuationRecord recordFor( + ContextContinuation id, DDTraceId traceId, long spanId, String spanName, byte source) { + ContinuationRecord existing = records.get(id); + if (existing != null) { + return existing; + } + ContinuationRecord orphan = + new ContinuationRecord(seq++, traceId, spanId, spanName, source, true, null); + records.put(id, orphan); + return orphan; + } + } +} diff --git a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsExtension.java b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsExtension.java new file mode 100644 index 00000000000..54eafd6dccd --- /dev/null +++ b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsExtension.java @@ -0,0 +1,62 @@ +package datadog.trace.agent.test.scopediag; + +import java.lang.reflect.AnnotatedElement; +import java.util.Optional; +import org.junit.jupiter.api.extension.AfterEachCallback; +import org.junit.jupiter.api.extension.BeforeEachCallback; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.platform.commons.support.AnnotationSupport; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * JUnit5 extension that enables {@link ScopeDiagnostics} for every test extending {@code + * AbstractInstrumentationTest}. {@link TrackScopeContinuations} can override or disable it. + * + *

Per test: resets and starts recording before; after, logs findings and asserts no leaks. + */ +public final class ScopeDiagnosticsExtension implements BeforeEachCallback, AfterEachCallback { + private static final Logger log = LoggerFactory.getLogger(ScopeDiagnosticsExtension.class); + + @Override + public void beforeEach(ExtensionContext context) { + TrackScopeContinuations config = resolve(context); + if (ScopeDiagnostics.isEnabled(config)) { + ScopeDiagnostics.startRecording(); + } + } + + @Override + public void afterEach(ExtensionContext context) { + TrackScopeContinuations config = resolve(context); + if (!ScopeDiagnostics.isEnabled(config)) { + return; + } + try { + ScopeDiagnostics.stop(); + ScopeDiagnosticsReport report = ScopeDiagnostics.report(); + if (report.hasFindings()) { + log.info("[{}] {}", context.getDisplayName(), report.renderTimeline()); + } + ScopeDiagnostics.assertNoLeaks(report); + } finally { + ScopeDiagnostics.reset(); + } + } + + /** Method-level annotation wins; otherwise the test class (incl. inherited). */ + private static TrackScopeContinuations resolve(ExtensionContext context) { + Optional element = context.getElement(); + if (element.isPresent()) { + Optional onElement = + AnnotationSupport.findAnnotation(element.get(), TrackScopeContinuations.class); + if (onElement.isPresent()) { + return onElement.get(); + } + } + return context + .getTestClass() + .flatMap(c -> AnnotationSupport.findAnnotation(c, TrackScopeContinuations.class)) + .orElse(null); + } +} diff --git a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsReport.java b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsReport.java new file mode 100644 index 00000000000..b494a68401c --- /dev/null +++ b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsReport.java @@ -0,0 +1,381 @@ +package datadog.trace.agent.test.scopediag; + +import datadog.trace.api.DDTraceId; +import java.util.ArrayList; +import java.util.EnumSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * An immutable snapshot of the recorded continuation and scope lifecycles plus the derived failure + * findings. Exposes two text renderings: a problem-only summary ({@link #renderSummary()}) for + * quick triage / assertion messages, and a complete timeline ({@link #renderTimeline()}) that dumps + * every continuation and scope with its full event lineage. The harness emits the timeline + * when it contains a finding, giving humans and diagnostic skills enough data to build a graph. + * + *

The two lifecycles are kept separate: {@link ContinuationRecord} (captured → resumed → + * finished) and {@link ScopeRecord} (opened → closed). A scope spawned by resuming a continuation + * is linked to it ({@link ContinuationRecord#scopeRecordSeqs()} / {@link + * ScopeRecord#continuationSeq}) and rendered nested under it in the timeline. + */ +public final class ScopeDiagnosticsReport { + private final List continuations; + private final List scopes; + private final long t0; + private final Map> continuationFailures; + private final Map> scopeFailures; + private final Map scopeBySeq; + + ScopeDiagnosticsReport( + List continuations, + List scopes, + Map rootWrittenNanos) { + this.continuations = new ArrayList<>(continuations.size()); + for (ContinuationRecord continuation : continuations) { + this.continuations.add(continuation.snapshot()); + } + this.continuations.sort((a, b) -> Long.compare(a.firstNanos(), b.firstNanos())); + this.scopes = new ArrayList<>(scopes.size()); + for (ScopeRecord scope : scopes) { + this.scopes.add(scope.snapshot()); + } + this.scopes.sort((a, b) -> Long.compare(a.firstNanos(), b.firstNanos())); + this.t0 = computeT0(this.continuations, this.scopes); + this.continuationFailures = classifyContinuations(this.continuations, rootWrittenNanos); + this.scopeFailures = classifyScopes(this.scopes); + this.scopeBySeq = new LinkedHashMap<>(); + for (ScopeRecord s : this.scopes) { + scopeBySeq.put(s.seq, s); + } + } + + private static long computeT0(List continuations, List scopes) { + long min = Long.MAX_VALUE; + for (ContinuationRecord r : continuations) { + min = Math.min(min, r.firstNanos()); + } + for (ScopeRecord s : scopes) { + min = Math.min(min, s.firstNanos()); + } + return min == Long.MAX_VALUE ? 0 : min; + } + + private static Map> classifyContinuations( + List records, Map rootWrittenNanos) { + Map> result = new LinkedHashMap<>(); + for (ContinuationRecord r : records) { + EnumSet failures = r.failures(rootWrittenNanos.get(r.traceId)); + if (!failures.isEmpty()) { + result.put(r, failures); + } + } + return result; + } + + private static Map> classifyScopes(List scopes) { + Map> result = new LinkedHashMap<>(); + for (ScopeRecord s : scopes) { + EnumSet failures = s.failures(); + if (!failures.isEmpty()) { + result.put(s, failures); + } + } + return result; + } + + // ---- accessors ----------------------------------------------------------- + + public List records() { + return new ArrayList<>(continuations); + } + + public List scopeRecords() { + return new ArrayList<>(scopes); + } + + public Map> findings() { + return new LinkedHashMap<>(continuationFailures); + } + + public Map> scopeFindings() { + return new LinkedHashMap<>(scopeFailures); + } + + public int leakCount() { + return countWith(continuationFailures, Failure.LEAKED); + } + + public int lateCount() { + return countWith(continuationFailures, Failure.LATE_FINISH); + } + + public int doubleCount() { + return countWith(continuationFailures, Failure.DOUBLE_FINISH); + } + + public int activateAfterResolveCount() { + return countWith(continuationFailures, Failure.ACTIVATE_AFTER_RESOLVE); + } + + public int neverClosedScopeCount() { + return countWith(scopeFailures, Failure.NEVER_CLOSED); + } + + public int closeWrongThreadCount() { + return countWith(scopeFailures, Failure.CLOSE_WRONG_THREAD); + } + + private static int countWith(Map> findings, Failure failure) { + int n = 0; + for (EnumSet f : findings.values()) { + if (f.contains(failure)) { + n++; + } + } + return n; + } + + /** + * True when there is a genuine bug to fail on: a never-resolved leak, a double finish, an + * activation after resolve, or a scope that was never closed. {@link Failure#LATE_FINISH} and + * {@link Failure#CLOSE_WRONG_THREAD} are reported but do not fail (frequently legitimate async or + * teardown ordering). + */ + public boolean hasProblems() { + return leakCount() > 0 + || doubleCount() > 0 + || activateAfterResolveCount() > 0 + || neverClosedScopeCount() > 0; + } + + /** True when the report contains either a failing problem or an advisory signal. */ + public boolean hasFindings() { + return !continuationFailures.isEmpty() || !scopeFailures.isEmpty(); + } + + // ---- rendering: text summary --------------------------------------------- + + private void appendHeader(StringBuilder sb, String title) { + sb.append(title) + .append(" (") + .append(continuations.size()) + .append(" continuations, ") + .append(scopes.size()) + .append(" scopes; ") + .append(leakCount()) + .append(" leaked, ") + .append(lateCount()) + .append(" late, ") + .append(doubleCount()) + .append(" double, ") + .append(activateAfterResolveCount()) + .append(" activate-after-resolve | scopes: ") + .append(neverClosedScopeCount()) + .append(" never-closed, ") + .append(closeWrongThreadCount()) + .append(" wrong-thread)\n"); + } + + /** Problem-only summary: just the flagged continuations and scopes with their callsites. */ + public String renderSummary() { + StringBuilder sb = new StringBuilder(); + appendHeader(sb, "Scope/continuation problems"); + if (continuationFailures.isEmpty() && scopeFailures.isEmpty()) { + sb.append(" (none)\n"); + return sb.toString(); + } + for (Map.Entry> e : continuationFailures.entrySet()) { + ContinuationRecord r = e.getKey(); + ScopeEvent capture = r.capture(); + sb.append(" ") + .append(e.getValue()) + .append(" #") + .append(r.seq) + .append(" trace=") + .append(r.traceId) + .append(" src=") + .append(r.sourceName()) + .append(" captured at ") + .append(capture == null || capture.callsite() == null ? "" : capture.callsite()) + .append('\n'); + } + for (Map.Entry> e : scopeFailures.entrySet()) { + ScopeRecord s = e.getKey(); + ScopeEvent open = s.open(); + sb.append(" ") + .append(e.getValue()) + .append(" scope#") + .append(s.seq) + .append(" trace=") + .append(s.traceId) + .append(" src=") + .append(s.sourceName()) + .append(" opened at ") + .append(open == null || open.callsite() == null ? "" : open.callsite()) + .append('\n'); + } + return sb.toString(); + } + + // ---- rendering: complete timeline ---------------------------------------- + + private static final int TIMELINE_FRAMES = 3; + + /** + * Complete cross-thread timeline: one block per continuation (capture → resume(s) → terminal), + * with the scopes it spawned nested under it, followed by any non-continuation scopes. Unlike + * {@link #renderSummary()} this lists all records, not just the flagged ones, so a graph + * (Gantt/DAG) or report can be reconstructed from it whether or not anything leaked. Each event + * carries its relative time ({@code +Δms} from the first recorded event), thread, and callsite. + */ + public String renderTimeline() { + StringBuilder sb = new StringBuilder(); + appendHeader(sb, "Scope/continuation timeline"); + if (continuations.isEmpty() && scopes.isEmpty()) { + sb.append(" (nothing captured)\n"); + return sb.toString(); + } + + for (ContinuationRecord r : continuations) { + EnumSet failures = + continuationFailures.getOrDefault(r, EnumSet.noneOf(Failure.class)); + sb.append("\n#") + .append(r.seq) + .append(' ') + .append(r.status()) + .append(" trace=") + .append(r.traceId) + .append(" span=") + .append(r.spanId); + if (r.spanName != null) { + sb.append(" \"").append(r.spanName).append('"'); + } + sb.append(" src=").append(r.sourceName()); + if (r.orphan) { + sb.append(" [ORPHAN]"); + } + if (r.threadHandoff()) { + sb.append(" [handoff]"); + } + if (!failures.isEmpty()) { + sb.append(' ').append(failures); + } + sb.append(timing(r)).append('\n'); + + appendEvent(sb, "capture ", r.capture()); + for (ScopeEvent a : r.resumes()) { + appendEvent(sb, "resume ", a); + } + for (ScopeEvent f : r.failedActivations()) { + appendEvent(sb, "act-fail", f); + } + ScopeEvent terminal = r.terminal(); + if (terminal != null) { + appendEvent( + sb, + terminal.type == ScopeEvent.Type.RESOLVE_CANCEL ? "cancel " : "finish ", + terminal); + } + for (ScopeEvent extra : r.extraTerminals()) { + appendEvent(sb, "DOUBLE ", extra); + } + for (long scopeSeq : r.scopeRecordSeqs()) { + ScopeRecord scope = scopeBySeq.get(scopeSeq); + if (scope != null) { + appendScopeLine(sb, " ", scope); + } + } + if (terminal == null) { + sb.append(" LEAKED (never finished or cancelled)\n"); + } + } + + List orphanScopes = new ArrayList<>(); + for (ScopeRecord s : scopes) { + if (s.continuationSeq == null) { + orphanScopes.add(s); + } + } + if (!orphanScopes.isEmpty()) { + sb.append("\nNon-continuation scopes:\n"); + for (ScopeRecord s : orphanScopes) { + appendScopeLine(sb, " ", s); + } + } + return sb.toString(); + } + + private String timing(ContinuationRecord r) { + StringBuilder sb = new StringBuilder(); + Long capToResume = r.captureToFirstResumeNanos(); + Long age = r.ageAtTerminalNanos(); + if (capToResume != null) { + sb.append(" cap->resume=").append(millis(capToResume)).append("ms"); + } + if (age != null) { + sb.append(" age=").append(millis(age)).append("ms"); + } + return sb.toString(); + } + + private void appendScopeLine(StringBuilder sb, String indent, ScopeRecord scope) { + EnumSet failures = scopeFailures.getOrDefault(scope, EnumSet.noneOf(Failure.class)); + sb.append(indent).append("scope#").append(scope.seq).append(' ').append(scope.sourceName()); + if (scope.spanName != null) { + sb.append(" \"").append(scope.spanName).append('"'); + } + ScopeEvent open = scope.open(); + ScopeEvent close = scope.close(); + if (open != null) { + sb.append(" open +").append(relMillis(open.nanos)).append("ms @ ").append(open.threadName); + } + if (close != null) { + sb.append(" close +") + .append(relMillis(close.nanos)) + .append("ms @ ") + .append(close.threadName); + Long active = scope.activeDurationNanos(); + if (active != null) { + sb.append(" (active ").append(millis(active)).append("ms)"); + } + } + if (scope.threadHandoff()) { + sb.append(" [handoff]"); + } + if (!failures.isEmpty()) { + sb.append(' ').append(failures); + } + sb.append('\n'); + } + + private void appendEvent(StringBuilder sb, String label, ScopeEvent event) { + if (event == null) { + sb.append(" ").append(label).append(" (not observed)\n"); + return; + } + sb.append(" ") + .append(label) + .append(" +") + .append(relMillis(event.nanos)) + .append("ms @ ") + .append(event.threadName) + .append(" at ") + .append(event.callsite() == null ? "" : event.callsite()) + .append('\n'); + StackTraceElement[] stack = event.stack; + if (stack != null) { + for (int i = 1; i < stack.length && i < TIMELINE_FRAMES; i++) { + sb.append(" from ").append(stack[i]).append('\n'); + } + } + } + + private String relMillis(long nanos) { + return String.format("%.3f", (nanos - t0) / 1_000_000.0); + } + + private static String millis(long nanos) { + return String.format("%.3f", nanos / 1_000_000.0); + } +} diff --git a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeEvent.java b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeEvent.java new file mode 100644 index 00000000000..4ad15cbbd9b --- /dev/null +++ b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeEvent.java @@ -0,0 +1,44 @@ +package datadog.trace.agent.test.scopediag; + +/** + * A single observed point in a continuation's lifecycle. Time, thread, and stack are captured by + * the recorder on the event's own thread (notifications are synchronous), so they reflect the + * thread that actually captured/activated/resolved the continuation. + */ +public final class ScopeEvent { + public enum Type { + CAPTURE, + ACTIVATE, + /** An {@code activate()} that returned the noop scope after the continuation was resolved. */ + ACTIVATE_FAILED, + RESOLVE_FINISH, + RESOLVE_CANCEL, + /** A scope became active (first activation). */ + SCOPE_OPEN, + /** A scope was popped from its thread's stack. */ + SCOPE_CLOSE, + /** A scope was closed while not on top of its thread's stack. */ + SCOPE_CLOSE_WRONG_THREAD + } + + public final Type type; + public final String threadName; + public final long nanos; + public final StackTraceElement[] stack; + + ScopeEvent(Type type, String threadName, long nanos, StackTraceElement[] stack) { + this.type = type; + this.threadName = threadName; + this.nanos = nanos; + this.stack = stack; + } + + ScopeEvent snapshot() { + return new ScopeEvent(type, threadName, nanos, stack == null ? null : stack.clone()); + } + + /** The most relevant (top, post-filter) frame, or {@code null} if none survived filtering. */ + public StackTraceElement callsite() { + return stack != null && stack.length > 0 ? stack[0] : null; + } +} diff --git a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeRecord.java b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeRecord.java new file mode 100644 index 00000000000..8bd5bca1153 --- /dev/null +++ b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeRecord.java @@ -0,0 +1,134 @@ +package datadog.trace.agent.test.scopediag; + +import datadog.trace.api.DDTraceId; +import java.util.ArrayList; +import java.util.EnumSet; +import java.util.List; + +/** + * The scope activation lifecycle: a scope opened (first activation) → closed (popped from + * its thread's stack). Distinct from the continuation lifecycle ({@link ContinuationRecord}); when + * a scope was spawned by resuming a continuation, {@link #continuationSeq} links back to that + * continuation's {@link ContinuationRecord#seq}. + */ +public final class ScopeRecord { + public final long seq; + public final DDTraceId traceId; + public final long spanId; + public final String spanName; + public final byte source; + + /** + * The seq of the continuation that spawned this scope, or {@code null} for a plain activation. + */ + public final Long continuationSeq; + + private final ScopeEvent open; + private ScopeEvent close; + private final List wrongThreadCloses = new ArrayList<>(0); + + ScopeRecord( + long seq, + DDTraceId traceId, + long spanId, + String spanName, + byte source, + Long continuationSeq, + ScopeEvent open) { + this.seq = seq; + this.traceId = traceId; + this.spanId = spanId; + this.spanName = spanName; + this.source = source; + this.continuationSeq = continuationSeq; + this.open = open; + } + + // ---- mutation ------------------------------------------------------------ + + synchronized void setClose(ScopeEvent event) { + if (close == null) { + close = event; + } + } + + synchronized void addWrongThreadClose(ScopeEvent event) { + wrongThreadCloses.add(event); + } + + synchronized ScopeRecord snapshot() { + ScopeRecord copy = + new ScopeRecord( + seq, + traceId, + spanId, + spanName, + source, + continuationSeq, + open == null ? null : open.snapshot()); + copy.close = close == null ? null : close.snapshot(); + for (ScopeEvent event : wrongThreadCloses) { + copy.wrongThreadCloses.add(event.snapshot()); + } + return copy; + } + + // ---- accessors ----------------------------------------------------------- + + public synchronized ScopeEvent open() { + return open; + } + + public synchronized ScopeEvent close() { + return close; + } + + public synchronized List wrongThreadCloses() { + return new ArrayList<>(wrongThreadCloses); + } + + public synchronized boolean closed() { + return close != null; + } + + // ---- derived ------------------------------------------------------------- + + /** {@code true} when the scope was opened and closed on different threads. */ + public synchronized boolean threadHandoff() { + return open != null && close != null && !open.threadName.equals(close.threadName); + } + + /** Nanos the scope was active, or {@code null} if it was never closed. */ + public synchronized Long activeDurationNanos() { + if (open == null || close == null) { + return null; + } + return close.nanos - open.nanos; + } + + public synchronized EnumSet failures() { + EnumSet failures = EnumSet.noneOf(Failure.class); + if (open != null && close == null) { + failures.add(Failure.NEVER_CLOSED); + } + if (!wrongThreadCloses.isEmpty()) { + failures.add(Failure.CLOSE_WRONG_THREAD); + } + return failures; + } + + public synchronized long firstNanos() { + long min = open != null ? open.nanos : Long.MAX_VALUE; + if (close != null) { + min = Math.min(min, close.nanos); + } + for (ScopeEvent e : wrongThreadCloses) { + min = Math.min(min, e.nanos); + } + return min; + } + + public String sourceName() { + return ScopeSources.name(source); + } +} diff --git a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeSources.java b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeSources.java new file mode 100644 index 00000000000..7bf7a81fa3b --- /dev/null +++ b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeSources.java @@ -0,0 +1,25 @@ +package datadog.trace.agent.test.scopediag; + +/** + * Maps the {@code byte} scope source used by the tracer to a readable name. Mirrors the constants + * in {@code datadog.trace.core.scopemanager.ContinuableScope} (which are package-private and not + * visible from here). + */ +final class ScopeSources { + private ScopeSources() {} + + static String name(byte source) { + switch (source) { + case 0: + return "INSTRUMENTATION"; + case 1: + return "MANUAL"; + case 2: + return "ITERATION"; + case 3: + return "CONTEXT"; + default: + return "UNKNOWN(" + source + ")"; + } + } +} diff --git a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/StackFilter.java b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/StackFilter.java new file mode 100644 index 00000000000..8b360d38fa0 --- /dev/null +++ b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/StackFilter.java @@ -0,0 +1,75 @@ +package datadog.trace.agent.test.scopediag; + +import java.util.ArrayList; +import java.util.List; + +/** + * Trims a raw stack trace down to the frames that point at where a continuation was + * captured/activated/resolved: it drops the diagnostic plumbing, the scope-manager internals, and + * the executor/reflection scaffolding, keeping the top {@code maxFrames} meaningful frames. + */ +final class StackFilter { + private static final String[] DROP_PREFIXES = { + // diagnostic harness itself + "datadog.trace.agent.test.scopediag.", + // tracer scope/continuation machinery and the capture/activate plumbing it sits behind + "datadog.trace.core.", + "datadog.trace.bootstrap.instrumentation.java.concurrent.", + "datadog.trace.bootstrap.instrumentation.api.", + "datadog.trace.bootstrap.InstrumentationContext", + // JDK executor/reflection scaffolding between the caller and the capture + "java.lang.Thread.getStackTrace", + "java.util.concurrent.ThreadPoolExecutor", + "java.util.concurrent.ScheduledThreadPoolExecutor", + "java.util.concurrent.ForkJoinPool", + "java.util.concurrent.ForkJoinWorkerThread", + "java.util.concurrent.Executors$", + "java.util.concurrent.FutureTask", + "java.util.concurrent.CompletableFuture", + "jdk.internal.reflect.", + "java.lang.reflect.", + "sun.reflect.", + // Spock/Groovy/ByteBuddy mock & dynamic-dispatch scaffolding (test harness, not a callsite) + "org.spockframework.mock.", + "org.codehaus.groovy.", + "groovy.lang.", + "net.bytebuddy.", + }; + + private final int maxFrames; + + StackFilter(int maxFrames) { + this.maxFrames = maxFrames; + } + + int maxFrames() { + return maxFrames; + } + + StackTraceElement[] filter(StackTraceElement[] raw) { + if (raw == null) { + return new StackTraceElement[0]; + } + List kept = new ArrayList<>(maxFrames); + for (StackTraceElement frame : raw) { + if (isDropped(frame)) { + continue; + } + kept.add(frame); + if (kept.size() >= maxFrames) { + break; + } + } + return kept.toArray(new StackTraceElement[0]); + } + + private static boolean isDropped(StackTraceElement frame) { + String fqn = frame.getClassName() + "." + frame.getMethodName(); + for (String prefix : DROP_PREFIXES) { + if (fqn.startsWith(prefix) || frame.getClassName().startsWith(prefix)) { + return true; + } + } + return false; + } +} diff --git a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/TrackScopeContinuations.java b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/TrackScopeContinuations.java new file mode 100644 index 00000000000..69b0d8556f6 --- /dev/null +++ b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/TrackScopeContinuations.java @@ -0,0 +1,25 @@ +package datadog.trace.agent.test.scopediag; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Inherited; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Overrides the scope-continuation diagnostic defaults for an instrumentation test class or method. + * Diagnostics run for every instrumentation test unless explicitly disabled. + * + *

Honored by both the JUnit5 {@link ScopeDiagnosticsExtension} and the Groovy/Spock {@code + * InstrumentationSpecification}. + */ +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.TYPE, ElementType.METHOD}) +@Inherited +public @interface TrackScopeContinuations { + /** Set to {@code false} only for a proven incompatibility with the diagnostic itself. */ + boolean enabled() default true; + + /** Required when disabling diagnostics. Explain the incompatibility, preferably with an issue. */ + String reason() default ""; +} diff --git a/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeContinuationProbeTest.java b/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeContinuationProbeTest.java new file mode 100644 index 00000000000..1cfc29256db --- /dev/null +++ b/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeContinuationProbeTest.java @@ -0,0 +1,99 @@ +package datadog.trace.agent.test.scopediag; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import org.junit.jupiter.api.Test; + +/** + * Guards the production names/constants that {@link ScopeContinuationProbe} relies on reflectively + * or mirrors. If any of these are renamed/changed in the tracer, these assertions fail loudly + * instead of the diagnostic silently going dark. + */ +class ScopeContinuationProbeTest { + + @Test + void cancelledSentinelMatchesProduction() throws Exception { + assertEquals(Integer.MIN_VALUE >> 1, ScopeContinuationProbe.CANCELLED); + + Class scopeContinuation = Class.forName("datadog.trace.core.scopemanager.ScopeContinuation"); + Field cancelled = scopeContinuation.getDeclaredField("CANCELLED"); + cancelled.setAccessible(true); + assertEquals( + cancelled.getInt(null), + ScopeContinuationProbe.CANCELLED, + "ScopeContinuationProbe.CANCELLED is out of sync with ScopeContinuation.CANCELLED"); + } + + @Test + void continuationHooksExist() throws Exception { + Class scopeContinuation = Class.forName("datadog.trace.core.scopemanager.ScopeContinuation"); + // methods woven by ScopeContinuationTransformer (matched by name) + assertNotNull(scopeContinuation.getDeclaredMethod("register"), "register() (capture)"); + assertNotNull(scopeContinuation.getDeclaredMethod("resume"), "resume()"); + assertNotNull(scopeContinuation.getDeclaredMethod("release"), "release() (resolve)"); + assertNotNull( + scopeContinuation.getDeclaredMethod("cancelFromContinuedScopeClose"), + "cancelFromContinuedScopeClose() (resolve)"); + // fields read by the Cancel advice (@Advice.FieldValue) and the probe (reflection) + assertNotNull(findField(scopeContinuation, "count"), "ScopeContinuation.count"); + assertNotNull(findField(scopeContinuation, "source"), "ScopeContinuation.source"); + } + + @Test + void rootWrittenHookExists() throws Exception { + Class pendingTrace = Class.forName("datadog.trace.core.PendingTrace"); + // PendingTraceAdvice matches write(boolean) and reads these fields via @Advice.FieldValue + assertNotNull( + pendingTrace.getDeclaredMethod("write", boolean.class), "PendingTrace.write(boolean)"); + assertNotNull(findField(pendingTrace, "rootSpanWritten"), "PendingTrace.rootSpanWritten"); + assertNotNull(findField(pendingTrace, "traceId"), "PendingTrace.traceId"); + } + + @Test + void scopeLifecycleHooksExist() throws Exception { + Class scope = Class.forName("datadog.trace.core.scopemanager.ContinuableScope"); + assertNotNull(scope.getDeclaredMethod("afterActivated"), "afterActivated() (scope open)"); + assertNotNull(scope.getDeclaredMethod("onProperClose"), "onProperClose() (scope close)"); + assertNotNull(scope.getDeclaredMethod("close"), "close() (wrong-thread check)"); + // source byte read reflectively in the probe + assertNotNull(findField(scope, "source"), "ContinuableScope.source"); + + Class continuing = Class.forName("datadog.trace.core.scopemanager.ContinuingScope"); + assertNotNull( + continuing.getDeclaredField("continuation"), "ContinuingScope.continuation (scope link)"); + } + + @Test + void wrongThreadCheckChainExists() throws Exception { + Class scope = Class.forName("datadog.trace.core.scopemanager.ContinuableScope"); + assertNotNull(findField(scope, "scopeManager"), "ContinuableScope.scopeManager"); + Class manager = Class.forName("datadog.trace.core.scopemanager.ContinuableScopeManager"); + assertNotNull(manager.getDeclaredMethod("scopeStack"), "ContinuableScopeManager.scopeStack()"); + Class stack = Class.forName("datadog.trace.core.scopemanager.ScopeStack"); + assertTrue(hasMethod(stack, "checkTop", 1), "ScopeStack.checkTop(scope)"); + } + + private static Field findField(Class cls, String name) { + for (Class c = cls; c != null; c = c.getSuperclass()) { + try { + return c.getDeclaredField(name); + } catch (NoSuchFieldException ignored) { + // keep walking + } + } + return null; + } + + private static boolean hasMethod(Class cls, String name, int params) { + for (Method m : cls.getDeclaredMethods()) { + if (m.getName().equals(name) && m.getParameterCount() == params) { + return true; + } + } + return false; + } +} diff --git a/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsConfigurationTest.java b/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsConfigurationTest.java new file mode 100644 index 00000000000..2b0ada14e7a --- /dev/null +++ b/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsConfigurationTest.java @@ -0,0 +1,36 @@ +package datadog.trace.agent.test.scopediag; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; + +class ScopeDiagnosticsConfigurationTest { + + @TrackScopeContinuations(enabled = false) + private static class UndocumentedOptOut {} + + @TrackScopeContinuations(enabled = false, reason = "incompatible synthetic tracer") + private static class DocumentedOptOut {} + + @Test + void diagnosticsAreEnabledByDefault() { + assertTrue(ScopeDiagnostics.isEnabled(null)); + } + + @Test + void documentedOptOutDisablesDiagnostics() { + assertFalse( + ScopeDiagnostics.isEnabled( + DocumentedOptOut.class.getAnnotation(TrackScopeContinuations.class))); + } + + @Test + void undocumentedOptOutIsRejected() { + TrackScopeContinuations config = + UndocumentedOptOut.class.getAnnotation(TrackScopeContinuations.class); + + assertThrows(IllegalArgumentException.class, () -> ScopeDiagnostics.isEnabled(config)); + } +} diff --git a/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsIntegrationTest.java b/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsIntegrationTest.java new file mode 100644 index 00000000000..4c4f1478a75 --- /dev/null +++ b/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsIntegrationTest.java @@ -0,0 +1,175 @@ +package datadog.trace.agent.test.scopediag; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import datadog.context.ContextContinuation; +import datadog.context.ContextScope; +import datadog.trace.bootstrap.instrumentation.api.AgentScope; +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.common.writer.ListWriter; +import datadog.trace.core.CoreTracer; +import java.util.List; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +/** + * Exercises the full path: a real {@link CoreTracer} capturing continuations via {@code + * captureSpan} drives {@code ScopeContinuation} -> {@code ScopeContinuationProbe} -> {@link + * ScopeDiagnostics}, and the derived report classifies the leak correctly. + */ +class ScopeDiagnosticsIntegrationTest { + + private CoreTracer tracer; + + @AfterEach + void tearDown() { + ScopeDiagnostics.stop(); + ScopeDiagnostics.reset(); + if (tracer != null) { + tracer.close(); + } + } + + @Test + void capturesRealLeakAndResolvedContinuation() { + tracer = CoreTracer.builder().writer(new ListWriter()).strictTraceWrites(false).build(); + + ScopeDiagnostics.startRecording(); + + AgentSpan span = tracer.startSpan("test", "op"); + ContextContinuation leaked = tracer.capture(span); // captured, never resolved + ContextContinuation resolved = tracer.capture(span); + resolved.release(); + + ScopeDiagnosticsReport report = ScopeDiagnostics.report(); + + assertEquals(2, report.records().size(), "both captures recorded"); + assertEquals(1, report.leakCount(), "exactly the un-resolved continuation leaks"); + assertTrue(report.hasProblems()); + // keep a reference so the leak isn't reclaimed before the assertion + assertFalse(leaked.toString().isEmpty()); + + span.finish(); + } + + @Test + void sameSpanReactivationIsNotFlaggedActivateAfterResolve() { + tracer = CoreTracer.builder().writer(new ListWriter()).strictTraceWrites(false).build(); + + ScopeDiagnostics.startRecording(); + + AgentSpan span = tracer.startSpan("test", "op"); + AgentScope active = tracer.activateSpan(span); // span becomes the active top scope + // Capturing then immediately activating the already-active span hits the continueSpan reuse + // optimization: it cancels the continuation from inside activate() before activate() returns. + // The resume must be timestamped at activate() entry (not exit) so it does not appear to occur + // after that internal resolution and spuriously trip ACTIVATE_AFTER_RESOLVE. + ContextContinuation continuation = tracer.capture(span); + ContextScope reused = continuation.resume(); + reused.close(); + active.close(); + span.finish(); + + ScopeDiagnosticsReport report = ScopeDiagnostics.report(); + + assertEquals(1, report.records().size()); + assertEquals( + 0, + report.activateAfterResolveCount(), + "a same-span re-activation resolved during activate() is not activate-after-resolve"); + assertEquals(0, report.leakCount()); + assertFalse(report.hasProblems()); + } + + @Test + void resolvedContinuationDoesNotLeak() { + tracer = CoreTracer.builder().writer(new ListWriter()).strictTraceWrites(false).build(); + + ScopeDiagnostics.startRecording(); + + AgentSpan span = tracer.startSpan("test", "op"); + ContextContinuation continuation = tracer.capture(span); + ContextScope scope = continuation.resume(); + scope.close(); + span.finish(); + + ScopeDiagnosticsReport report = ScopeDiagnostics.report(); + + assertEquals(1, report.records().size()); + assertEquals(0, report.leakCount(), "activated then closed continuation is resolved"); + } + + @Test + void scopeLifetimeRecordedAndLinkedToContinuation() { + tracer = CoreTracer.builder().writer(new ListWriter()).strictTraceWrites(false).build(); + + ScopeDiagnostics.startRecording(); + + AgentSpan span = tracer.startSpan("test", "op"); + ContextContinuation continuation = tracer.capture(span); + ContextScope scope = continuation.resume(); + scope.close(); + span.finish(); + + ScopeDiagnosticsReport report = ScopeDiagnostics.report(); + + ScopeRecord linked = continuationScope(report); + assertNotNull(linked, "the resumed scope was recorded"); + assertNotNull(linked.open(), "scope open observed"); + assertTrue(linked.closed(), "scope close observed"); + assertEquals(0, report.neverClosedScopeCount()); + // the scope links back to its continuation record + assertEquals(1, report.records().size()); + assertEquals(Long.valueOf(report.records().get(0).seq), linked.continuationSeq); + } + + @Test + void neverClosedScopeIsFlagged() { + tracer = CoreTracer.builder().writer(new ListWriter()).strictTraceWrites(false).build(); + + ScopeDiagnostics.startRecording(); + + AgentSpan span = tracer.startSpan("test", "op"); + ContextContinuation continuation = tracer.capture(span); + ContextScope scope = continuation.resume(); // opened, never closed + + ScopeDiagnosticsReport report = ScopeDiagnostics.report(); + + assertEquals(1, report.neverClosedScopeCount(), "the open scope never closed"); + assertEquals(1, report.leakCount(), "and the continuation it backs also leaks"); + assertTrue(report.hasProblems()); + + // clean up so the open scope does not pollute this thread's scope stack for later tests + scope.close(); + span.finish(); + } + + @Test + void eventsAfterStopAreRejected() { + tracer = CoreTracer.builder().writer(new ListWriter()).strictTraceWrites(false).build(); + + ScopeDiagnostics.startRecording(); + AgentSpan span = tracer.startSpan("test", "op"); + ContextContinuation continuation = tracer.capture(span); + + ScopeDiagnostics.stop(); + continuation.release(); + ScopeDiagnosticsReport report = ScopeDiagnostics.report(); + + assertEquals(1, report.leakCount(), "the resolution happened outside the recording window"); + span.finish(); + } + + private static ScopeRecord continuationScope(ScopeDiagnosticsReport report) { + List scopes = report.scopeRecords(); + for (ScopeRecord s : scopes) { + if (s.continuationSeq != null) { + return s; + } + } + return null; + } +} diff --git a/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsReportTest.java b/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsReportTest.java new file mode 100644 index 00000000000..a2462b294ae --- /dev/null +++ b/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsReportTest.java @@ -0,0 +1,179 @@ +package datadog.trace.agent.test.scopediag; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import datadog.trace.api.DDTraceId; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; + +class ScopeDiagnosticsReportTest { + + private static final StackTraceElement[] STACK = { + new StackTraceElement("com.app.Worker", "submit", "Worker.java", 42) + }; + + private static ScopeEvent event(ScopeEvent.Type type, String thread, long nanos) { + return new ScopeEvent(type, thread, nanos, STACK); + } + + private static ContinuationRecord record(long seq, DDTraceId trace) { + return new ContinuationRecord( + seq, trace, 7L, "op", (byte) 0, false, event(ScopeEvent.Type.CAPTURE, "main", 1000)); + } + + @Test + void resolvedContinuationHasNoFailures() { + ContinuationRecord r = record(0, DDTraceId.from(10)); + r.addResume(event(ScopeEvent.Type.ACTIVATE, "pool-1", 2000)); + r.setTerminalOrExtra(event(ScopeEvent.Type.RESOLVE_FINISH, "pool-1", 3000)); + + ScopeDiagnosticsReport report = report(list(r), map()); + + assertEquals(0, report.leakCount()); + assertEquals(0, report.lateCount()); + assertEquals(0, report.doubleCount()); + assertEquals(ContinuationStatus.FINISHED, r.status()); + assertTrue(r.threadHandoff()); // captured on main, resolved on pool-1 + assertFalse(report.hasProblems()); + } + + @Test + void neverResolvedIsFlaggedAsLeak() { + ContinuationRecord r = record(0, DDTraceId.from(11)); + + ScopeDiagnosticsReport report = report(list(r), map()); + + assertEquals(1, report.leakCount()); + assertEquals(ContinuationStatus.LEAKED, r.status()); + assertTrue(report.hasProblems()); + assertTrue(report.renderSummary().contains("LEAKED")); + // the capture callsite is surfaced in the problem summary + assertTrue(report.renderSummary().contains("Worker.java:42")); + } + + @Test + void resolutionAfterRootWriteIsFlaggedLate() { + DDTraceId trace = DDTraceId.from(12); + ContinuationRecord r = record(0, trace); + r.addResume(event(ScopeEvent.Type.ACTIVATE, "pool-1", 5000)); + r.setTerminalOrExtra(event(ScopeEvent.Type.RESOLVE_FINISH, "pool-1", 6000)); + + Map rootWritten = map(); + rootWritten.put(trace, 4000L); // root written before the activation/resolution + + ScopeDiagnosticsReport report = report(list(r), rootWritten); + + assertEquals(1, report.lateCount()); + assertEquals(0, report.leakCount()); // it is resolved, just late + } + + @Test + void lateFinishDoesNotFail() { + DDTraceId trace = DDTraceId.from(120); + ContinuationRecord r = record(0, trace); + r.addResume(event(ScopeEvent.Type.ACTIVATE, "pool-1", 5000)); + r.setTerminalOrExtra(event(ScopeEvent.Type.RESOLVE_FINISH, "pool-1", 6000)); + Map rootWritten = map(); + rootWritten.put(trace, 4000L); + + ScopeDiagnosticsReport report = report(list(r), rootWritten); + + assertEquals(1, report.lateCount()); + assertFalse(report.hasProblems()); // late-finish is report-only + } + + @Test + void multipleResolutionsAreFlaggedDouble() { + ContinuationRecord r = record(0, DDTraceId.from(13)); + r.addResume(event(ScopeEvent.Type.ACTIVATE, "pool-1", 2000)); + r.setTerminalOrExtra(event(ScopeEvent.Type.RESOLVE_FINISH, "pool-1", 3000)); + r.setTerminalOrExtra(event(ScopeEvent.Type.RESOLVE_FINISH, "pool-1", 3500)); + + ScopeDiagnosticsReport report = report(list(r), map()); + + assertEquals(1, report.doubleCount()); + assertTrue(report.hasProblems()); + } + + @Test + void activationAfterResolveIsFailure() { + ContinuationRecord r = record(0, DDTraceId.from(14)); + r.setTerminalOrExtra(event(ScopeEvent.Type.RESOLVE_CANCEL, "pool-1", 2000)); + r.addResume(event(ScopeEvent.Type.ACTIVATE, "pool-2", 3000)); // resume after cancel + + ScopeDiagnosticsReport report = report(list(r), map()); + + assertEquals(1, report.activateAfterResolveCount()); + assertEquals(0, report.doubleCount()); + assertTrue(report.hasProblems()); + } + + @Test + void failedActivationIsActivateAfterResolve() { + ContinuationRecord r = record(0, DDTraceId.from(141)); + r.setTerminalOrExtra(event(ScopeEvent.Type.RESOLVE_CANCEL, "pool-1", 2000)); + r.addFailedActivation(event(ScopeEvent.Type.ACTIVATE_FAILED, "pool-2", 3000)); + + ScopeDiagnosticsReport report = report(list(r), map()); + + assertEquals(1, report.activateAfterResolveCount()); + assertTrue(report.hasProblems()); + } + + @Test + void timelineRendersResolvedContinuationEvenWithoutProblems() { + ContinuationRecord r = record(0, DDTraceId.from(30)); + r.addResume(event(ScopeEvent.Type.ACTIVATE, "pool-1", 2000)); + r.setTerminalOrExtra(event(ScopeEvent.Type.RESOLVE_FINISH, "pool-1", 3000)); + + ScopeDiagnosticsReport report = report(list(r), map()); + + // a clean run: the summary reports no problems ... + assertFalse(report.hasProblems()); + assertTrue(report.renderSummary().contains("(none)")); + + // ... but the timeline still dumps the full lineage so a graph/report can be built + String timeline = report.renderTimeline(); + assertTrue(timeline.contains("#0 FINISHED")); + assertTrue(timeline.contains("capture")); + assertTrue(timeline.contains("resume")); + assertTrue(timeline.contains("finish")); + assertTrue(timeline.contains("Worker.java:42")); // callsite preserved + assertTrue(timeline.contains("@ pool-1")); // resume/finish thread preserved + } + + @Test + void reportIsAnImmutableSnapshot() { + ContinuationRecord record = record(0, DDTraceId.from(31)); + ScopeDiagnosticsReport report = report(list(record), map()); + + record.setTerminalOrExtra(event(ScopeEvent.Type.RESOLVE_FINISH, "pool-1", 3000)); + + assertEquals(1, report.leakCount()); + assertEquals(ContinuationStatus.LEAKED, report.records().get(0).status()); + } + + // ---- helpers ------------------------------------------------------------- + + private static ScopeDiagnosticsReport report( + List records, Map rootWritten) { + return new ScopeDiagnosticsReport(records, new ArrayList<>(), rootWritten); + } + + private static List list(ContinuationRecord... rs) { + List l = new ArrayList<>(); + for (ContinuationRecord r : rs) { + l.add(r); + } + return l; + } + + private static Map map() { + return new HashMap<>(); + } +} diff --git a/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeRecordTest.java b/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeRecordTest.java new file mode 100644 index 00000000000..7ad0112a432 --- /dev/null +++ b/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeRecordTest.java @@ -0,0 +1,86 @@ +package datadog.trace.agent.test.scopediag; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import datadog.trace.api.DDTraceId; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import org.junit.jupiter.api.Test; + +class ScopeRecordTest { + + private static final StackTraceElement[] STACK = { + new StackTraceElement("com.app.Worker", "run", "Worker.java", 7) + }; + + private static ScopeEvent event(ScopeEvent.Type type, String thread, long nanos) { + return new ScopeEvent(type, thread, nanos, STACK); + } + + private static ScopeRecord scope(long seq, Long continuationSeq, String openThread, long nanos) { + return new ScopeRecord( + seq, + DDTraceId.from(1), + 9L, + "op", + (byte) 0, + continuationSeq, + event(ScopeEvent.Type.SCOPE_OPEN, openThread, nanos)); + } + + private static ScopeDiagnosticsReport report(ScopeRecord... scopes) { + List list = new ArrayList<>(); + for (ScopeRecord s : scopes) { + list.add(s); + } + return new ScopeDiagnosticsReport(new ArrayList<>(), list, new HashMap<>()); + } + + @Test + void openAndClosedHasNoFailures() { + ScopeRecord s = scope(0, null, "main", 1000); + s.setClose(event(ScopeEvent.Type.SCOPE_CLOSE, "main", 3000)); + + assertTrue(s.closed()); + assertEquals(0, s.failures().size()); + assertFalse(s.threadHandoff()); + assertEquals(Long.valueOf(2000), s.activeDurationNanos()); + assertFalse(report(s).hasProblems()); + } + + @Test + void openWithoutCloseIsNeverClosed() { + ScopeRecord s = scope(0, null, "main", 1000); + + assertFalse(s.closed()); + assertTrue(s.failures().contains(Failure.NEVER_CLOSED)); + + ScopeDiagnosticsReport report = report(s); + assertEquals(1, report.neverClosedScopeCount()); + assertTrue(report.hasProblems()); // never-closed is a genuine bug + } + + @Test + void openAndCloseOnDifferentThreadsIsHandoff() { + ScopeRecord s = scope(0, null, "main", 1000); + s.setClose(event(ScopeEvent.Type.SCOPE_CLOSE, "pool-1", 2000)); + + assertTrue(s.threadHandoff()); + } + + @Test + void wrongThreadCloseIsReportedButDoesNotFail() { + ScopeRecord s = scope(0, null, "main", 1000); + s.setClose(event(ScopeEvent.Type.SCOPE_CLOSE, "main", 2000)); + s.addWrongThreadClose(event(ScopeEvent.Type.SCOPE_CLOSE_WRONG_THREAD, "pool-2", 1500)); + + assertTrue(s.failures().contains(Failure.CLOSE_WRONG_THREAD)); + + ScopeDiagnosticsReport report = report(s); + assertEquals(1, report.closeWrongThreadCount()); + assertFalse(report.hasProblems()); // wrong-thread is report-only + } +} diff --git a/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/StackFilterTest.java b/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/StackFilterTest.java new file mode 100644 index 00000000000..3853ab274fd --- /dev/null +++ b/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/StackFilterTest.java @@ -0,0 +1,53 @@ +package datadog.trace.agent.test.scopediag; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; + +class StackFilterTest { + + private static StackTraceElement frame(String cls, String method) { + return new StackTraceElement(cls, method, "Src.java", 1); + } + + @Test + void dropsPlumbingAndKeepsAppFrames() { + StackTraceElement[] raw = { + frame("java.lang.Thread", "getStackTrace"), + frame("datadog.trace.agent.test.scopediag.ScopeDiagnostics", "event"), + frame("datadog.trace.core.scopemanager.ScopeContinuation", "register"), + frame("java.util.concurrent.ThreadPoolExecutor", "execute"), + frame("com.app.Service", "doWork"), + frame("com.app.Main", "main"), + }; + + StackTraceElement[] filtered = new StackFilter(6).filter(raw); + + assertEquals(2, filtered.length); + assertEquals("com.app.Service", filtered[0].getClassName()); + assertEquals("com.app.Main", filtered[1].getClassName()); + } + + @Test + void respectsMaxFrames() { + StackTraceElement[] raw = { + frame("com.app.A", "a"), frame("com.app.B", "b"), frame("com.app.C", "c"), + }; + + assertEquals(2, new StackFilter(2).filter(raw).length); + } + + @Test + void handlesNullStack() { + assertEquals(0, new StackFilter(6).filter(null).length); + } + + @Test + void keepsScopeManagerFreeStacks() { + StackTraceElement[] raw = {frame("com.app.Only", "here")}; + StackTraceElement[] filtered = new StackFilter(6).filter(raw); + assertEquals(1, filtered.length); + assertTrue(filtered[0].getClassName().startsWith("com.app")); + } +} From ea50ad5c41292226e7de87d75be9916ba2bc0cf1 Mon Sep 17 00:00:00 2001 From: Andrea Marziali Date: Fri, 11 Sep 2026 10:28:23 +0200 Subject: [PATCH 02/35] code hygiene --- .../test/InstrumentationSpecification.groovy | 2 - .../scopediag/ContinuableScopeAdvice.java | 12 +-- .../test/scopediag/ContinuationAdvice.java | 32 +------ .../test/scopediag/ContinuationRecord.java | 24 +---- .../test/scopediag/ContinuationStatus.java | 9 +- .../trace/agent/test/scopediag/Failure.java | 6 +- .../test/scopediag/PendingTraceAdvice.java | 6 +- .../scopediag/ScopeContinuationProbe.java | 93 ++++--------------- .../ScopeContinuationTransformer.java | 65 ++++++++++--- .../test/scopediag/ScopeDiagnostics.java | 38 +------- .../scopediag/ScopeDiagnosticsExtension.java | 9 +- .../scopediag/ScopeDiagnosticsReport.java | 36 +------ .../agent/test/scopediag/ScopeEvent.java | 8 +- .../agent/test/scopediag/ScopeRecord.java | 17 +--- .../agent/test/scopediag/ScopeSources.java | 6 +- .../agent/test/scopediag/StackFilter.java | 10 +- .../scopediag/TrackScopeContinuations.java | 10 +- .../scopediag/ScopeContinuationProbeTest.java | 11 +-- .../ScopeDiagnosticsIntegrationTest.java | 20 +--- .../scopediag/ScopeDiagnosticsReportTest.java | 19 ++-- .../agent/test/scopediag/ScopeRecordTest.java | 2 +- 21 files changed, 109 insertions(+), 326 deletions(-) diff --git a/dd-java-agent/instrumentation-testing/src/main/groovy/datadog/trace/agent/test/InstrumentationSpecification.groovy b/dd-java-agent/instrumentation-testing/src/main/groovy/datadog/trace/agent/test/InstrumentationSpecification.groovy index 0ec8227a0a8..44aa4926b0a 100644 --- a/dd-java-agent/instrumentation-testing/src/main/groovy/datadog/trace/agent/test/InstrumentationSpecification.groovy +++ b/dd-java-agent/instrumentation-testing/src/main/groovy/datadog/trace/agent/test/InstrumentationSpecification.groovy @@ -540,7 +540,6 @@ abstract class InstrumentationSpecification extends DDSpecification implements A } } - /** Resolves the {@link TrackScopeContinuations} annotation from the feature method or spec class. */ private TrackScopeContinuations scopeDiagConfig() { def method = specificationContext?.currentFeature?.featureMethod?.reflection def ann = method?.getAnnotation(TrackScopeContinuations) @@ -554,7 +553,6 @@ abstract class InstrumentationSpecification extends DDSpecification implements A return ScopeDiagnostics.isEnabled(scopeDiagConfig()) } - /** Captures the diagnostic failure so the rest of Spock cleanup always runs first. */ private Throwable reportScopeDiagnostics() { def config = scopeDiagConfig() if (!ScopeDiagnostics.isEnabled(config)) { diff --git a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ContinuableScopeAdvice.java b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ContinuableScopeAdvice.java index 3dab0453128..1bbac24d23a 100644 --- a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ContinuableScopeAdvice.java +++ b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ContinuableScopeAdvice.java @@ -2,18 +2,10 @@ import net.bytebuddy.asm.Advice; -/** - * Test-only ByteBuddy advice woven into {@code datadog.trace.core.scopemanager.ContinuableScope} - * (and, by inheritance, {@code ContinuingScope}) to track the scope activation lifecycle. - * - *

The target type is package-private, so {@code this} is typed as {@link Object} and re-cast - * inside {@link ScopeContinuationProbe}. {@code afterActivated} is the open point (first call per - * scope identity), {@code onProperClose} the pop, and {@code close} the wrong-thread check. - */ +/** Test-only advice for {@code ContinuableScope}. */ public final class ContinuableScopeAdvice { private ContinuableScopeAdvice() {} - /** {@code afterActivated()} — the scope became active. */ public static final class AfterActivated { @Advice.OnMethodExit(suppress = Throwable.class) public static void exit(@Advice.This Object scope) { @@ -21,7 +13,6 @@ public static void exit(@Advice.This Object scope) { } } - /** {@code onProperClose()} — the scope was popped from its thread's stack. */ public static final class OnProperClose { @Advice.OnMethodExit(suppress = Throwable.class) public static void exit(@Advice.This Object scope) { @@ -29,7 +20,6 @@ public static void exit(@Advice.This Object scope) { } } - /** {@code close()} — check for an out-of-order / wrong-thread close. */ public static final class Close { @Advice.OnMethodEnter(suppress = Throwable.class) public static void enter(@Advice.This Object scope) { diff --git a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ContinuationAdvice.java b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ContinuationAdvice.java index 8264d3c87da..cf430b5ed22 100644 --- a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ContinuationAdvice.java +++ b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ContinuationAdvice.java @@ -2,18 +2,10 @@ import net.bytebuddy.asm.Advice; -/** - * Test-only ByteBuddy advice woven into {@code datadog.trace.core.scopemanager.ScopeContinuation}. - * - *

The target type is package-private and cannot be named here, so {@code this} is typed as - * {@link Object} and re-cast to the public {@code ContextContinuation} supertype inside {@link - * ScopeContinuationProbe}. {@link Advice.FieldValue} reads the private {@code count} field — legal - * because the advice is inlined into the field's own class. - */ +/** Test-only advice for {@code ScopeContinuation}. */ public final class ContinuationAdvice { private ContinuationAdvice() {} - /** {@code register()} — the continuation was captured. */ public static final class Register { @Advice.OnMethodExit(suppress = Throwable.class) public static void exit(@Advice.This Object self) { @@ -21,14 +13,7 @@ public static void exit(@Advice.This Object self) { } } - /** - * {@code resume()} — a (possibly noop) activation; the probe filters the rollback branch. - * - *

The activation timestamp is captured at method entry, not exit: the same-span reuse - * optimization ({@code ContinuableScopeManager.continueSpan}) cancels the continuation from - * inside {@code resume()} before it returns, so timestamping the resume at exit would - * order it after that internal resolution and spuriously flag {@code ACTIVATE_AFTER_RESOLVE}. - */ + /** Timestamps entry because {@code resume()} may resolve the continuation before returning. */ public static final class Activate { @Advice.OnMethodEnter public static long enter() { @@ -42,18 +27,7 @@ public static void exit( } } - /** - * Resolution detected via the {@code count} transition. Applied to both {@code release()} and - * {@code cancelFromContinuedScopeClose()} — they need identical before/after observation. The - * originating method name ({@code #m}) distinguishes an explicit cancel from a normal - * finish-on-scope-close. - * - *

The resolve timestamp is captured at method entry (the {@code ddResolveNanos} - * local), not at exit: the body itself may call {@code removeContinuation() -> - * PendingTrace.write()}, which is exactly where the root-written timestamp is taken. Timestamping - * at exit would place the resolution after the root write it triggered, producing a spurious - * late-finish. - */ + /** Timestamps entry because resolution may write the trace before the method returns. */ public static final class Cancel { @Advice.OnMethodEnter public static int enter( diff --git a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ContinuationRecord.java b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ContinuationRecord.java index d272eb90f83..845218efa2f 100644 --- a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ContinuationRecord.java +++ b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ContinuationRecord.java @@ -5,15 +5,7 @@ import java.util.EnumSet; import java.util.List; -/** - * The correlated continuation lifecycle: its capture, every resume (activation), any - * failed activation, and its terminal resolution (finish/cancel). Scope activation lifetimes are - * modelled separately by {@link ScopeRecord}; the scopes a continuation spawned are linked here by - * their seq ids ({@link #scopeRecordSeqs()}). - * - *

Built incrementally as events arrive on different threads, so all mutating access is - * synchronized on the instance. - */ +/** Records a continuation's capture, activations, scopes, and resolution. */ public final class ContinuationRecord { public final long seq; public final DDTraceId traceId; @@ -48,8 +40,6 @@ public final class ContinuationRecord { this.capture = capture; } - // ---- mutation ------------------------------------------------------------ - synchronized void addResume(ScopeEvent event) { resumes.add(event); } @@ -95,8 +85,6 @@ synchronized ContinuationRecord snapshot() { return copy; } - // ---- accessors ----------------------------------------------------------- - public synchronized ScopeEvent capture() { return capture; } @@ -121,16 +109,10 @@ public synchronized List scopeRecordSeqs() { return new ArrayList<>(scopeRecordSeqs); } - public synchronized boolean isResumed() { - return !resumes.isEmpty(); - } - public synchronized boolean isResolved() { return terminal != null; } - // ---- derived ------------------------------------------------------------- - public synchronized ContinuationStatus status() { if (terminal != null) { return terminal.type == ScopeEvent.Type.RESOLVE_CANCEL @@ -140,9 +122,7 @@ public synchronized ContinuationStatus status() { return ContinuationStatus.LEAKED; } - /** - * Derives the failure set for this continuation. {@code rootWrittenNanos} may be {@code null}. - */ + /** Derives failures, including events after {@code rootWrittenNanos} when provided. */ public synchronized EnumSet failures(Long rootWrittenNanos) { EnumSet failures = EnumSet.noneOf(Failure.class); if (terminal == null) { diff --git a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ContinuationStatus.java b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ContinuationStatus.java index 213a57a62a8..e26bf691fa4 100644 --- a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ContinuationStatus.java +++ b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ContinuationStatus.java @@ -1,14 +1,7 @@ package datadog.trace.agent.test.scopediag; -/** - * Derived lifecycle state of a continuation, for rendering. The authoritative bug signal is the - * {@link Failure} set, not this status. - */ +/** Derived continuation state used when rendering a timeline. */ public enum ContinuationStatus { - /** Captured but not yet resumed or resolved. */ - CAPTURED, - /** Resumed at least once but not yet resolved. */ - RESUMED, /** Resolved normally (all activations closed or a clean cancel with no outstanding work). */ FINISHED, /** Resolved via the cancel-with-outstanding-work path. */ diff --git a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/Failure.java b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/Failure.java index 6632609584d..64a596bf473 100644 --- a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/Failure.java +++ b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/Failure.java @@ -1,10 +1,6 @@ package datadog.trace.agent.test.scopediag; -/** - * A derived failure classification. Shared by {@link ContinuationRecord} (continuation-lifetime - * failures) and {@link ScopeRecord} (scope-lifetime failures). See {@link - * ScopeDiagnosticsReport#hasProblems()} for which of these fail a test versus are report-only. - */ +/** A derived scope or continuation lifecycle finding. */ public enum Failure { /** Continuation captured but never resolved within the window. */ LEAKED, diff --git a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/PendingTraceAdvice.java b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/PendingTraceAdvice.java index 8d583cf0618..a93b2936b7c 100644 --- a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/PendingTraceAdvice.java +++ b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/PendingTraceAdvice.java @@ -3,11 +3,7 @@ import net.bytebuddy.asm.Advice; /** - * Test-only ByteBuddy advice woven into {@code datadog.trace.core.PendingTrace}. Fires the - * root-written signal after {@code write(boolean)} actually changes {@code rootSpanWritten} from - * false to true. Observing the completed transition avoids treating an empty write as a root write. - * The timestamp is conservative: a resolution racing inside {@code write} may be omitted from the - * late category, but it cannot be falsely classified as late. + * Records a root write only after {@code PendingTrace.write(boolean)} sets {@code rootSpanWritten}. */ public final class PendingTraceAdvice { private PendingTraceAdvice() {} diff --git a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeContinuationProbe.java b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeContinuationProbe.java index f4240759077..18ffe219f28 100644 --- a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeContinuationProbe.java +++ b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeContinuationProbe.java @@ -8,54 +8,37 @@ import java.lang.reflect.Field; import java.lang.reflect.Method; -/** - * Recorder hook that the test-only ByteBuddy advice ({@link ContinuationAdvice}, {@link - * PendingTraceAdvice}) funnels scope-continuation lifecycle events into. It replaces the former - * production {@code ContinuationDiagnostics} seam: the advice is woven into {@code - * datadog.trace.core.scopemanager.ScopeContinuation} and {@code datadog.trace.core.PendingTrace} at - * test time only, so production tracer code carries no diagnostic footprint at all. - * - *

Inlined advice runs in the same app classloader as this class at test time, so it can call - * these statics directly. Every entry point first checks the {@link #recording} flag and is fully - * wrapped so a diagnostic failure can never propagate back into the tracer. - */ +/** Forwards test-only Byte Buddy advice events to {@link ScopeDiagnostics}. */ public final class ScopeContinuationProbe { /** - * Mirrors {@code ScopeContinuation.CANCELLED} (see {@code - * dd-trace-core/.../scopemanager/ScopeContinuation.java}). A continuation is resolved exactly - * when its {@code count} field transitions to this sentinel during a cancel call. Kept in sync by - * {@code ScopeContinuationProbeTest}. + * Mirrors {@code ScopeContinuation.CANCELLED}. Reaching this value marks a resolved continuation; + * {@code ScopeContinuationProbeTest} detects drift. */ static final int CANCELLED = Integer.MIN_VALUE >> 1; private static volatile boolean recording = false; - /** Cached reflective handle to the package-private {@code ScopeContinuation.source} field. */ private static volatile Field sourceField; - // cached reflective handles for scope-lifecycle reads (set-once, best-effort) - private static volatile Field scopeSourceField; // ContinuableScope.source - private static volatile Field continuationField; // ContinuingScope.continuation - private static volatile Field scopeManagerField; // ContinuableScope.scopeManager - private static volatile Method scopeStackMethod; // ContinuableScopeManager.scopeStack() - private static volatile Method checkTopMethod; // ScopeStack.checkTop(ContinuableScope) + private static volatile Field scopeSourceField; + private static volatile Field continuationField; + private static volatile Field scopeManagerField; + private static volatile Method scopeStackMethod; + private static volatile Method checkTopMethod; private ScopeContinuationProbe() {} - /** Installs the transformer (once per JVM) and opens the cheap advice-side recording gate. */ + /** Installs the transformer once and starts recording. */ static synchronized void enable() { ScopeContinuationTransformer.install(); recording = true; } - /** Stops recording. The transformer stays installed (inert while not recording). */ + /** Stops recording without uninstalling the transformer. */ static void disable() { recording = false; } - // ---- advice entry points (public so inlined advice can reference them) ------------------- - - /** {@code ScopeContinuation.register()} exit: the continuation was captured. */ public static void onCapture(Object self) { if (!recording) { return; @@ -68,15 +51,10 @@ public static void onCapture(Object self) { continuation, span.getTraceId(), span.getSpanId(), spanName(span), sourceOf(self)); } } catch (Throwable ignored) { - // diagnostics must never disturb the tracer + // Diagnostics must never affect the tracer. } } - /** - * {@code ScopeContinuation.resume()} exit: a real activation happened. The rollback branch - * returns the {@link NoopScope#INSTANCE noop scope} singleton, so a returned noop scope is - * skipped — this exactly reproduces the original "success branch only" semantics. - */ public static void onActivate(Object self, Object returnedScope, long activateNanos) { if (!recording) { return; @@ -84,8 +62,7 @@ public static void onActivate(Object self, Object returnedScope, long activateNa try { ContextContinuation continuation = (ContextContinuation) self; if (returnedScope == NoopScope.INSTANCE) { - // activate() returned the noop scope: the continuation was already resolved. This is the - // activate-after-resolve signal — the engine records it only if a terminal was seen. + // A noop result may indicate activation after resolution. ScopeDiagnostics.recordActivateFailed(continuation); return; } @@ -103,35 +80,25 @@ public static void onActivate(Object self, Object returnedScope, long activateNa } } - /** - * {@code ScopeContinuation.release()} / {@code cancelFromContinuedScopeClose()} exit. The - * original production seam fired only from inside the clean resolution branch; here we detect - * that branch by observing the {@code count} field transition to {@link #CANCELLED} during this - * call. A cancel with outstanding activations leaves {@code count} unchanged (not a resolution). - */ public static void onResolve( Object self, String method, int countBefore, int countAfter, long resolveNanos) { if (!recording) { return; } if (countAfter != CANCELLED) { - return; // not a resolution + return; } - // An explicit cancel() is a discard; cancelFromContinuedScopeClose() is a normal finish once - // the continued scope closes. (Caveat: the rare cancelFromContinuedScopeClose slow path - // delegates to cancel(), so a multi-activation finish is recorded as a cancel.) + // release discards; cancelFromContinuedScopeClose finishes. Its slow path delegates to release, + // so a multi-activation finish can appear as a cancellation. boolean cancelled = "release".equals(method); try { ContextContinuation continuation = (ContextContinuation) self; - // countBefore == CANCELLED is a genuine second finish/cancel. First transitions are deduped - // by the engine under the same lock as stop/reset/report, including nested advice frames. ScopeDiagnostics.recordResolve( continuation, cancelled, resolveNanos, countBefore == CANCELLED); } catch (Throwable ignored) { } } - /** {@code PendingTrace.write()} root-written site. */ public static void onRootWritten(Object traceId) { if (!recording) { return; @@ -142,11 +109,6 @@ public static void onRootWritten(Object traceId) { } } - /** - * {@code ContinuableScope.afterActivated()} exit: a scope became active. Re-activations (parent - * restored after a child closes) reach here too; the engine keeps only the first per scope - * identity. Links to the spawning continuation when the scope is a {@code ContinuingScope}. - */ public static void onScopeOpen(Object scope) { if (!recording) { return; @@ -162,9 +124,6 @@ public static void onScopeOpen(Object scope) { } } - /** - * {@code ContinuableScope.onProperClose()} exit: the scope was popped from its thread's stack. - */ public static void onScopeClose(Object scope) { if (!recording) { return; @@ -175,11 +134,7 @@ public static void onScopeClose(Object scope) { } } - /** - * {@code ContinuableScope.close()} entry: if the scope is not on top of its thread's stack, this - * is an out-of-order / wrong-thread close. Best-effort — silently does nothing if the internal - * stack check cannot be reached reflectively. - */ + /** Records an out-of-order close when the internal stack can be inspected. */ public static void onScopeClosing(Object scope) { if (!recording) { return; @@ -192,7 +147,7 @@ public static void onScopeClosing(Object scope) { } } - /** Snapshots the span name as a String (the CharSequence may mutate later), or {@code null}. */ + /** Copies the possibly mutable span name. */ private static String spanName(AgentSpan span) { try { CharSequence name = span.getSpanName(); @@ -202,9 +157,6 @@ private static String spanName(AgentSpan span) { } } - /** - * Reads the package-private {@code source} byte field, falling back to the {@code -1} sentinel. - */ private static byte sourceOf(Object self) { try { Field field = sourceField; @@ -219,7 +171,6 @@ private static byte sourceOf(Object self) { } } - /** Reads the {@code source} byte of a scope (declared on {@code ContinuableScope}). */ private static byte scopeSourceOf(Object scope) { try { Field field = scopeSourceField; @@ -233,10 +184,6 @@ private static byte scopeSourceOf(Object scope) { } } - /** - * The continuation that spawned a scope, read from {@code ContinuingScope.continuation}; {@code - * null} for a plain (non-continuation) scope. - */ private static ContextContinuation continuationOf(Object scope) { try { Field field = continuationField; @@ -245,7 +192,7 @@ private static ContextContinuation continuationOf(Object scope) { continuationField = field; } if (field == null || !field.getDeclaringClass().isInstance(scope)) { - return null; // not a ContinuingScope + return null; } Object value = field.get(scope); return value instanceof ContextContinuation ? (ContextContinuation) value : null; @@ -254,7 +201,6 @@ private static ContextContinuation continuationOf(Object scope) { } } - /** Best-effort: {@code true} when the scope is not on top of its thread's scope stack. */ private static boolean isNotOnTop(Object scope) { try { Field managerField = scopeManagerField; @@ -290,7 +236,6 @@ private static boolean isNotOnTop(Object scope) { } } - /** Finds a named field declared on a class or any superclass, made accessible. */ private static Field findField(Class cls, String name) { for (Class c = cls; c != null; c = c.getSuperclass()) { try { @@ -298,13 +243,11 @@ private static Field findField(Class cls, String name) { f.setAccessible(true); return f; } catch (NoSuchFieldException ignored) { - // keep walking up } } return null; } - /** Finds a named method with the given parameter count on a class or superclass, accessible. */ private static Method findMethod(Class cls, String name, int paramCount) { for (Class c = cls; c != null; c = c.getSuperclass()) { for (Method m : c.getDeclaredMethods()) { diff --git a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeContinuationTransformer.java b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeContinuationTransformer.java index c364729de22..3bb99277a6a 100644 --- a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeContinuationTransformer.java +++ b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeContinuationTransformer.java @@ -1,6 +1,8 @@ package datadog.trace.agent.test.scopediag; +import static net.bytebuddy.matcher.ElementMatchers.isMethod; import static net.bytebuddy.matcher.ElementMatchers.named; +import static net.bytebuddy.matcher.ElementMatchers.returns; import static net.bytebuddy.matcher.ElementMatchers.takesArguments; import java.lang.instrument.Instrumentation; @@ -10,14 +12,8 @@ import net.bytebuddy.asm.Advice; /** - * Installs the test-only diagnostic advice into the tracer's own {@code - * datadog.trace.core.scopemanager.ScopeContinuation} and {@code datadog.trace.core.PendingTrace}. - * - *

These classes sit under {@code datadog.trace.core.*}, which the tracer's own {@code - * AgentBuilder} hard-ignores. We therefore install a separate {@link AgentBuilder} on the - * raw {@link Instrumentation} with no global-ignore filter, using retransformation so the - * already-loaded classes are rewoven on install. The advice is schema-preserving, so {@code - * disableClassFormatChanges()} + {@code REDEFINE} keep it retransform-safe. Installed once per JVM. + * Installs test-only advice with a separate {@link AgentBuilder} because the tracer ignores its own + * core classes. Retransformation covers classes loaded before the diagnostic starts. */ final class ScopeContinuationTransformer { private static volatile ResettableClassFileTransformer transformer; @@ -38,28 +34,67 @@ static synchronized void install() { .transform( (builder, type, classLoader, module, pd) -> builder - .visit(Advice.to(ContinuationAdvice.Register.class).on(named("register"))) - .visit(Advice.to(ContinuationAdvice.Activate.class).on(named("resume"))) + .visit( + Advice.to(ContinuationAdvice.Register.class) + .on( + isMethod() + .and(named("register")) + .and(takesArguments(0)) + .and( + returns( + named( + "datadog.trace.core.scopemanager.ScopeContinuation"))))) + .visit( + Advice.to(ContinuationAdvice.Activate.class) + .on( + isMethod() + .and(named("resume")) + .and(takesArguments(0)) + .and(returns(named("datadog.context.ContextScope"))))) .visit( Advice.to(ContinuationAdvice.Cancel.class) - .on(named("release").or(named("cancelFromContinuedScopeClose"))))) + .on( + isMethod() + .and( + named("release") + .or(named("cancelFromContinuedScopeClose"))) + .and(takesArguments(0)) + .and(returns(void.class))))) .type(named("datadog.trace.core.PendingTrace")) .transform( (builder, type, classLoader, module, pd) -> builder.visit( Advice.to(PendingTraceAdvice.Write.class) - .on(named("write").and(takesArguments(boolean.class))))) + .on( + isMethod() + .and(named("write")) + .and(takesArguments(boolean.class)) + .and(returns(int.class))))) .type(named("datadog.trace.core.scopemanager.ContinuableScope")) .transform( (builder, type, classLoader, module, pd) -> builder .visit( Advice.to(ContinuableScopeAdvice.AfterActivated.class) - .on(named("afterActivated"))) + .on( + isMethod() + .and(named("afterActivated")) + .and(takesArguments(0)) + .and(returns(void.class)))) .visit( Advice.to(ContinuableScopeAdvice.OnProperClose.class) - .on(named("onProperClose"))) - .visit(Advice.to(ContinuableScopeAdvice.Close.class).on(named("close")))) + .on( + isMethod() + .and(named("onProperClose")) + .and(takesArguments(0)) + .and(returns(void.class)))) + .visit( + Advice.to(ContinuableScopeAdvice.Close.class) + .on( + isMethod() + .and(named("close")) + .and(takesArguments(0)) + .and(returns(void.class))))) .installOn(instrumentation); } } diff --git a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeDiagnostics.java b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeDiagnostics.java index d06777fa6ac..3c135721c22 100644 --- a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeDiagnostics.java +++ b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeDiagnostics.java @@ -9,27 +9,7 @@ import java.util.Map; import java.util.Set; -/** - * Test-time engine that records scope/continuation lifecycle events and renders leak reports. - * - *

It models two correlated lifecycles separately: continuations ({@link ContinuationRecord} — - * captured/resumed/finished) and scopes ({@link ScopeRecord} — opened/closed). While recording, - * {@link ScopeContinuationProbe} (test-only bytecode advice) feeds it events, which it correlates - * by identity (an {@link IdentityHashMap}, never {@code equals}/{@code hashCode}) — continuations - * by their {@code ContextContinuation} instance, scopes by their scope instance. It assumes a - * single test runs at a time per JVM (true for instrumentation tests); {@link #reset()} isolates - * one test from the next. - * - *

Usage: - * - * 

- *   ScopeDiagnostics.startRecording();
- *   ... exercise code under test ...
- *   System.out.println(ScopeDiagnostics.report().renderSummary());
- *   ScopeDiagnostics.assertNoLeaks();   // optional
- *   ScopeDiagnostics.stop();
- *
- */ +/** Records test-time scope and continuation lifecycles and reports leaks. */ public final class ScopeDiagnostics { private static final int DEFAULT_MAX_FRAMES = 6; @@ -52,8 +32,6 @@ public final class ScopeDiagnostics { private ScopeDiagnostics() {} - // ---- public static facade ------------------------------------------------ - /** Clears any prior data and starts recording with the default stack depth. */ public static void startRecording() { startRecording(DEFAULT_MAX_FRAMES); @@ -90,7 +68,7 @@ public static void reset() { } } - /** Builds an immutable snapshot report of everything recorded so far. */ + /** Returns an immutable snapshot of the events recorded so far. */ public static ScopeDiagnosticsReport report() { synchronized (INSTANCE.lifecycleLock) { return new ScopeDiagnosticsReport( @@ -116,7 +94,7 @@ public static void assertNoLeaks(ScopeDiagnosticsReport report) { } } - /** Resolves the default-on policy and rejects undocumented opt-outs. */ + /** Resolves the default-on policy and rejects opt-outs without a reason. */ public static boolean isEnabled(TrackScopeContinuations config) { if (config == null || config.enabled()) { return true; @@ -137,27 +115,21 @@ private void clear() { scopeSeq = 0; } - // ---- listener implementation --------------------------------------------- - private static final StackTraceElement[] NO_STACK = new StackTraceElement[0]; private ScopeEvent event(ScopeEvent.Type type) { return event(type, System.nanoTime()); } - /** Builds an event with an explicit timestamp (thread and stack are still captured now). */ + /** Uses the supplied event time while capturing the thread and stack at the call site. */ private ScopeEvent event(ScopeEvent.Type type, long nanos) { - // Capturing a stack per event is the dominant cost and perturbs the very timings we record; - // skip it entirely when callsites are disabled (maxFrames <= 0) rather than walking then - // trimming. + // Avoid stack walking when call sites are disabled because it perturbs recorded timings. StackFilter filter = stackFilter; StackTraceElement[] stack = filter.maxFrames() <= 0 ? NO_STACK : filter.filter(new Throwable().getStackTrace()); return new ScopeEvent(type, Thread.currentThread().getName(), nanos, stack); } - // ---- static forwarders called by ScopeContinuationProbe ------------------ - static void recordCapture( ContextContinuation id, DDTraceId traceId, long spanId, String spanName, byte source) { synchronized (INSTANCE.lifecycleLock) { diff --git a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsExtension.java b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsExtension.java index 54eafd6dccd..ef82829a6c0 100644 --- a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsExtension.java +++ b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsExtension.java @@ -9,12 +9,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/** - * JUnit5 extension that enables {@link ScopeDiagnostics} for every test extending {@code - * AbstractInstrumentationTest}. {@link TrackScopeContinuations} can override or disable it. - * - *

Per test: resets and starts recording before; after, logs findings and asserts no leaks. - */ +/** Runs {@link ScopeDiagnostics} around each JUnit instrumentation test. */ public final class ScopeDiagnosticsExtension implements BeforeEachCallback, AfterEachCallback { private static final Logger log = LoggerFactory.getLogger(ScopeDiagnosticsExtension.class); @@ -44,7 +39,7 @@ public void afterEach(ExtensionContext context) { } } - /** Method-level annotation wins; otherwise the test class (incl. inherited). */ + /** Resolves method configuration before inherited class configuration. */ private static TrackScopeContinuations resolve(ExtensionContext context) { Optional element = context.getElement(); if (element.isPresent()) { diff --git a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsReport.java b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsReport.java index b494a68401c..a9d83583bcf 100644 --- a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsReport.java +++ b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsReport.java @@ -7,18 +7,7 @@ import java.util.List; import java.util.Map; -/** - * An immutable snapshot of the recorded continuation and scope lifecycles plus the derived failure - * findings. Exposes two text renderings: a problem-only summary ({@link #renderSummary()}) for - * quick triage / assertion messages, and a complete timeline ({@link #renderTimeline()}) that dumps - * every continuation and scope with its full event lineage. The harness emits the timeline - * when it contains a finding, giving humans and diagnostic skills enough data to build a graph. - * - *

The two lifecycles are kept separate: {@link ContinuationRecord} (captured → resumed → - * finished) and {@link ScopeRecord} (opened → closed). A scope spawned by resuming a continuation - * is linked to it ({@link ContinuationRecord#scopeRecordSeqs()} / {@link - * ScopeRecord#continuationSeq}) and rendered nested under it in the timeline. - */ +/** Immutable scope and continuation lifecycle snapshot with derived failures. */ public final class ScopeDiagnosticsReport { private final List continuations; private final List scopes; @@ -84,8 +73,6 @@ private static Map> classifyScopes(List records() { return new ArrayList<>(continuations); } @@ -136,12 +123,7 @@ private static int countWith(Map> findings, Failure fail return n; } - /** - * True when there is a genuine bug to fail on: a never-resolved leak, a double finish, an - * activation after resolve, or a scope that was never closed. {@link Failure#LATE_FINISH} and - * {@link Failure#CLOSE_WRONG_THREAD} are reported but do not fail (frequently legitimate async or - * teardown ordering). - */ + /** Returns whether the report contains a failure that should fail the test. */ public boolean hasProblems() { return leakCount() > 0 || doubleCount() > 0 @@ -154,8 +136,6 @@ public boolean hasFindings() { return !continuationFailures.isEmpty() || !scopeFailures.isEmpty(); } - // ---- rendering: text summary --------------------------------------------- - private void appendHeader(StringBuilder sb, String title) { sb.append(title) .append(" (") @@ -177,7 +157,7 @@ private void appendHeader(StringBuilder sb, String title) { .append(" wrong-thread)\n"); } - /** Problem-only summary: just the flagged continuations and scopes with their callsites. */ + /** Renders flagged continuations and scopes with their call sites. */ public String renderSummary() { StringBuilder sb = new StringBuilder(); appendHeader(sb, "Scope/continuation problems"); @@ -218,17 +198,9 @@ public String renderSummary() { return sb.toString(); } - // ---- rendering: complete timeline ---------------------------------------- - private static final int TIMELINE_FRAMES = 3; - /** - * Complete cross-thread timeline: one block per continuation (capture → resume(s) → terminal), - * with the scopes it spawned nested under it, followed by any non-continuation scopes. Unlike - * {@link #renderSummary()} this lists all records, not just the flagged ones, so a graph - * (Gantt/DAG) or report can be reconstructed from it whether or not anything leaked. Each event - * carries its relative time ({@code +Δms} from the first recorded event), thread, and callsite. - */ + /** Renders all events with relative times, threads, call sites, and linked scopes. */ public String renderTimeline() { StringBuilder sb = new StringBuilder(); appendHeader(sb, "Scope/continuation timeline"); diff --git a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeEvent.java b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeEvent.java index 4ad15cbbd9b..c1b77c461e5 100644 --- a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeEvent.java +++ b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeEvent.java @@ -1,10 +1,6 @@ package datadog.trace.agent.test.scopediag; -/** - * A single observed point in a continuation's lifecycle. Time, thread, and stack are captured by - * the recorder on the event's own thread (notifications are synchronous), so they reflect the - * thread that actually captured/activated/resolved the continuation. - */ +/** A timestamped scope or continuation lifecycle event with its thread and call site. */ public final class ScopeEvent { public enum Type { CAPTURE, @@ -13,9 +9,7 @@ public enum Type { ACTIVATE_FAILED, RESOLVE_FINISH, RESOLVE_CANCEL, - /** A scope became active (first activation). */ SCOPE_OPEN, - /** A scope was popped from its thread's stack. */ SCOPE_CLOSE, /** A scope was closed while not on top of its thread's stack. */ SCOPE_CLOSE_WRONG_THREAD diff --git a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeRecord.java b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeRecord.java index 8bd5bca1153..825a3444274 100644 --- a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeRecord.java +++ b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeRecord.java @@ -5,12 +5,7 @@ import java.util.EnumSet; import java.util.List; -/** - * The scope activation lifecycle: a scope opened (first activation) → closed (popped from - * its thread's stack). Distinct from the continuation lifecycle ({@link ContinuationRecord}); when - * a scope was spawned by resuming a continuation, {@link #continuationSeq} links back to that - * continuation's {@link ContinuationRecord#seq}. - */ +/** Records a scope's activation and close events. */ public final class ScopeRecord { public final long seq; public final DDTraceId traceId; @@ -18,9 +13,7 @@ public final class ScopeRecord { public final String spanName; public final byte source; - /** - * The seq of the continuation that spawned this scope, or {@code null} for a plain activation. - */ + /** The continuation that spawned this scope, or {@code null} for a plain activation. */ public final Long continuationSeq; private final ScopeEvent open; @@ -44,8 +37,6 @@ public final class ScopeRecord { this.open = open; } - // ---- mutation ------------------------------------------------------------ - synchronized void setClose(ScopeEvent event) { if (close == null) { close = event; @@ -73,8 +64,6 @@ synchronized ScopeRecord snapshot() { return copy; } - // ---- accessors ----------------------------------------------------------- - public synchronized ScopeEvent open() { return open; } @@ -91,8 +80,6 @@ public synchronized boolean closed() { return close != null; } - // ---- derived ------------------------------------------------------------- - /** {@code true} when the scope was opened and closed on different threads. */ public synchronized boolean threadHandoff() { return open != null && close != null && !open.threadName.equals(close.threadName); diff --git a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeSources.java b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeSources.java index 7bf7a81fa3b..f152c11ac39 100644 --- a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeSources.java +++ b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeSources.java @@ -1,10 +1,6 @@ package datadog.trace.agent.test.scopediag; -/** - * Maps the {@code byte} scope source used by the tracer to a readable name. Mirrors the constants - * in {@code datadog.trace.core.scopemanager.ContinuableScope} (which are package-private and not - * visible from here). - */ +/** Maps package-private {@code ContinuableScope} source values to readable names. */ final class ScopeSources { private ScopeSources() {} diff --git a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/StackFilter.java b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/StackFilter.java index 8b360d38fa0..e2e735a850e 100644 --- a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/StackFilter.java +++ b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/StackFilter.java @@ -3,21 +3,14 @@ import java.util.ArrayList; import java.util.List; -/** - * Trims a raw stack trace down to the frames that point at where a continuation was - * captured/activated/resolved: it drops the diagnostic plumbing, the scope-manager internals, and - * the executor/reflection scaffolding, keeping the top {@code maxFrames} meaningful frames. - */ +/** Removes diagnostic and runtime plumbing from captured call stacks. */ final class StackFilter { private static final String[] DROP_PREFIXES = { - // diagnostic harness itself "datadog.trace.agent.test.scopediag.", - // tracer scope/continuation machinery and the capture/activate plumbing it sits behind "datadog.trace.core.", "datadog.trace.bootstrap.instrumentation.java.concurrent.", "datadog.trace.bootstrap.instrumentation.api.", "datadog.trace.bootstrap.InstrumentationContext", - // JDK executor/reflection scaffolding between the caller and the capture "java.lang.Thread.getStackTrace", "java.util.concurrent.ThreadPoolExecutor", "java.util.concurrent.ScheduledThreadPoolExecutor", @@ -29,7 +22,6 @@ final class StackFilter { "jdk.internal.reflect.", "java.lang.reflect.", "sun.reflect.", - // Spock/Groovy/ByteBuddy mock & dynamic-dispatch scaffolding (test harness, not a callsite) "org.spockframework.mock.", "org.codehaus.groovy.", "groovy.lang.", diff --git a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/TrackScopeContinuations.java b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/TrackScopeContinuations.java index 69b0d8556f6..dcef5bb15a3 100644 --- a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/TrackScopeContinuations.java +++ b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/TrackScopeContinuations.java @@ -6,13 +6,7 @@ import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; -/** - * Overrides the scope-continuation diagnostic defaults for an instrumentation test class or method. - * Diagnostics run for every instrumentation test unless explicitly disabled. - * - *

Honored by both the JUnit5 {@link ScopeDiagnosticsExtension} and the Groovy/Spock {@code - * InstrumentationSpecification}. - */ +/** Configures the default-on scope and continuation diagnostic for a test class or method. */ @Retention(RetentionPolicy.RUNTIME) @Target({ElementType.TYPE, ElementType.METHOD}) @Inherited @@ -20,6 +14,6 @@ /** Set to {@code false} only for a proven incompatibility with the diagnostic itself. */ boolean enabled() default true; - /** Required when disabling diagnostics. Explain the incompatibility, preferably with an issue. */ + /** Explains why the diagnostic is disabled. Required when {@link #enabled()} is false. */ String reason() default ""; } diff --git a/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeContinuationProbeTest.java b/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeContinuationProbeTest.java index 1cfc29256db..3752b466af1 100644 --- a/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeContinuationProbeTest.java +++ b/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeContinuationProbeTest.java @@ -8,11 +8,7 @@ import java.lang.reflect.Method; import org.junit.jupiter.api.Test; -/** - * Guards the production names/constants that {@link ScopeContinuationProbe} relies on reflectively - * or mirrors. If any of these are renamed/changed in the tracer, these assertions fail loudly - * instead of the diagnostic silently going dark. - */ +/** Verifies the tracer internals used by {@link ScopeContinuationProbe}. */ class ScopeContinuationProbeTest { @Test @@ -31,14 +27,12 @@ void cancelledSentinelMatchesProduction() throws Exception { @Test void continuationHooksExist() throws Exception { Class scopeContinuation = Class.forName("datadog.trace.core.scopemanager.ScopeContinuation"); - // methods woven by ScopeContinuationTransformer (matched by name) assertNotNull(scopeContinuation.getDeclaredMethod("register"), "register() (capture)"); assertNotNull(scopeContinuation.getDeclaredMethod("resume"), "resume()"); assertNotNull(scopeContinuation.getDeclaredMethod("release"), "release() (resolve)"); assertNotNull( scopeContinuation.getDeclaredMethod("cancelFromContinuedScopeClose"), "cancelFromContinuedScopeClose() (resolve)"); - // fields read by the Cancel advice (@Advice.FieldValue) and the probe (reflection) assertNotNull(findField(scopeContinuation, "count"), "ScopeContinuation.count"); assertNotNull(findField(scopeContinuation, "source"), "ScopeContinuation.source"); } @@ -46,7 +40,6 @@ void continuationHooksExist() throws Exception { @Test void rootWrittenHookExists() throws Exception { Class pendingTrace = Class.forName("datadog.trace.core.PendingTrace"); - // PendingTraceAdvice matches write(boolean) and reads these fields via @Advice.FieldValue assertNotNull( pendingTrace.getDeclaredMethod("write", boolean.class), "PendingTrace.write(boolean)"); assertNotNull(findField(pendingTrace, "rootSpanWritten"), "PendingTrace.rootSpanWritten"); @@ -59,7 +52,6 @@ void scopeLifecycleHooksExist() throws Exception { assertNotNull(scope.getDeclaredMethod("afterActivated"), "afterActivated() (scope open)"); assertNotNull(scope.getDeclaredMethod("onProperClose"), "onProperClose() (scope close)"); assertNotNull(scope.getDeclaredMethod("close"), "close() (wrong-thread check)"); - // source byte read reflectively in the probe assertNotNull(findField(scope, "source"), "ContinuableScope.source"); Class continuing = Class.forName("datadog.trace.core.scopemanager.ContinuingScope"); @@ -82,7 +74,6 @@ private static Field findField(Class cls, String name) { try { return c.getDeclaredField(name); } catch (NoSuchFieldException ignored) { - // keep walking } } return null; diff --git a/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsIntegrationTest.java b/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsIntegrationTest.java index 4c4f1478a75..33b050542d6 100644 --- a/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsIntegrationTest.java +++ b/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsIntegrationTest.java @@ -15,11 +15,7 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; -/** - * Exercises the full path: a real {@link CoreTracer} capturing continuations via {@code - * captureSpan} drives {@code ScopeContinuation} -> {@code ScopeContinuationProbe} -> {@link - * ScopeDiagnostics}, and the derived report classifies the leak correctly. - */ +/** Exercises diagnostics against continuations created by a real {@link CoreTracer}. */ class ScopeDiagnosticsIntegrationTest { private CoreTracer tracer; @@ -40,7 +36,7 @@ void capturesRealLeakAndResolvedContinuation() { ScopeDiagnostics.startRecording(); AgentSpan span = tracer.startSpan("test", "op"); - ContextContinuation leaked = tracer.capture(span); // captured, never resolved + ContextContinuation leaked = tracer.capture(span); ContextContinuation resolved = tracer.capture(span); resolved.release(); @@ -49,7 +45,6 @@ void capturesRealLeakAndResolvedContinuation() { assertEquals(2, report.records().size(), "both captures recorded"); assertEquals(1, report.leakCount(), "exactly the un-resolved continuation leaks"); assertTrue(report.hasProblems()); - // keep a reference so the leak isn't reclaimed before the assertion assertFalse(leaked.toString().isEmpty()); span.finish(); @@ -62,11 +57,8 @@ void sameSpanReactivationIsNotFlaggedActivateAfterResolve() { ScopeDiagnostics.startRecording(); AgentSpan span = tracer.startSpan("test", "op"); - AgentScope active = tracer.activateSpan(span); // span becomes the active top scope - // Capturing then immediately activating the already-active span hits the continueSpan reuse - // optimization: it cancels the continuation from inside activate() before activate() returns. - // The resume must be timestamped at activate() entry (not exit) so it does not appear to occur - // after that internal resolution and spuriously trip ACTIVATE_AFTER_RESOLVE. + AgentScope active = tracer.activateSpan(span); + // Same-span reuse resolves the continuation before resume() returns. ContextContinuation continuation = tracer.capture(span); ContextScope reused = continuation.resume(); reused.close(); @@ -121,7 +113,6 @@ void scopeLifetimeRecordedAndLinkedToContinuation() { assertNotNull(linked.open(), "scope open observed"); assertTrue(linked.closed(), "scope close observed"); assertEquals(0, report.neverClosedScopeCount()); - // the scope links back to its continuation record assertEquals(1, report.records().size()); assertEquals(Long.valueOf(report.records().get(0).seq), linked.continuationSeq); } @@ -134,7 +125,7 @@ void neverClosedScopeIsFlagged() { AgentSpan span = tracer.startSpan("test", "op"); ContextContinuation continuation = tracer.capture(span); - ContextScope scope = continuation.resume(); // opened, never closed + ContextScope scope = continuation.resume(); ScopeDiagnosticsReport report = ScopeDiagnostics.report(); @@ -142,7 +133,6 @@ void neverClosedScopeIsFlagged() { assertEquals(1, report.leakCount(), "and the continuation it backs also leaks"); assertTrue(report.hasProblems()); - // clean up so the open scope does not pollute this thread's scope stack for later tests scope.close(); span.finish(); } diff --git a/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsReportTest.java b/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsReportTest.java index a2462b294ae..5155d563e0b 100644 --- a/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsReportTest.java +++ b/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsReportTest.java @@ -38,7 +38,7 @@ void resolvedContinuationHasNoFailures() { assertEquals(0, report.lateCount()); assertEquals(0, report.doubleCount()); assertEquals(ContinuationStatus.FINISHED, r.status()); - assertTrue(r.threadHandoff()); // captured on main, resolved on pool-1 + assertTrue(r.threadHandoff()); assertFalse(report.hasProblems()); } @@ -52,7 +52,6 @@ void neverResolvedIsFlaggedAsLeak() { assertEquals(ContinuationStatus.LEAKED, r.status()); assertTrue(report.hasProblems()); assertTrue(report.renderSummary().contains("LEAKED")); - // the capture callsite is surfaced in the problem summary assertTrue(report.renderSummary().contains("Worker.java:42")); } @@ -64,12 +63,12 @@ void resolutionAfterRootWriteIsFlaggedLate() { r.setTerminalOrExtra(event(ScopeEvent.Type.RESOLVE_FINISH, "pool-1", 6000)); Map rootWritten = map(); - rootWritten.put(trace, 4000L); // root written before the activation/resolution + rootWritten.put(trace, 4000L); ScopeDiagnosticsReport report = report(list(r), rootWritten); assertEquals(1, report.lateCount()); - assertEquals(0, report.leakCount()); // it is resolved, just late + assertEquals(0, report.leakCount()); } @Test @@ -84,7 +83,7 @@ void lateFinishDoesNotFail() { ScopeDiagnosticsReport report = report(list(r), rootWritten); assertEquals(1, report.lateCount()); - assertFalse(report.hasProblems()); // late-finish is report-only + assertFalse(report.hasProblems()); } @Test @@ -104,7 +103,7 @@ void multipleResolutionsAreFlaggedDouble() { void activationAfterResolveIsFailure() { ContinuationRecord r = record(0, DDTraceId.from(14)); r.setTerminalOrExtra(event(ScopeEvent.Type.RESOLVE_CANCEL, "pool-1", 2000)); - r.addResume(event(ScopeEvent.Type.ACTIVATE, "pool-2", 3000)); // resume after cancel + r.addResume(event(ScopeEvent.Type.ACTIVATE, "pool-2", 3000)); ScopeDiagnosticsReport report = report(list(r), map()); @@ -133,18 +132,16 @@ void timelineRendersResolvedContinuationEvenWithoutProblems() { ScopeDiagnosticsReport report = report(list(r), map()); - // a clean run: the summary reports no problems ... assertFalse(report.hasProblems()); assertTrue(report.renderSummary().contains("(none)")); - // ... but the timeline still dumps the full lineage so a graph/report can be built String timeline = report.renderTimeline(); assertTrue(timeline.contains("#0 FINISHED")); assertTrue(timeline.contains("capture")); assertTrue(timeline.contains("resume")); assertTrue(timeline.contains("finish")); - assertTrue(timeline.contains("Worker.java:42")); // callsite preserved - assertTrue(timeline.contains("@ pool-1")); // resume/finish thread preserved + assertTrue(timeline.contains("Worker.java:42")); + assertTrue(timeline.contains("@ pool-1")); } @Test @@ -158,8 +155,6 @@ void reportIsAnImmutableSnapshot() { assertEquals(ContinuationStatus.LEAKED, report.records().get(0).status()); } - // ---- helpers ------------------------------------------------------------- - private static ScopeDiagnosticsReport report( List records, Map rootWritten) { return new ScopeDiagnosticsReport(records, new ArrayList<>(), rootWritten); diff --git a/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeRecordTest.java b/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeRecordTest.java index 7ad0112a432..b3f1b0b4140 100644 --- a/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeRecordTest.java +++ b/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeRecordTest.java @@ -60,7 +60,7 @@ void openWithoutCloseIsNeverClosed() { ScopeDiagnosticsReport report = report(s); assertEquals(1, report.neverClosedScopeCount()); - assertTrue(report.hasProblems()); // never-closed is a genuine bug + assertTrue(report.hasProblems()); } @Test From 6ddaa3235ace3f9ad6ec773dd92aaebb9fac5766 Mon Sep 17 00:00:00 2001 From: Andrea Marziali Date: Fri, 11 Sep 2026 11:13:17 +0200 Subject: [PATCH 03/35] use the right named --- .../agent/test/scopediag/ScopeContinuationTransformer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeContinuationTransformer.java b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeContinuationTransformer.java index 3bb99277a6a..7862eb79d6f 100644 --- a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeContinuationTransformer.java +++ b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeContinuationTransformer.java @@ -1,7 +1,7 @@ package datadog.trace.agent.test.scopediag; +import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named; import static net.bytebuddy.matcher.ElementMatchers.isMethod; -import static net.bytebuddy.matcher.ElementMatchers.named; import static net.bytebuddy.matcher.ElementMatchers.returns; import static net.bytebuddy.matcher.ElementMatchers.takesArguments; From ce5f43bb3ccd92c632dc261811f3aa9d39f036e8 Mon Sep 17 00:00:00 2001 From: Andrea Marziali Date: Fri, 11 Sep 2026 14:19:56 +0200 Subject: [PATCH 04/35] Add deferred cleanup use case --- .../skills/fix-continuation-leakage/SKILL.md | 3 + .../test/InstrumentationSpecification.groovy | 1 + .../scopediag/ContinuableScopeAdvice.java | 7 -- .../ContinuableScopeManagerAdvice.java | 15 ++++ .../scopediag/ScopeContinuationProbe.java | 10 +++ .../ScopeContinuationTransformer.java | 51 +++++++++++-- .../test/scopediag/ScopeDiagnostics.java | 62 ++++++++++++++-- .../scopediag/ScopeDiagnosticsExtension.java | 1 + .../scopediag/ScopeDiagnosticsReport.java | 19 +++++ .../agent/test/scopediag/ScopeRecord.java | 14 +++- .../test/scopediag/ScopeStackAdvice.java | 15 ++++ .../ScopeDiagnosticsIntegrationTest.java | 71 +++++++++++++++++++ .../agent/test/scopediag/ScopeRecordTest.java | 13 ++++ 13 files changed, 262 insertions(+), 20 deletions(-) create mode 100644 dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ContinuableScopeManagerAdvice.java create mode 100644 dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeStackAdvice.java diff --git a/.agents/skills/fix-continuation-leakage/SKILL.md b/.agents/skills/fix-continuation-leakage/SKILL.md index 961f3d88eb4..d87a570cca6 100644 --- a/.agents/skills/fix-continuation-leakage/SKILL.md +++ b/.agents/skills/fix-continuation-leakage/SKILL.md @@ -38,6 +38,9 @@ resolution, scope, thread, timing, and callsite data needed to find the missing - `DOUBLE_FINISH`: find two owners of the same cleanup. - `ACTIVATE_AFTER_RESOLVE`: find work scheduled after ownership ended. - `LATE_FINISH` / `CLOSE_WRONG_THREAD`: advisory evidence; verify whether ordering is valid. + - `[deferred-cleanup]`: a root iteration scope transferred cleanup to the bounded iteration + cleaner. It may remain open at the test boundary and is not a leak. Do not generalize this to + other `ITERATION` scopes; an unregistered iteration scope must still close normally. 4. Fix ownership where it breaks. Prefer one owner and `try/finally` cleanup across every exit. 5. Rerun the failing test, then its module. diff --git a/dd-java-agent/instrumentation-testing/src/main/groovy/datadog/trace/agent/test/InstrumentationSpecification.groovy b/dd-java-agent/instrumentation-testing/src/main/groovy/datadog/trace/agent/test/InstrumentationSpecification.groovy index 44aa4926b0a..ed16a1f1f11 100644 --- a/dd-java-agent/instrumentation-testing/src/main/groovy/datadog/trace/agent/test/InstrumentationSpecification.groovy +++ b/dd-java-agent/instrumentation-testing/src/main/groovy/datadog/trace/agent/test/InstrumentationSpecification.groovy @@ -559,6 +559,7 @@ abstract class InstrumentationSpecification extends DDSpecification implements A return null } try { + ScopeDiagnostics.awaitQuiescence() ScopeDiagnostics.stop() def report = ScopeDiagnostics.report() if (report.hasFindings()) { diff --git a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ContinuableScopeAdvice.java b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ContinuableScopeAdvice.java index 1bbac24d23a..59b2a3c23b1 100644 --- a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ContinuableScopeAdvice.java +++ b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ContinuableScopeAdvice.java @@ -6,13 +6,6 @@ public final class ContinuableScopeAdvice { private ContinuableScopeAdvice() {} - public static final class AfterActivated { - @Advice.OnMethodExit(suppress = Throwable.class) - public static void exit(@Advice.This Object scope) { - ScopeContinuationProbe.onScopeOpen(scope); - } - } - public static final class OnProperClose { @Advice.OnMethodExit(suppress = Throwable.class) public static void exit(@Advice.This Object scope) { diff --git a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ContinuableScopeManagerAdvice.java b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ContinuableScopeManagerAdvice.java new file mode 100644 index 00000000000..c98cfa301c9 --- /dev/null +++ b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ContinuableScopeManagerAdvice.java @@ -0,0 +1,15 @@ +package datadog.trace.agent.test.scopediag; + +import net.bytebuddy.asm.Advice; + +/** Test-only advice for scopes owned by the iteration cleaner. */ +public final class ContinuableScopeManagerAdvice { + private ContinuableScopeManagerAdvice() {} + + public static final class ScheduleRootIterationCleanup { + @Advice.OnMethodExit(suppress = Throwable.class) + public static void exit(@Advice.Argument(1) Object scope) { + ScopeContinuationProbe.onDeferredScopeCleanup(scope); + } + } +} diff --git a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeContinuationProbe.java b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeContinuationProbe.java index 18ffe219f28..4cd5ef08003 100644 --- a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeContinuationProbe.java +++ b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeContinuationProbe.java @@ -134,6 +134,16 @@ public static void onScopeClose(Object scope) { } } + public static void onDeferredScopeCleanup(Object scope) { + if (!recording) { + return; + } + try { + ScopeDiagnostics.recordDeferredScopeCleanup(scope); + } catch (Throwable ignored) { + } + } + /** Records an out-of-order close when the internal stack can be inspected. */ public static void onScopeClosing(Object scope) { if (!recording) { diff --git a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeContinuationTransformer.java b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeContinuationTransformer.java index 7862eb79d6f..f00a942eb2f 100644 --- a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeContinuationTransformer.java +++ b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeContinuationTransformer.java @@ -3,6 +3,7 @@ import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named; import static net.bytebuddy.matcher.ElementMatchers.isMethod; import static net.bytebuddy.matcher.ElementMatchers.returns; +import static net.bytebuddy.matcher.ElementMatchers.takesArgument; import static net.bytebuddy.matcher.ElementMatchers.takesArguments; import java.lang.instrument.Instrumentation; @@ -24,6 +25,16 @@ static synchronized void install() { if (transformer != null) { return; } + try { + // Related core types can otherwise load this target reentrantly while they are transformed. + Class.forName( + "datadog.trace.core.scopemanager.ScopeContinuation", + false, + ScopeContinuationTransformer.class.getClassLoader()); + } catch (ClassNotFoundException missingCoreTracer) { + throw new IllegalStateException( + "Scope continuation diagnostics require dd-trace-core", missingCoreTracer); + } Instrumentation instrumentation = ByteBuddyAgent.getInstrumentation(); transformer = new AgentBuilder.Default() @@ -74,13 +85,6 @@ static synchronized void install() { .transform( (builder, type, classLoader, module, pd) -> builder - .visit( - Advice.to(ContinuableScopeAdvice.AfterActivated.class) - .on( - isMethod() - .and(named("afterActivated")) - .and(takesArguments(0)) - .and(returns(void.class)))) .visit( Advice.to(ContinuableScopeAdvice.OnProperClose.class) .on( @@ -95,6 +99,39 @@ static synchronized void install() { .and(named("close")) .and(takesArguments(0)) .and(returns(void.class))))) + .type(named("datadog.trace.core.scopemanager.ScopeStack")) + .transform( + (builder, type, classLoader, module, pd) -> + builder.visit( + Advice.to(ScopeStackAdvice.Push.class) + .on( + isMethod() + .and(named("push")) + .and(takesArguments(1)) + .and( + takesArgument( + 0, + named( + "datadog.trace.core.scopemanager.ContinuableScope"))) + .and(returns(void.class))))) + .type(named("datadog.trace.core.scopemanager.ContinuableScopeManager")) + .transform( + (builder, type, classLoader, module, pd) -> + builder.visit( + Advice.to(ContinuableScopeManagerAdvice.ScheduleRootIterationCleanup.class) + .on( + isMethod() + .and(named("scheduleRootIterationScopeCleanup")) + .and(takesArguments(2)) + .and( + takesArgument( + 0, named("datadog.trace.core.scopemanager.ScopeStack"))) + .and( + takesArgument( + 1, + named( + "datadog.trace.core.scopemanager.ContinuableScope"))) + .and(returns(void.class))))) .installOn(instrumentation); } } diff --git a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeDiagnostics.java b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeDiagnostics.java index 3c135721c22..c95582b545f 100644 --- a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeDiagnostics.java +++ b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeDiagnostics.java @@ -12,6 +12,7 @@ /** Records test-time scope and continuation lifecycles and reports leaks. */ public final class ScopeDiagnostics { private static final int DEFAULT_MAX_FRAMES = 6; + private static final long DEFAULT_QUIESCENCE_TIMEOUT_MILLIS = 250; private static final ScopeDiagnostics INSTANCE = new ScopeDiagnostics(); @@ -20,6 +21,8 @@ public final class ScopeDiagnostics { private final Map records = new IdentityHashMap<>(); private final Map scopeRecords = new IdentityHashMap<>(); + private final Set deferredCleanupScopes = + Collections.newSetFromMap(new IdentityHashMap()); private final Map rootWrittenNanos = new HashMap<>(); private final Set resolved = Collections.newSetFromMap(new IdentityHashMap()); @@ -71,10 +74,28 @@ public static void reset() { /** Returns an immutable snapshot of the events recorded so far. */ public static ScopeDiagnosticsReport report() { synchronized (INSTANCE.lifecycleLock) { - return new ScopeDiagnosticsReport( - new ArrayList<>(INSTANCE.records.values()), - new ArrayList<>(INSTANCE.scopeRecords.values()), - new HashMap<>(INSTANCE.rootWrittenNanos)); + return INSTANCE.snapshot(); + } + } + + /** Gives asynchronous cleanup a bounded opportunity to resolve lifecycles still in flight. */ + public static void awaitQuiescence() { + long deadline = System.nanoTime() + DEFAULT_QUIESCENCE_TIMEOUT_MILLIS * 1_000_000L; + synchronized (INSTANCE.lifecycleLock) { + while (INSTANCE.recording && INSTANCE.snapshot().hasIncompleteLifecycles()) { + long remaining = deadline - System.nanoTime(); + if (remaining <= 0) { + return; + } + try { + long millis = remaining / 1_000_000L; + int nanos = (int) (remaining % 1_000_000L); + INSTANCE.lifecycleLock.wait(millis, nanos); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + return; + } + } } } @@ -109,12 +130,20 @@ public static boolean isEnabled(TrackScopeContinuations config) { private void clear() { records.clear(); scopeRecords.clear(); + deferredCleanupScopes.clear(); rootWrittenNanos.clear(); resolved.clear(); seq = 0; scopeSeq = 0; } + private ScopeDiagnosticsReport snapshot() { + return new ScopeDiagnosticsReport( + new ArrayList<>(records.values()), + new ArrayList<>(scopeRecords.values()), + new HashMap<>(rootWrittenNanos)); + } + private static final StackTraceElement[] NO_STACK = new StackTraceElement[0]; private ScopeEvent event(ScopeEvent.Type type) { @@ -127,7 +156,9 @@ private ScopeEvent event(ScopeEvent.Type type, long nanos) { StackFilter filter = stackFilter; StackTraceElement[] stack = filter.maxFrames() <= 0 ? NO_STACK : filter.filter(new Throwable().getStackTrace()); - return new ScopeEvent(type, Thread.currentThread().getName(), nanos, stack); + ScopeEvent event = new ScopeEvent(type, Thread.currentThread().getName(), nanos, stack); + lifecycleLock.notifyAll(); + return event; } static void recordCapture( @@ -208,6 +239,14 @@ static void recordScopeCloseWrongThread(Object scope) { } } + static void recordDeferredScopeCleanup(Object scope) { + synchronized (INSTANCE.lifecycleLock) { + if (INSTANCE.recording) { + INSTANCE.listener.onDeferredScopeCleanup(scope); + } + } + } + private final class Listener { void onCapture( ContextContinuation id, DDTraceId traceId, long spanId, String spanName, byte source) { @@ -287,6 +326,7 @@ void onScopeOpen( spanName, source, continuationSeq, + deferredCleanupScopes.remove(scope), event(ScopeEvent.Type.SCOPE_OPEN))); if (owner != null) { owner.linkScope(s); @@ -315,6 +355,18 @@ void onScopeCloseWrongThread(Object scope) { } } + void onDeferredScopeCleanup(Object scope) { + try { + ScopeRecord record = scopeRecords.get(scope); + if (record != null) { + record.markDeferredCleanup(); + } else { + deferredCleanupScopes.add(scope); + } + } catch (Throwable ignored) { + } + } + /** Returns the record for an id, creating an orphan record if capture was not observed. */ private ContinuationRecord recordFor( ContextContinuation id, DDTraceId traceId, long spanId, String spanName, byte source) { diff --git a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsExtension.java b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsExtension.java index ef82829a6c0..7ba889284d0 100644 --- a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsExtension.java +++ b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsExtension.java @@ -28,6 +28,7 @@ public void afterEach(ExtensionContext context) { return; } try { + ScopeDiagnostics.awaitQuiescence(); ScopeDiagnostics.stop(); ScopeDiagnosticsReport report = ScopeDiagnostics.report(); if (report.hasFindings()) { diff --git a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsReport.java b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsReport.java index a9d83583bcf..d7e3c56595a 100644 --- a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsReport.java +++ b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsReport.java @@ -113,6 +113,20 @@ public int closeWrongThreadCount() { return countWith(scopeFailures, Failure.CLOSE_WRONG_THREAD); } + public int deferredCleanupScopeCount() { + int count = 0; + for (ScopeRecord scope : scopes) { + if (scope.deferredCleanup() && !scope.closed()) { + count++; + } + } + return count; + } + + boolean hasIncompleteLifecycles() { + return leakCount() > 0 || neverClosedScopeCount() > 0; + } + private static int countWith(Map> findings, Failure failure) { int n = 0; for (EnumSet f : findings.values()) { @@ -153,6 +167,8 @@ private void appendHeader(StringBuilder sb, String title) { .append(" activate-after-resolve | scopes: ") .append(neverClosedScopeCount()) .append(" never-closed, ") + .append(deferredCleanupScopeCount()) + .append(" deferred, ") .append(closeWrongThreadCount()) .append(" wrong-thread)\n"); } @@ -315,6 +331,9 @@ private void appendScopeLine(StringBuilder sb, String indent, ScopeRecord scope) if (scope.threadHandoff()) { sb.append(" [handoff]"); } + if (scope.deferredCleanup() && !scope.closed()) { + sb.append(" [deferred-cleanup]"); + } if (!failures.isEmpty()) { sb.append(' ').append(failures); } diff --git a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeRecord.java b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeRecord.java index 825a3444274..6d1e1d66116 100644 --- a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeRecord.java +++ b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeRecord.java @@ -18,6 +18,7 @@ public final class ScopeRecord { private final ScopeEvent open; private ScopeEvent close; + private boolean deferredCleanup; private final List wrongThreadCloses = new ArrayList<>(0); ScopeRecord( @@ -27,6 +28,7 @@ public final class ScopeRecord { String spanName, byte source, Long continuationSeq, + boolean deferredCleanup, ScopeEvent open) { this.seq = seq; this.traceId = traceId; @@ -34,6 +36,7 @@ public final class ScopeRecord { this.spanName = spanName; this.source = source; this.continuationSeq = continuationSeq; + this.deferredCleanup = deferredCleanup; this.open = open; } @@ -56,6 +59,7 @@ synchronized ScopeRecord snapshot() { spanName, source, continuationSeq, + deferredCleanup, open == null ? null : open.snapshot()); copy.close = close == null ? null : close.snapshot(); for (ScopeEvent event : wrongThreadCloses) { @@ -80,6 +84,14 @@ public synchronized boolean closed() { return close != null; } + synchronized void markDeferredCleanup() { + deferredCleanup = true; + } + + public synchronized boolean deferredCleanup() { + return deferredCleanup; + } + /** {@code true} when the scope was opened and closed on different threads. */ public synchronized boolean threadHandoff() { return open != null && close != null && !open.threadName.equals(close.threadName); @@ -95,7 +107,7 @@ public synchronized Long activeDurationNanos() { public synchronized EnumSet failures() { EnumSet failures = EnumSet.noneOf(Failure.class); - if (open != null && close == null) { + if (open != null && close == null && !deferredCleanup) { failures.add(Failure.NEVER_CLOSED); } if (!wrongThreadCloses.isEmpty()) { diff --git a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeStackAdvice.java b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeStackAdvice.java new file mode 100644 index 00000000000..1b6bef31bae --- /dev/null +++ b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeStackAdvice.java @@ -0,0 +1,15 @@ +package datadog.trace.agent.test.scopediag; + +import net.bytebuddy.asm.Advice; + +/** Test-only advice for scopes added to the active stack. */ +public final class ScopeStackAdvice { + private ScopeStackAdvice() {} + + public static final class Push { + @Advice.OnMethodEnter(suppress = Throwable.class) + public static void enter(@Advice.Argument(0) Object scope) { + ScopeContinuationProbe.onScopeOpen(scope); + } + } +} diff --git a/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsIntegrationTest.java b/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsIntegrationTest.java index 33b050542d6..89a4af142bd 100644 --- a/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsIntegrationTest.java +++ b/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsIntegrationTest.java @@ -5,6 +5,7 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; +import datadog.context.Context; import datadog.context.ContextContinuation; import datadog.context.ContextScope; import datadog.trace.bootstrap.instrumentation.api.AgentScope; @@ -12,6 +13,8 @@ import datadog.trace.common.writer.ListWriter; import datadog.trace.core.CoreTracer; import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; @@ -117,6 +120,74 @@ void scopeLifetimeRecordedAndLinkedToContinuation() { assertEquals(Long.valueOf(report.records().get(0).seq), linked.continuationSeq); } + @Test + void swappedContextDoesNotCreateCloseOwnedScope() { + tracer = CoreTracer.builder().writer(new ListWriter()).strictTraceWrites(false).build(); + + ScopeDiagnostics.startRecording(); + + AgentSpan span = tracer.startSpan("test", "op"); + Context previous = tracer.swap(span); + previous.swap(); + span.finish(); + + ScopeDiagnosticsReport report = ScopeDiagnostics.report(); + + assertEquals(0, report.neverClosedScopeCount()); + assertTrue(report.scopeRecords().isEmpty(), "stack swaps do not own scope closure"); + } + + @Test + void rootIterationScopeDelegatesCleanup() { + tracer = CoreTracer.builder().writer(new ListWriter()).strictTraceWrites(false).build(); + + ScopeDiagnostics.startRecording(); + + AgentSpan span = tracer.startSpan("test", "iteration"); + tracer.activateNext(span); + + ScopeDiagnosticsReport report = ScopeDiagnostics.report(); + assertEquals(1, report.deferredCleanupScopeCount()); + assertEquals(0, report.neverClosedScopeCount()); + assertFalse(report.hasProblems()); + + tracer.closePrevious(true); + } + + @Test + void waitsForAsynchronousScopeCleanup() throws Exception { + tracer = CoreTracer.builder().writer(new ListWriter()).strictTraceWrites(false).build(); + + ScopeDiagnostics.startRecording(); + + AgentSpan span = tracer.startSpan("test", "op"); + ContextContinuation continuation = tracer.capture(span); + CountDownLatch scopeOpened = new CountDownLatch(1); + CountDownLatch closeScope = new CountDownLatch(1); + Thread worker = + new Thread( + () -> { + try (ContextScope ignored = continuation.resume()) { + scopeOpened.countDown(); + closeScope.await(); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + } + }); + worker.start(); + + assertTrue(scopeOpened.await(5, TimeUnit.SECONDS)); + assertTrue(ScopeDiagnostics.report().hasIncompleteLifecycles()); + closeScope.countDown(); + ScopeDiagnostics.awaitQuiescence(); + worker.join(); + + ScopeDiagnosticsReport report = ScopeDiagnostics.report(); + assertFalse(report.hasIncompleteLifecycles()); + assertFalse(report.hasProblems()); + span.finish(); + } + @Test void neverClosedScopeIsFlagged() { tracer = CoreTracer.builder().writer(new ListWriter()).strictTraceWrites(false).build(); diff --git a/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeRecordTest.java b/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeRecordTest.java index b3f1b0b4140..6af4a059c86 100644 --- a/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeRecordTest.java +++ b/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeRecordTest.java @@ -28,6 +28,7 @@ private static ScopeRecord scope(long seq, Long continuationSeq, String openThre "op", (byte) 0, continuationSeq, + false, event(ScopeEvent.Type.SCOPE_OPEN, openThread, nanos)); } @@ -63,6 +64,18 @@ void openWithoutCloseIsNeverClosed() { assertTrue(report.hasProblems()); } + @Test + void deferredCleanupIsNotALeak() { + ScopeRecord s = scope(0, null, "main", 1000); + s.markDeferredCleanup(); + + ScopeDiagnosticsReport report = report(s); + assertFalse(s.failures().contains(Failure.NEVER_CLOSED)); + assertEquals(1, report.deferredCleanupScopeCount()); + assertEquals(0, report.neverClosedScopeCount()); + assertFalse(report.hasProblems()); + } + @Test void openAndCloseOnDifferentThreadsIsHandoff() { ScopeRecord s = scope(0, null, "main", 1000); From 80d82fcbbef7ba6abb6d414d5517b86bf4acf438 Mon Sep 17 00:00:00 2001 From: Andrea Marziali Date: Wed, 16 Sep 2026 09:57:39 +0200 Subject: [PATCH 05/35] Fix diagnostic snapshot synchronization --- .../trace/agent/test/scopediag/ContinuationRecord.java | 4 +++- .../java/datadog/trace/agent/test/scopediag/ScopeRecord.java | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ContinuationRecord.java b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ContinuationRecord.java index 845218efa2f..208de65c19e 100644 --- a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ContinuationRecord.java +++ b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ContinuationRecord.java @@ -77,7 +77,9 @@ synchronized ContinuationRecord snapshot() { for (ScopeEvent event : failedActivations) { copy.failedActivations.add(event.snapshot()); } - copy.terminal = terminal == null ? null : terminal.snapshot(); + if (terminal != null) { + copy.setTerminalOrExtra(terminal.snapshot()); + } for (ScopeEvent event : extraTerminals) { copy.extraTerminals.add(event.snapshot()); } diff --git a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeRecord.java b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeRecord.java index 6d1e1d66116..ec7f72b4117 100644 --- a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeRecord.java +++ b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeRecord.java @@ -61,7 +61,9 @@ synchronized ScopeRecord snapshot() { continuationSeq, deferredCleanup, open == null ? null : open.snapshot()); - copy.close = close == null ? null : close.snapshot(); + if (close != null) { + copy.setClose(close.snapshot()); + } for (ScopeEvent event : wrongThreadCloses) { copy.wrongThreadCloses.add(event.snapshot()); } From e7a4fe1c0e7a906bd176cd81969bed5acc3230bd Mon Sep 17 00:00:00 2001 From: Andrea Marziali Date: Wed, 16 Sep 2026 12:20:15 +0200 Subject: [PATCH 06/35] Remove strictTraceWrite override --- .../src/test/groovy/CouchbaseAsyncClientTest.groovy | 6 ------ .../groovy/springdata/CouchbaseSpringRepositoryTest.groovy | 6 ------ .../src/test/groovy/CassandraClientTest.groovy | 6 ------ .../src/test/groovy/CassandraClientTest.groovy | 6 ------ .../src/test/groovy/CassandraClientTest.groovy | 6 ------ .../grizzly-http-2.3.20/src/test/groovy/GrizzlyTest.groovy | 6 ------ .../src/test/groovy/ProcedureCallTest.groovy | 7 ------- .../src/test/groovy/timer/TimerTaskContinuationTest.groovy | 5 ----- .../groovy/KafkaClientCustomPropagationConfigTest.groovy | 6 ------ .../src/test/groovy/KafkaReactorForkedTest.groovy | 5 ----- .../groovy/KafkaClientCustomPropagationConfigTest.groovy | 6 ------ .../src/test/groovy/KafkaClientTestBase.groovy | 6 ------ .../netty-4.0/src/test/groovy/Netty40ClientTest.groovy | 6 ------ .../okhttp-2.2/src/test/groovy/OkHttp2AsyncTest.groovy | 6 ------ .../okhttp/okhttp-2.2/src/test/groovy/OkHttp2Test.groovy | 6 ------ .../okhttp/okhttp-3.0/src/test/groovy/OkHttp3Test.groovy | 6 ------ .../latestDepTest/groovy/ScalaInstrumentationTest.groovy | 6 ------ .../src/test/groovy/ScalaInstrumentationTest.groovy | 6 ------ .../groovy/test/boot/SpringBootBasedTest.groovy | 6 ------ .../src/test/groovy/test/boot/SpringBootBasedTest.groovy | 6 ------ .../src/test/groovy/test/dynamic/DynamicRoutingTest.groovy | 6 ------ .../src/test/groovy/test/filter/ServletFilterTest.groovy | 6 ------ .../springweb6/boot/SpringBootBasedTest.groovy | 6 ------ .../src/test/groovy/VertxSqlClientForkedTest.groovy | 7 ------- .../src/test/groovy/VertxSqlClientForkedTest.groovy | 7 ------- .../src/test/groovy/VertxRedisTestBase.groovy | 5 ----- .../client/VertxRxCircuitBreakerWebClientForkedTest.groovy | 5 ----- .../test/groovy/client/VertxRxWebClientForkedTest.groovy | 5 ----- .../test/groovy/client/VertxHttpClientForkedTest.groovy | 5 ----- .../groovy/client/VertxHttpClientForkedTest.groovy | 5 ----- .../test/groovy/client/VertxHttpClientForkedTest.groovy | 5 ----- .../test/groovy/client/VertxHttpClientForkedTest.groovy | 5 ----- 32 files changed, 186 deletions(-) diff --git a/dd-java-agent/instrumentation/couchbase/couchbase-2.0/src/test/groovy/CouchbaseAsyncClientTest.groovy b/dd-java-agent/instrumentation/couchbase/couchbase-2.0/src/test/groovy/CouchbaseAsyncClientTest.groovy index 3042c820235..d2d71aa655a 100644 --- a/dd-java-agent/instrumentation/couchbase/couchbase-2.0/src/test/groovy/CouchbaseAsyncClientTest.groovy +++ b/dd-java-agent/instrumentation/couchbase/couchbase-2.0/src/test/groovy/CouchbaseAsyncClientTest.groovy @@ -16,12 +16,6 @@ import static datadog.trace.agent.test.utils.TraceUtils.runUnderTrace abstract class CouchbaseAsyncClientTest extends AbstractCouchbaseTest { static final int TIMEOUT = 30 - @Override - boolean useStrictTraceWrites() { - // Async spans often finish out of order, so allow buffering. - return false - } - def "test hasBucket #type"() { setup: diff --git a/dd-java-agent/instrumentation/couchbase/couchbase-2.0/src/test/groovy/springdata/CouchbaseSpringRepositoryTest.groovy b/dd-java-agent/instrumentation/couchbase/couchbase-2.0/src/test/groovy/springdata/CouchbaseSpringRepositoryTest.groovy index b4f7dbc1653..1d7e31fc12b 100644 --- a/dd-java-agent/instrumentation/couchbase/couchbase-2.0/src/test/groovy/springdata/CouchbaseSpringRepositoryTest.groovy +++ b/dd-java-agent/instrumentation/couchbase/couchbase-2.0/src/test/groovy/springdata/CouchbaseSpringRepositoryTest.groovy @@ -37,12 +37,6 @@ abstract class CouchbaseSpringRepositoryTest extends AbstractCouchbaseTest { } } - @Override - boolean useStrictTraceWrites() { - // TODO fix this by making sure that spans get closed properly - return false - } - @Shared ConfigurableApplicationContext applicationContext @Shared diff --git a/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-3.0/src/test/groovy/CassandraClientTest.groovy b/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-3.0/src/test/groovy/CassandraClientTest.groovy index d4e0ac21065..8afb4029bb3 100644 --- a/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-3.0/src/test/groovy/CassandraClientTest.groovy +++ b/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-3.0/src/test/groovy/CassandraClientTest.groovy @@ -23,12 +23,6 @@ import java.util.concurrent.TimeUnit abstract class CassandraClientTest extends VersionedNamingTestBase { private static final int ASYNC_TIMEOUT_MS = 5000 - @Override - boolean useStrictTraceWrites() { - // TODO fix this by making sure that spans get closed properly - return false - } - @Shared Cluster cluster diff --git a/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-3.8/src/test/groovy/CassandraClientTest.groovy b/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-3.8/src/test/groovy/CassandraClientTest.groovy index d4e0ac21065..8afb4029bb3 100644 --- a/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-3.8/src/test/groovy/CassandraClientTest.groovy +++ b/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-3.8/src/test/groovy/CassandraClientTest.groovy @@ -23,12 +23,6 @@ import java.util.concurrent.TimeUnit abstract class CassandraClientTest extends VersionedNamingTestBase { private static final int ASYNC_TIMEOUT_MS = 5000 - @Override - boolean useStrictTraceWrites() { - // TODO fix this by making sure that spans get closed properly - return false - } - @Shared Cluster cluster diff --git a/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-4.0/src/test/groovy/CassandraClientTest.groovy b/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-4.0/src/test/groovy/CassandraClientTest.groovy index ba3aadb8773..5172c669524 100644 --- a/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-4.0/src/test/groovy/CassandraClientTest.groovy +++ b/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-4.0/src/test/groovy/CassandraClientTest.groovy @@ -28,12 +28,6 @@ import static datadog.trace.api.config.TraceInstrumentationConfig.DB_CLIENT_HOST abstract class CassandraClientTest extends VersionedNamingTestBase { private static final int TIMEOUT = 30 - @Override - boolean useStrictTraceWrites() { - // TODO fix this by making sure that spans get closed properly - return false - } - @Shared int port diff --git a/dd-java-agent/instrumentation/grizzly/grizzly-http-2.3.20/src/test/groovy/GrizzlyTest.groovy b/dd-java-agent/instrumentation/grizzly/grizzly-http-2.3.20/src/test/groovy/GrizzlyTest.groovy index 18b4ce8aec4..c6a7459834b 100644 --- a/dd-java-agent/instrumentation/grizzly/grizzly-http-2.3.20/src/test/groovy/GrizzlyTest.groovy +++ b/dd-java-agent/instrumentation/grizzly/grizzly-http-2.3.20/src/test/groovy/GrizzlyTest.groovy @@ -4,12 +4,6 @@ import datadog.trace.test.util.Flaky import org.glassfish.grizzly.http.server.HttpServer class GrizzlyTest extends HttpServerTest { - @Override - boolean useStrictTraceWrites() { - // FIXME: ASM blocking test fails with "Interaction with TraceSegment after root span has already finished" - false - } - @Override datadog.trace.agent.test.base.HttpServer server() { new GrizzlyServer(resource()) diff --git a/dd-java-agent/instrumentation/hibernate/hibernate-core-4.3/src/test/groovy/ProcedureCallTest.groovy b/dd-java-agent/instrumentation/hibernate/hibernate-core-4.3/src/test/groovy/ProcedureCallTest.groovy index 80acda2aa1e..35495ff5753 100644 --- a/dd-java-agent/instrumentation/hibernate/hibernate-core-4.3/src/test/groovy/ProcedureCallTest.groovy +++ b/dd-java-agent/instrumentation/hibernate/hibernate-core-4.3/src/test/groovy/ProcedureCallTest.groovy @@ -18,12 +18,6 @@ import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.activeSpan class ProcedureCallTest extends InstrumentationSpecification { - @Override - boolean useStrictTraceWrites() { - // TODO fix this by making sure that spans get closed properly - return false - } - @Shared protected SessionFactory sessionFactory @@ -203,4 +197,3 @@ class ProcedureCallTest extends InstrumentationSpecification { } } } - diff --git a/dd-java-agent/instrumentation/java/java-concurrent/java-concurrent-1.8/src/test/groovy/timer/TimerTaskContinuationTest.groovy b/dd-java-agent/instrumentation/java/java-concurrent/java-concurrent-1.8/src/test/groovy/timer/TimerTaskContinuationTest.groovy index 9545dd9ac61..93f2e841668 100644 --- a/dd-java-agent/instrumentation/java/java-concurrent/java-concurrent-1.8/src/test/groovy/timer/TimerTaskContinuationTest.groovy +++ b/dd-java-agent/instrumentation/java/java-concurrent/java-concurrent-1.8/src/test/groovy/timer/TimerTaskContinuationTest.groovy @@ -21,11 +21,6 @@ class TimerTaskContinuationTest extends InstrumentationSpecification { CountDownLatch runLatch = new CountDownLatch(1) - @Override - boolean useStrictTraceWrites() { - false - } - @Override def setup() { timerTask = new TimerTask() { diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/test/groovy/KafkaClientCustomPropagationConfigTest.groovy b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/test/groovy/KafkaClientCustomPropagationConfigTest.groovy index 98460601e5c..2f25f947027 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/test/groovy/KafkaClientCustomPropagationConfigTest.groovy +++ b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/test/groovy/KafkaClientCustomPropagationConfigTest.groovy @@ -37,12 +37,6 @@ class KafkaClientCustomPropagationConfigTest extends InstrumentationSpecificatio ] } - @Override - boolean useStrictTraceWrites() { - // TODO fix this by making sure that spans get closed properly - return false - } - KafkaEmbedded embeddedKafka def setup() { diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/test/groovy/KafkaReactorForkedTest.groovy b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/test/groovy/KafkaReactorForkedTest.groovy index 08ec9fb0513..e163e9c32d7 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/test/groovy/KafkaReactorForkedTest.groovy +++ b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/test/groovy/KafkaReactorForkedTest.groovy @@ -29,11 +29,6 @@ class KafkaReactorForkedTest extends InstrumentationSpecification { // create 4 partitions for more parallelism KafkaEmbedded embeddedKafka - @Override - boolean useStrictTraceWrites() { - false - } - def setup() { embeddedKafka = new KafkaEmbedded(1, true, 4, KafkaClientTestBase.SHARED_TOPIC) embeddedKafka.before() diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/test/groovy/KafkaClientCustomPropagationConfigTest.groovy b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/test/groovy/KafkaClientCustomPropagationConfigTest.groovy index 0dec7bbe6bc..2a914f09647 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/test/groovy/KafkaClientCustomPropagationConfigTest.groovy +++ b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/test/groovy/KafkaClientCustomPropagationConfigTest.groovy @@ -51,12 +51,6 @@ class KafkaClientCustomPropagationConfigTest extends InstrumentationSpecificatio ] } - @Override - boolean useStrictTraceWrites() { - // TODO fix this by making sure that spans get closed properly - return false - } - @Override void configurePreAgent() { diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/test/groovy/KafkaClientTestBase.groovy b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/test/groovy/KafkaClientTestBase.groovy index 579532664bd..f4ae23b6d0e 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/test/groovy/KafkaClientTestBase.groovy +++ b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/test/groovy/KafkaClientTestBase.groovy @@ -64,12 +64,6 @@ abstract class KafkaClientTestBase extends VersionedNamingTestBase { embeddedKafka.destroy() } - @Override - boolean useStrictTraceWrites() { - // TODO fix this by making sure that spans get closed properly - return false - } - @Override void configurePreAgent() { super.configurePreAgent() diff --git a/dd-java-agent/instrumentation/netty/netty-4.0/src/test/groovy/Netty40ClientTest.groovy b/dd-java-agent/instrumentation/netty/netty-4.0/src/test/groovy/Netty40ClientTest.groovy index 898454d6267..6fdf5dd48d2 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.0/src/test/groovy/Netty40ClientTest.groovy +++ b/dd-java-agent/instrumentation/netty/netty-4.0/src/test/groovy/Netty40ClientTest.groovy @@ -27,12 +27,6 @@ import static org.asynchttpclient.Dsl.asyncHttpClient abstract class Netty40ClientTest extends HttpClientTest { - @Override - boolean useStrictTraceWrites() { - // NettyPromiseInstrumentation results in unfinished continuations. - return false - } - def clientConfig = DefaultAsyncHttpClientConfig.Builder.newInstance() .setConnectTimeout(CONNECT_TIMEOUT_MS) .setRequestTimeout(READ_TIMEOUT_MS) diff --git a/dd-java-agent/instrumentation/okhttp/okhttp-2.2/src/test/groovy/OkHttp2AsyncTest.groovy b/dd-java-agent/instrumentation/okhttp/okhttp-2.2/src/test/groovy/OkHttp2AsyncTest.groovy index 7d0299581b6..273289580de 100644 --- a/dd-java-agent/instrumentation/okhttp/okhttp-2.2/src/test/groovy/OkHttp2AsyncTest.groovy +++ b/dd-java-agent/instrumentation/okhttp/okhttp-2.2/src/test/groovy/OkHttp2AsyncTest.groovy @@ -11,12 +11,6 @@ import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.noopSpan import static java.util.concurrent.TimeUnit.SECONDS abstract class OkHttp2AsyncTest extends OkHttp2Test { - @Override - boolean useStrictTraceWrites() { - // TODO fix this by making sure that spans get closed properly - return false - } - @Override int doRequest(String method, URI uri, Map headers, String body, Closure callback) { final contentType = headers.remove("Content-Type") diff --git a/dd-java-agent/instrumentation/okhttp/okhttp-2.2/src/test/groovy/OkHttp2Test.groovy b/dd-java-agent/instrumentation/okhttp/okhttp-2.2/src/test/groovy/OkHttp2Test.groovy index b2e24739fb3..3d64f4fc6b7 100644 --- a/dd-java-agent/instrumentation/okhttp/okhttp-2.2/src/test/groovy/OkHttp2Test.groovy +++ b/dd-java-agent/instrumentation/okhttp/okhttp-2.2/src/test/groovy/OkHttp2Test.groovy @@ -9,12 +9,6 @@ import spock.lang.Timeout import java.util.concurrent.TimeUnit abstract class OkHttp2Test extends HttpClientTest { - @Override - boolean useStrictTraceWrites() { - // TODO fix this by making sure that spans get closed properly - return false - } - @Shared def client = new OkHttpClient() diff --git a/dd-java-agent/instrumentation/okhttp/okhttp-3.0/src/test/groovy/OkHttp3Test.groovy b/dd-java-agent/instrumentation/okhttp/okhttp-3.0/src/test/groovy/OkHttp3Test.groovy index 903bf436c56..2aed1c0682a 100644 --- a/dd-java-agent/instrumentation/okhttp/okhttp-3.0/src/test/groovy/OkHttp3Test.groovy +++ b/dd-java-agent/instrumentation/okhttp/okhttp-3.0/src/test/groovy/OkHttp3Test.groovy @@ -32,12 +32,6 @@ abstract class OkHttp3Test extends HttpClientTest { true } - @Override - boolean useStrictTraceWrites() { - // TODO fix this by making sure that spans get closed properly - return false - } - OkHttpClient getClient() { new OkHttpClient.Builder() .connectTimeout(CONNECT_TIMEOUT_MS, TimeUnit.MILLISECONDS) diff --git a/dd-java-agent/instrumentation/scala/scala-concurrent-2.8/src/latestDepTest/groovy/ScalaInstrumentationTest.groovy b/dd-java-agent/instrumentation/scala/scala-concurrent-2.8/src/latestDepTest/groovy/ScalaInstrumentationTest.groovy index dfc848ec74e..92b49bd4a33 100644 --- a/dd-java-agent/instrumentation/scala/scala-concurrent-2.8/src/latestDepTest/groovy/ScalaInstrumentationTest.groovy +++ b/dd-java-agent/instrumentation/scala/scala-concurrent-2.8/src/latestDepTest/groovy/ScalaInstrumentationTest.groovy @@ -3,12 +3,6 @@ import datadog.trace.core.DDSpan class ScalaInstrumentationTest extends InstrumentationSpecification { - @Override - boolean useStrictTraceWrites() { - // TODO fix this by making sure that spans get closed properly - return false - } - def "scala futures and callbacks"() { setup: ScalaConcurrentTests scalaTest = new ScalaConcurrentTests() diff --git a/dd-java-agent/instrumentation/scala/scala-concurrent-2.8/src/test/groovy/ScalaInstrumentationTest.groovy b/dd-java-agent/instrumentation/scala/scala-concurrent-2.8/src/test/groovy/ScalaInstrumentationTest.groovy index dfc848ec74e..92b49bd4a33 100644 --- a/dd-java-agent/instrumentation/scala/scala-concurrent-2.8/src/test/groovy/ScalaInstrumentationTest.groovy +++ b/dd-java-agent/instrumentation/scala/scala-concurrent-2.8/src/test/groovy/ScalaInstrumentationTest.groovy @@ -3,12 +3,6 @@ import datadog.trace.core.DDSpan class ScalaInstrumentationTest extends InstrumentationSpecification { - @Override - boolean useStrictTraceWrites() { - // TODO fix this by making sure that spans get closed properly - return false - } - def "scala futures and callbacks"() { setup: ScalaConcurrentTests scalaTest = new ScalaConcurrentTests() diff --git a/dd-java-agent/instrumentation/spring/spring-webmvc/spring-webmvc-3.1/src/latestDepTest/groovy/test/boot/SpringBootBasedTest.groovy b/dd-java-agent/instrumentation/spring/spring-webmvc/spring-webmvc-3.1/src/latestDepTest/groovy/test/boot/SpringBootBasedTest.groovy index caa5f574334..f4138d3a312 100644 --- a/dd-java-agent/instrumentation/spring/spring-webmvc/spring-webmvc-3.1/src/latestDepTest/groovy/test/boot/SpringBootBasedTest.groovy +++ b/dd-java-agent/instrumentation/spring/spring-webmvc/spring-webmvc-3.1/src/latestDepTest/groovy/test/boot/SpringBootBasedTest.groovy @@ -38,12 +38,6 @@ import static datadog.trace.agent.test.base.HttpServerTest.ServerEndpoint.WEBSOC class SpringBootBasedTest extends HttpServerTest { - @Override - boolean useStrictTraceWrites() { - // TODO fix this by making sure that spans get closed properly - return false - } - Map extraServerTags = [:] SpringApplication application() { diff --git a/dd-java-agent/instrumentation/spring/spring-webmvc/spring-webmvc-3.1/src/test/groovy/test/boot/SpringBootBasedTest.groovy b/dd-java-agent/instrumentation/spring/spring-webmvc/spring-webmvc-3.1/src/test/groovy/test/boot/SpringBootBasedTest.groovy index 89b0389c3da..37703a57ce1 100644 --- a/dd-java-agent/instrumentation/spring/spring-webmvc/spring-webmvc-3.1/src/test/groovy/test/boot/SpringBootBasedTest.groovy +++ b/dd-java-agent/instrumentation/spring/spring-webmvc/spring-webmvc-3.1/src/test/groovy/test/boot/SpringBootBasedTest.groovy @@ -42,12 +42,6 @@ import static datadog.trace.agent.test.base.HttpServerTest.ServerEndpoint.WEBSOC class SpringBootBasedTest extends HttpServerTest { - @Override - boolean useStrictTraceWrites() { - // TODO fix this by making sure that spans get closed properly - return false - } - Map extraServerTags = [:] SpringApplication application() { diff --git a/dd-java-agent/instrumentation/spring/spring-webmvc/spring-webmvc-3.1/src/test/groovy/test/dynamic/DynamicRoutingTest.groovy b/dd-java-agent/instrumentation/spring/spring-webmvc/spring-webmvc-3.1/src/test/groovy/test/dynamic/DynamicRoutingTest.groovy index e2ff666182d..3e03d64e38a 100644 --- a/dd-java-agent/instrumentation/spring/spring-webmvc/spring-webmvc-3.1/src/test/groovy/test/dynamic/DynamicRoutingTest.groovy +++ b/dd-java-agent/instrumentation/spring/spring-webmvc/spring-webmvc-3.1/src/test/groovy/test/dynamic/DynamicRoutingTest.groovy @@ -21,12 +21,6 @@ import static java.util.Collections.singletonMap class DynamicRoutingTest extends HttpServerTest { - @Override - boolean useStrictTraceWrites() { - // TODO fix this by making sure that spans get closed properly - return false - } - class SpringBootServer implements HttpServer { def port = 0 def context diff --git a/dd-java-agent/instrumentation/spring/spring-webmvc/spring-webmvc-3.1/src/test/groovy/test/filter/ServletFilterTest.groovy b/dd-java-agent/instrumentation/spring/spring-webmvc/spring-webmvc-3.1/src/test/groovy/test/filter/ServletFilterTest.groovy index abce0764793..c63524d96b3 100644 --- a/dd-java-agent/instrumentation/spring/spring-webmvc/spring-webmvc-3.1/src/test/groovy/test/filter/ServletFilterTest.groovy +++ b/dd-java-agent/instrumentation/spring/spring-webmvc/spring-webmvc-3.1/src/test/groovy/test/filter/ServletFilterTest.groovy @@ -23,12 +23,6 @@ import static java.util.Collections.singletonMap class ServletFilterTest extends HttpServerTest { - @Override - boolean useStrictTraceWrites() { - // TODO fix this by making sure that spans get closed properly - return false - } - class SpringBootServer implements HttpServer { def port = 0 def context diff --git a/dd-java-agent/instrumentation/spring/spring-webmvc/spring-webmvc-6.0/src/test/groovy/datadog/trace/instrumentation/springweb6/boot/SpringBootBasedTest.groovy b/dd-java-agent/instrumentation/spring/spring-webmvc/spring-webmvc-6.0/src/test/groovy/datadog/trace/instrumentation/springweb6/boot/SpringBootBasedTest.groovy index 7bf01c8905f..885831440f5 100644 --- a/dd-java-agent/instrumentation/spring/spring-webmvc/spring-webmvc-6.0/src/test/groovy/datadog/trace/instrumentation/springweb6/boot/SpringBootBasedTest.groovy +++ b/dd-java-agent/instrumentation/spring/spring-webmvc/spring-webmvc-6.0/src/test/groovy/datadog/trace/instrumentation/springweb6/boot/SpringBootBasedTest.groovy @@ -44,12 +44,6 @@ import static datadog.trace.agent.test.base.HttpServerTest.ServerEndpoint.WEBSOC class SpringBootBasedTest extends HttpServerTest { - @Override - boolean useStrictTraceWrites() { - // TODO fix this by making sure that spans get closed properly - return false - } - @Shared def context diff --git a/dd-java-agent/instrumentation/vertx/vertx-mysql-client/vertx-mysql-client-3.9/src/test/groovy/VertxSqlClientForkedTest.groovy b/dd-java-agent/instrumentation/vertx/vertx-mysql-client/vertx-mysql-client-3.9/src/test/groovy/VertxSqlClientForkedTest.groovy index 19ce31ce76b..9c45b205a0f 100644 --- a/dd-java-agent/instrumentation/vertx/vertx-mysql-client/vertx-mysql-client-3.9/src/test/groovy/VertxSqlClientForkedTest.groovy +++ b/dd-java-agent/instrumentation/vertx/vertx-mysql-client/vertx-mysql-client-3.9/src/test/groovy/VertxSqlClientForkedTest.groovy @@ -37,13 +37,6 @@ class VertxSqlClientForkedTest extends InstrumentationSpecification { @Shared def vertx = Vertx.vertx(new VertxOptions()) - @Override - boolean useStrictTraceWrites() { - // given how the test without parent is done, we cannot guarantee that the handler callback finishes always - // after the sql span since things are going asynchronously. Indeed we should not enforce that unbuffered write. - false - } - def "test #type"() { when: AsyncResult> asyncResult = runUnderTrace("parent") { diff --git a/dd-java-agent/instrumentation/vertx/vertx-mysql-client/vertx-mysql-client-4.0/src/test/groovy/VertxSqlClientForkedTest.groovy b/dd-java-agent/instrumentation/vertx/vertx-mysql-client/vertx-mysql-client-4.0/src/test/groovy/VertxSqlClientForkedTest.groovy index 3e43b95bed0..26919cf46b3 100644 --- a/dd-java-agent/instrumentation/vertx/vertx-mysql-client/vertx-mysql-client-4.0/src/test/groovy/VertxSqlClientForkedTest.groovy +++ b/dd-java-agent/instrumentation/vertx/vertx-mysql-client/vertx-mysql-client-4.0/src/test/groovy/VertxSqlClientForkedTest.groovy @@ -39,13 +39,6 @@ class VertxSqlClientForkedTest extends InstrumentationSpecification { @Shared def vertx = Vertx.vertx(new VertxOptions()) - @Override - boolean useStrictTraceWrites() { - // given how the test without parent is done, we cannot guarantee that the handler callback finishes always - // after the sql span since things are going asynchronously. Indeed we should not enforce that unbuffered write. - false - } - def "test #type"() { when: AsyncResult> asyncResult = runUnderTrace("parent") { diff --git a/dd-java-agent/instrumentation/vertx/vertx-redis-client/vertx-redis-client-3.9/src/test/groovy/VertxRedisTestBase.groovy b/dd-java-agent/instrumentation/vertx/vertx-redis-client/vertx-redis-client-3.9/src/test/groovy/VertxRedisTestBase.groovy index 8c26b18680a..27be952dc19 100644 --- a/dd-java-agent/instrumentation/vertx/vertx-redis-client/vertx-redis-client-3.9/src/test/groovy/VertxRedisTestBase.groovy +++ b/dd-java-agent/instrumentation/vertx/vertx-redis-client/vertx-redis-client-3.9/src/test/groovy/VertxRedisTestBase.groovy @@ -62,11 +62,6 @@ abstract class VertxRedisTestBase extends VersionedNamingTestBase { return "redis.query" } - @Override - boolean useStrictTraceWrites() { - false - } - def setupSpec() { redisServer.start() redis = Redis.createClient(vertx, redisServer.getRedisURI()) diff --git a/dd-java-agent/instrumentation/vertx/vertx-rx-3.5/src/test/groovy/client/VertxRxCircuitBreakerWebClientForkedTest.groovy b/dd-java-agent/instrumentation/vertx/vertx-rx-3.5/src/test/groovy/client/VertxRxCircuitBreakerWebClientForkedTest.groovy index cb8361bffe5..57dc07d602b 100644 --- a/dd-java-agent/instrumentation/vertx/vertx-rx-3.5/src/test/groovy/client/VertxRxCircuitBreakerWebClientForkedTest.groovy +++ b/dd-java-agent/instrumentation/vertx/vertx-rx-3.5/src/test/groovy/client/VertxRxCircuitBreakerWebClientForkedTest.groovy @@ -19,11 +19,6 @@ import java.util.concurrent.CompletableFuture @Timeout(10) class VertxRxCircuitBreakerWebClientForkedTest extends HttpClientTest implements TestingNettyHttpNamingConventions.ClientV0 { - @Override - boolean useStrictTraceWrites() { - return false - } - @AutoCleanup @Shared Vertx vertx = Vertx.vertx(new VertxOptions()) diff --git a/dd-java-agent/instrumentation/vertx/vertx-rx-3.5/src/test/groovy/client/VertxRxWebClientForkedTest.groovy b/dd-java-agent/instrumentation/vertx/vertx-rx-3.5/src/test/groovy/client/VertxRxWebClientForkedTest.groovy index c79d43d1a30..f2fe6e48811 100644 --- a/dd-java-agent/instrumentation/vertx/vertx-rx-3.5/src/test/groovy/client/VertxRxWebClientForkedTest.groovy +++ b/dd-java-agent/instrumentation/vertx/vertx-rx-3.5/src/test/groovy/client/VertxRxWebClientForkedTest.groovy @@ -15,11 +15,6 @@ import spock.lang.Timeout @Timeout(10) class VertxRxWebClientForkedTest extends HttpClientTest implements TestingNettyHttpNamingConventions.ClientV0 { - @Override - boolean useStrictTraceWrites() { - return false - } - @AutoCleanup @Shared Vertx vertx = Vertx.vertx(new VertxOptions()) diff --git a/dd-java-agent/instrumentation/vertx/vertx-web/vertx-web-3.4/src/test/groovy/client/VertxHttpClientForkedTest.groovy b/dd-java-agent/instrumentation/vertx/vertx-web/vertx-web-3.4/src/test/groovy/client/VertxHttpClientForkedTest.groovy index dd308077653..0d059a8320a 100644 --- a/dd-java-agent/instrumentation/vertx/vertx-web/vertx-web-3.4/src/test/groovy/client/VertxHttpClientForkedTest.groovy +++ b/dd-java-agent/instrumentation/vertx/vertx-web/vertx-web-3.4/src/test/groovy/client/VertxHttpClientForkedTest.groovy @@ -16,11 +16,6 @@ import java.util.concurrent.TimeUnit class VertxHttpClientForkedTest extends HttpClientTest implements TestingNettyHttpNamingConventions.ClientV0 { - @Override - boolean useStrictTraceWrites() { - return false - } - @AutoCleanup @Shared def vertx = Vertx.vertx(new VertxOptions()) diff --git a/dd-java-agent/instrumentation/vertx/vertx-web/vertx-web-4.0/src/latestDepTest/groovy/client/VertxHttpClientForkedTest.groovy b/dd-java-agent/instrumentation/vertx/vertx-web/vertx-web-4.0/src/latestDepTest/groovy/client/VertxHttpClientForkedTest.groovy index 18abd3841a6..d8be4179afd 100644 --- a/dd-java-agent/instrumentation/vertx/vertx-web/vertx-web-4.0/src/latestDepTest/groovy/client/VertxHttpClientForkedTest.groovy +++ b/dd-java-agent/instrumentation/vertx/vertx-web/vertx-web-4.0/src/latestDepTest/groovy/client/VertxHttpClientForkedTest.groovy @@ -17,11 +17,6 @@ import java.util.concurrent.CompletableFuture import java.util.concurrent.TimeUnit class VertxHttpClientForkedTest extends HttpClientTest implements TestingNettyHttpNamingConventions.ClientV0 { - @Override - boolean useStrictTraceWrites() { - return false - } - @AutoCleanup @Shared def vertx = Vertx.vertx(new VertxOptions()) diff --git a/dd-java-agent/instrumentation/vertx/vertx-web/vertx-web-4.0/src/test/groovy/client/VertxHttpClientForkedTest.groovy b/dd-java-agent/instrumentation/vertx/vertx-web/vertx-web-4.0/src/test/groovy/client/VertxHttpClientForkedTest.groovy index dd2035ba75f..bda788a52f6 100644 --- a/dd-java-agent/instrumentation/vertx/vertx-web/vertx-web-4.0/src/test/groovy/client/VertxHttpClientForkedTest.groovy +++ b/dd-java-agent/instrumentation/vertx/vertx-web/vertx-web-4.0/src/test/groovy/client/VertxHttpClientForkedTest.groovy @@ -17,11 +17,6 @@ import java.util.concurrent.TimeUnit class VertxHttpClientForkedTest extends HttpClientTest implements TestingNettyHttpNamingConventions.ClientV0 { - @Override - boolean useStrictTraceWrites() { - return false - } - @AutoCleanup @Shared def vertx = Vertx.vertx(new VertxOptions()) diff --git a/dd-java-agent/instrumentation/vertx/vertx-web/vertx-web-5.0/src/test/groovy/client/VertxHttpClientForkedTest.groovy b/dd-java-agent/instrumentation/vertx/vertx-web/vertx-web-5.0/src/test/groovy/client/VertxHttpClientForkedTest.groovy index 0e528befe84..09848373696 100644 --- a/dd-java-agent/instrumentation/vertx/vertx-web/vertx-web-5.0/src/test/groovy/client/VertxHttpClientForkedTest.groovy +++ b/dd-java-agent/instrumentation/vertx/vertx-web/vertx-web-5.0/src/test/groovy/client/VertxHttpClientForkedTest.groovy @@ -17,11 +17,6 @@ import java.util.concurrent.CompletableFuture import java.util.concurrent.TimeUnit class VertxHttpClientForkedTest extends HttpClientTest implements TestingNettyHttpNamingConventions.ClientV0 { - @Override - boolean useStrictTraceWrites() { - return false - } - @AutoCleanup @Shared def vertx = Vertx.vertx(new VertxOptions()) From d2bdfd2e49f2fbf9b2c65f5d215fb83716930a11 Mon Sep 17 00:00:00 2001 From: Andrea Marziali Date: Wed, 16 Sep 2026 13:41:39 +0200 Subject: [PATCH 07/35] Incorporate skill takeaways in the main text --- .../skills/fix-continuation-leakage/SKILL.md | 33 +++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/.agents/skills/fix-continuation-leakage/SKILL.md b/.agents/skills/fix-continuation-leakage/SKILL.md index d87a570cca6..498155ddb4a 100644 --- a/.agents/skills/fix-continuation-leakage/SKILL.md +++ b/.agents/skills/fix-continuation-leakage/SKILL.md @@ -41,8 +41,37 @@ resolution, scope, thread, timing, and callsite data needed to find the missing - `[deferred-cleanup]`: a root iteration scope transferred cleanup to the bounded iteration cleaner. It may remain open at the test boundary and is not a leak. Do not generalize this to other `ITERATION` scopes; an unregistered iteration scope must still close normally. -4. Fix ownership where it breaks. Prefer one owner and `try/finally` cleanup across every exit. -5. Rerun the failing test, then its module. +4. Classify the captured work before changing code: + - For a real asynchronous operation, repair success, failure, cancellation, and rejection + cleanup. + - If the test started the work, wait for its terminal event and dispose or close it before the + test ends. + - If a framework initializer creates a permanent sentinel with no context consumer, disable + propagation only around that creation boundary. Match the exact type and method, and update + `knownMatchingTypes()` when shortcut matching is used. + - If an executor replaces a task before delegating, avoid capturing the discarded task while + preserving capture for the task actually submitted. + - For intentionally delayed work, wait for its documented terminal event rather than + suppressing propagation. + - For a context swap, verify both restoration and resource cleanup. Restore or close the + returned ownership object in `finally`; do not ignore every swap. +5. Prefer a test-lifecycle fix when production behavior is correct. Otherwise fix ownership where + it breaks, with one owner and `try/finally` cleanup across every exit. +6. Rerun the failing test, then its module. Validate the leaked record and root-trace publication + separately from trace-count or arrival-order assertions; fixing a leak may expose an unrelated + flaky assertion. + +## Fixture setup failures + +Automatic recording may start after `setupSpec()` or equivalent fixture initialization. For an +initialization error or a trace wait inside setup, temporarily record around that setup block and +remove the diagnostic scaffolding after finding the owner. + +Apply process-wide configuration before starting servers, actor systems, executors, or other +long-lived fixtures. Use a forked test or recreate the fixture when its static state cannot be +reset safely. + +## Do not hide evidence Do not make the test green with `strictTraceWrites(false)` or `@TrackScopeContinuations(enabled=false, reason="...")`. Those hide evidence. The opt-out requires From 4fd36840457b83f0a61cc8e9e24698a8fc0d8698 Mon Sep 17 00:00:00 2001 From: Andrea Marziali Date: Thu, 17 Sep 2026 11:11:40 +0200 Subject: [PATCH 08/35] Ensure scope diagnostics transform scope stacks --- .../test/scopediag/ScopeContinuationTransformer.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeContinuationTransformer.java b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeContinuationTransformer.java index f00a942eb2f..a5a97136a91 100644 --- a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeContinuationTransformer.java +++ b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeContinuationTransformer.java @@ -26,11 +26,10 @@ static synchronized void install() { return; } try { - // Related core types can otherwise load this target reentrantly while they are transformed. - Class.forName( - "datadog.trace.core.scopemanager.ScopeContinuation", - false, - ScopeContinuationTransformer.class.getClassLoader()); + // Related core types can otherwise load these targets reentrantly while they are transformed. + ClassLoader loader = ScopeContinuationTransformer.class.getClassLoader(); + Class.forName("datadog.trace.core.scopemanager.ScopeContinuation", false, loader); + Class.forName("datadog.trace.core.scopemanager.ScopeStack", false, loader); } catch (ClassNotFoundException missingCoreTracer) { throw new IllegalStateException( "Scope continuation diagnostics require dd-trace-core", missingCoreTracer); From 246657397ad1cc61c344123fbf91355bc89e7ad7 Mon Sep 17 00:00:00 2001 From: Andrea Marziali Date: Fri, 18 Sep 2026 09:30:17 +0200 Subject: [PATCH 09/35] Skill update --- .../skills/fix-continuation-leakage/SKILL.md | 16 +++++++++ docs/how_instrumentations_work.md | 36 ++++++++++++++++++- docs/how_to_test_with_junit.md | 9 +++-- 3 files changed, 58 insertions(+), 3 deletions(-) diff --git a/.agents/skills/fix-continuation-leakage/SKILL.md b/.agents/skills/fix-continuation-leakage/SKILL.md index 498155ddb4a..8b906fd6750 100644 --- a/.agents/skills/fix-continuation-leakage/SKILL.md +++ b/.agents/skills/fix-continuation-leakage/SKILL.md @@ -49,6 +49,9 @@ resolution, scope, thread, timing, and callsite data needed to find the missing - If a framework initializer creates a permanent sentinel with no context consumer, disable propagation only around that creation boundary. Match the exact type and method, and update `knownMatchingTypes()` when shortcut matching is used. + Check static initialization (``): first use under an active request can capture its + context in singleton tasks, including shaded Netty `GlobalEventExecutor` sentinels. Reproduce + first use in a fresh JVM; prewarming can hide the bug. Do not suppress all class initializers. - If an executor replaces a task before delegating, avoid capturing the discarded task while preserving capture for the task actually submitted. - For intentionally delayed work, wait for its documented terminal event rather than @@ -79,6 +82,19 @@ a reason and is only for a proven diagnostic incompatibility. If the failure is intermittent, treat that as a flaky-test finding, keep diagnostics enabled, and link the `@Flaky` annotation to a tracked issue. +## Assess production impact + +An unresolved continuation blocks normal reference-count completion, not necessarily publication. +The production `PendingTrace` buffer can still write finished spans; strict tests remove that +delayed-write fallback, but partial flush remains possible. Do not infer lost traces or a fixed +UI delay from a diagnostic failure. Check the collector and configuration; see +[continuation effects](../../../docs/how_instrumentations_work.md#continuation-effects). + +Memory retention requires a reachable owner of the continuation/context; it does not prove the +whole trace remains retained or memory grows without bound. Wrong parentage requires activation +of unrelated context or a leaked active scope. Separate the observed lifecycle defect from its +possible production effects and from test-only cleanup failures. + ## Explain it to a human Lead with one sentence: what was captured, which cleanup edge was missing, and where. Cite the diff --git a/docs/how_instrumentations_work.md b/docs/how_instrumentations_work.md index 97e97fa062a..7ca993c1e0a 100644 --- a/docs/how_instrumentations_work.md +++ b/docs/how_instrumentations_work.md @@ -828,13 +828,47 @@ methods. is used to pass context between threads. - Continuations must be either resumed or released. - If a Continuation is resumed it returns a `ContextScope` which must eventually be closed. -- Only after all scopes are closed and any non-resumed continuations are released may the Trace finally close. +- Resolve every continuation and close its resumed scopes. A held continuation also needs its + hold released. Normal reference-count completion requires all spans and continuations to resolve; + publication can happen earlier through buffering or partial flush. Notice in [`HttpClientRequestTracingHandler`](https://github.com/DataDog/dd-trace-java/blob/3fe1b2d6010e50f61518fa25af3bdeb03ae7712b/dd-java-agent/instrumentation/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/client/HttpClientRequestTracingHandler.java#L56) how the Continuation is used to obtain the `parentScope` which is finally [closed](https://github.com/DataDog/dd-trace-java/blob/3fe1b2d6010e50f61518fa25af3bdeb03ae7712b/dd-java-agent/instrumentation/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/client/HttpClientRequestTracingHandler.java#L111). +### Continuation effects + +An unresolved continuation keeps `PendingTrace`'s reference count positive and prevents its normal +completion write. It does not by itself prove that a production trace is lost. The default delaying +buffer can write finished spans despite pending references: it checks for 500 ms since the last +trace reference or 5 seconds since the oldest finished span. These are worker eligibility thresholds, +not hard latency bounds or guarantees about UI visibility. Partial flush, explicit flush and buffer +pressure can publish earlier; long-running and streaming trace collectors have other paths. + +Strict writes replace the delaying buffer with a discarding buffer, exposing unresolved ownership +in tests. Partial flush is still possible, so seeing spans arrive does not establish that all +continuations were released. See [PendingTrace](../dd-trace-core/src/main/java/datadog/trace/core/PendingTrace.java), +[PendingTraceBuffer](../dd-trace-core/src/main/java/datadog/trace/core/PendingTraceBuffer.java) and +[CoreTracer](../dd-trace-core/src/main/java/datadog/trace/core/CoreTracer.java) for the publication paths. + +A reachable task or callback can retain its captured context even after finished spans are written. +This does not establish that the entire trace stays in memory or that retention grows without bound. +Wrong parentage requires unrelated work to activate that context, or a scope to remain active on the +thread; an abandoned continuation alone does not contaminate another thread. + +### Static initialization + +Class initialization (``) runs on the thread triggering first use, which may carry request +context. If it creates singleton workers, timers or permanent sentinels, generic async instrumentation +can capture that request for infrastructure that outlives it. Netty's `GlobalEventExecutor`, including +its Couchbase-shaded variant, is one example: its permanent sentinel does not run or cancel normally. + +Reproduce first use under an active span in a fresh JVM; prewarming during test setup can hide the +capture. Where the task has no request-context consumer, suppress propagation only at the verified +creation boundary, including the exact type initializer when appropriate. Do not suppress propagation +for all static initializers or for legitimate request work. + ## Naming ### Gradle Module Names diff --git a/docs/how_to_test_with_junit.md b/docs/how_to_test_with_junit.md index f2d4a3bad18..ceeac3b987f 100644 --- a/docs/how_to_test_with_junit.md +++ b/docs/how_to_test_with_junit.md @@ -308,7 +308,7 @@ Call it from a static initializer (runs before `@BeforeAll`): ```java class MyTest extends AbstractInstrumentationTest { static { - testConfig.idGenerationStrategy("RANDOM").strictTraceWrites(false); + testConfig.idGenerationStrategy("RANDOM"); } } ``` @@ -318,7 +318,12 @@ Available settings: | Method | Default | Description | |--------------------------------|----------------|--------------------------------------| | `idGenerationStrategy(String)` | `"SEQUENTIAL"` | Span ID generation strategy | -| `strictTraceWrites(boolean)` | `true` | Enable strict trace write validation | + +Keep strict trace writes enabled in instrumentation tests; disabling them is not a supported way to +resolve a lifecycle failure. Production buffering can publish finished spans despite unresolved +continuations, so a successful trace assertion alone does not prove cleanup. See +[continuation effects](how_instrumentations_work.md#continuation-effects) and the +[continuation investigation skill](../.agents/skills/fix-continuation-leakage/SKILL.md). ### Basic test From e69e93c12ef634b29c025e2f4cb4294de7d01ba4 Mon Sep 17 00:00:00 2001 From: Andrea Marziali Date: Fri, 18 Sep 2026 16:02:16 +0200 Subject: [PATCH 10/35] review findings --- .../skills/fix-continuation-leakage/SKILL.md | 11 ++++-- .../scopediag/ScopeContinuationProbe.java | 2 +- .../scopediag/ScopeDiagnosticsReport.java | 3 ++ .../ScopeDiagnosticsIntegrationTest.java | 39 +++++++++++++++++++ .../agent/test/scopediag/ScopeRecordTest.java | 29 ++++++++++++++ 5 files changed, 80 insertions(+), 4 deletions(-) diff --git a/.agents/skills/fix-continuation-leakage/SKILL.md b/.agents/skills/fix-continuation-leakage/SKILL.md index 8b906fd6750..d287bf19466 100644 --- a/.agents/skills/fix-continuation-leakage/SKILL.md +++ b/.agents/skills/fix-continuation-leakage/SKILL.md @@ -24,13 +24,18 @@ resolution, scope, thread, timing, and callsite data needed to find the missing ## Work the failure -1. Run the smallest failing test with full output: +1. Resolve the actual Gradle module path and failing suite from CI or the module's tasks. Nesting + varies (for example, `netty:netty-4.1` versus `kotlin-coroutines-1.3`). Use `forkedTest` for + `*ForkedTest*` classes, or the suite-specific forked task such as `latestDepForkedTest`. + Run the smallest failing test with full output: ```bash -./gradlew :dd-java-agent:instrumentation:-:test --tests '' --info 2>&1 | tee /tmp/scopediag-run.txt +set -o pipefail +./gradlew :dd-java-agent:instrumentation:: --tests '' --info 2>&1 | tee /tmp/scopediag-run.txt ``` -2. Find `Scope/continuation timeline` in the output. If Gradle hides it, inspect the test XML's +2. Verify the selected test ran in that task's XML results; a successful build with no discovered + tests is not validation. Find `Scope/continuation timeline` in the output. If Gradle hides it, inspect the test XML's `` under the module's `build/test-results` directory. 3. Follow the failing record from its first event: - `LEAKED` / `NEVER_CLOSED`: find the success, error, cancellation, and rejection exits that diff --git a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeContinuationProbe.java b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeContinuationProbe.java index 4cd5ef08003..90b1e918c02 100644 --- a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeContinuationProbe.java +++ b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeContinuationProbe.java @@ -85,7 +85,7 @@ public static void onResolve( if (!recording) { return; } - if (countAfter != CANCELLED) { + if (countBefore != CANCELLED && countAfter != CANCELLED) { return; } // release discards; cancelFromContinuedScopeClose finishes. Its slow path delegates to release, diff --git a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsReport.java b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsReport.java index d7e3c56595a..6bd0c3ee214 100644 --- a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsReport.java +++ b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsReport.java @@ -338,6 +338,9 @@ private void appendScopeLine(StringBuilder sb, String indent, ScopeRecord scope) sb.append(' ').append(failures); } sb.append('\n'); + for (ScopeEvent event : scope.wrongThreadCloses()) { + appendEvent(sb, indent + "wrong-thread close", event); + } } private void appendEvent(StringBuilder sb, String label, ScopeEvent event) { diff --git a/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsIntegrationTest.java b/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsIntegrationTest.java index 89a4af142bd..288a5ba580a 100644 --- a/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsIntegrationTest.java +++ b/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsIntegrationTest.java @@ -97,6 +97,45 @@ void resolvedContinuationDoesNotLeak() { assertEquals(0, report.leakCount(), "activated then closed continuation is resolved"); } + @Test + void repeatedContinuedScopeCloseIsFlagged() { + tracer = CoreTracer.builder().writer(new ListWriter()).strictTraceWrites(false).build(); + ScopeDiagnostics.startRecording(); + + AgentSpan span = tracer.startSpan("test", "op"); + ContextContinuation continuation = tracer.capture(span); + ContextScope scope = continuation.resume(); + scope.close(); + assertFalse(ScopeDiagnostics.report().hasProblems()); + scope.close(); + span.finish(); + + ScopeDiagnosticsReport report = ScopeDiagnostics.report(); + assertEquals(1, report.doubleCount()); + assertEquals(0, report.leakCount()); + assertTrue(report.hasProblems()); + assertTrue(report.renderTimeline().contains("DOUBLE_FINISH")); + } + + @Test + void heldContinuationScopeCloseAndReleaseResolveOnce() { + tracer = CoreTracer.builder().writer(new ListWriter()).strictTraceWrites(false).build(); + ScopeDiagnostics.startRecording(); + + AgentSpan span = tracer.startSpan("test", "op"); + ContextContinuation continuation = tracer.capture(span).hold(); + ContextScope scope = continuation.resume(); + scope.close(); + assertEquals(1, ScopeDiagnostics.report().leakCount()); + continuation.release(); + span.finish(); + + ScopeDiagnosticsReport report = ScopeDiagnostics.report(); + assertEquals(0, report.doubleCount()); + assertEquals(0, report.leakCount()); + assertFalse(report.hasProblems()); + } + @Test void scopeLifetimeRecordedAndLinkedToContinuation() { tracer = CoreTracer.builder().writer(new ListWriter()).strictTraceWrites(false).build(); diff --git a/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeRecordTest.java b/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeRecordTest.java index 6af4a059c86..5e0b09cb41a 100644 --- a/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeRecordTest.java +++ b/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeRecordTest.java @@ -52,6 +52,35 @@ void openAndClosedHasNoFailures() { assertFalse(report(s).hasProblems()); } + @Test + void timelineIncludesEveryWrongThreadCloseCallsite() { + ScopeRecord s = scope(0, null, "main", 1000); + s.addWrongThreadClose( + new ScopeEvent( + ScopeEvent.Type.SCOPE_CLOSE_WRONG_THREAD, + "worker-one", + 2000, + new StackTraceElement[] { + new StackTraceElement("com.app.First", "close", "First.java", 12) + })); + s.addWrongThreadClose( + new ScopeEvent( + ScopeEvent.Type.SCOPE_CLOSE_WRONG_THREAD, + "worker-two", + 3000, + new StackTraceElement[] { + new StackTraceElement("com.app.Second", "close", "Second.java", 34) + })); + s.setClose(event(ScopeEvent.Type.SCOPE_CLOSE, "main", 4000)); + + ScopeDiagnosticsReport report = report(s); + assertFalse(report.hasProblems(), "wrong-thread cleanup remains advisory"); + String timeline = report.renderTimeline(); + assertTrue(timeline.contains("wrong-thread close")); + assertTrue(timeline.contains("@ worker-one at com.app.First.close(First.java:12)")); + assertTrue(timeline.contains("@ worker-two at com.app.Second.close(Second.java:34)")); + } + @Test void openWithoutCloseIsNeverClosed() { ScopeRecord s = scope(0, null, "main", 1000); From 5870a972dce31ee8c468e77d33c3f29177c200f3 Mon Sep 17 00:00:00 2001 From: Andrea Marziali Date: Fri, 18 Sep 2026 17:53:24 +0200 Subject: [PATCH 11/35] codex findings --- .../test/scopediag/ContinuationRecord.java | 2 +- .../scopediag/ScopeContinuationProbe.java | 59 ++++++------- .../test/scopediag/ScopeDiagnostics.java | 75 +++++++++------- .../ScopeDiagnosticsAdmissionTest.java | 85 +++++++++++++++++++ 4 files changed, 159 insertions(+), 62 deletions(-) create mode 100644 dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsAdmissionTest.java diff --git a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ContinuationRecord.java b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ContinuationRecord.java index 208de65c19e..9eab87ee408 100644 --- a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ContinuationRecord.java +++ b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ContinuationRecord.java @@ -133,7 +133,7 @@ public synchronized EnumSet failures(Long rootWrittenNanos) { if (!extraTerminals.isEmpty()) { failures.add(Failure.DOUBLE_FINISH); } - if (!failedActivations.isEmpty() || resumedAfterTerminal()) { + if ((terminal != null && !failedActivations.isEmpty()) || resumedAfterTerminal()) { failures.add(Failure.ACTIVATE_AFTER_RESOLVE); } if (rootWrittenNanos != null diff --git a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeContinuationProbe.java b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeContinuationProbe.java index 90b1e918c02..b5418a142e9 100644 --- a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeContinuationProbe.java +++ b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeContinuationProbe.java @@ -16,8 +16,6 @@ public final class ScopeContinuationProbe { */ static final int CANCELLED = Integer.MIN_VALUE >> 1; - private static volatile boolean recording = false; - private static volatile Field sourceField; private static volatile Field scopeSourceField; @@ -28,19 +26,9 @@ public final class ScopeContinuationProbe { private ScopeContinuationProbe() {} - /** Installs the transformer once and starts recording. */ - static synchronized void enable() { - ScopeContinuationTransformer.install(); - recording = true; - } - - /** Stops recording without uninstalling the transformer. */ - static void disable() { - recording = false; - } - public static void onCapture(Object self) { - if (!recording) { + Object window = ScopeDiagnostics.recordingWindow(); + if (window == null) { return; } try { @@ -48,7 +36,12 @@ public static void onCapture(Object self) { AgentSpan span = AgentSpan.fromContext(continuation.context()); if (span != null) { ScopeDiagnostics.recordCapture( - continuation, span.getTraceId(), span.getSpanId(), spanName(span), sourceOf(self)); + window, + continuation, + span.getTraceId(), + span.getSpanId(), + spanName(span), + sourceOf(self)); } } catch (Throwable ignored) { // Diagnostics must never affect the tracer. @@ -56,19 +49,21 @@ public static void onCapture(Object self) { } public static void onActivate(Object self, Object returnedScope, long activateNanos) { - if (!recording) { + Object window = ScopeDiagnostics.recordingWindow(); + if (window == null) { return; } try { ContextContinuation continuation = (ContextContinuation) self; if (returnedScope == NoopScope.INSTANCE) { // A noop result may indicate activation after resolution. - ScopeDiagnostics.recordActivateFailed(continuation); + ScopeDiagnostics.recordActivateFailed(window, continuation); return; } AgentSpan span = AgentSpan.fromContext(continuation.context()); if (span != null) { ScopeDiagnostics.recordActivate( + window, continuation, span.getTraceId(), span.getSpanId(), @@ -82,7 +77,8 @@ public static void onActivate(Object self, Object returnedScope, long activateNa public static void onResolve( Object self, String method, int countBefore, int countAfter, long resolveNanos) { - if (!recording) { + Object window = ScopeDiagnostics.recordingWindow(); + if (window == null) { return; } if (countBefore != CANCELLED && countAfter != CANCELLED) { @@ -94,23 +90,25 @@ public static void onResolve( try { ContextContinuation continuation = (ContextContinuation) self; ScopeDiagnostics.recordResolve( - continuation, cancelled, resolveNanos, countBefore == CANCELLED); + window, continuation, cancelled, resolveNanos, countBefore == CANCELLED); } catch (Throwable ignored) { } } public static void onRootWritten(Object traceId) { - if (!recording) { + Object window = ScopeDiagnostics.recordingWindow(); + if (window == null) { return; } try { - ScopeDiagnostics.recordRootWritten((DDTraceId) traceId); + ScopeDiagnostics.recordRootWritten(window, (DDTraceId) traceId); } catch (Throwable ignored) { } } public static void onScopeOpen(Object scope) { - if (!recording) { + Object window = ScopeDiagnostics.recordingWindow(); + if (window == null) { return; } try { @@ -119,39 +117,42 @@ public static void onScopeOpen(Object scope) { long spanId = span != null ? span.getSpanId() : 0L; String name = span != null ? spanName(span) : null; ScopeDiagnostics.recordScopeOpen( - scope, traceId, spanId, name, scopeSourceOf(scope), continuationOf(scope)); + window, scope, traceId, spanId, name, scopeSourceOf(scope), continuationOf(scope)); } catch (Throwable ignored) { } } public static void onScopeClose(Object scope) { - if (!recording) { + Object window = ScopeDiagnostics.recordingWindow(); + if (window == null) { return; } try { - ScopeDiagnostics.recordScopeClose(scope); + ScopeDiagnostics.recordScopeClose(window, scope); } catch (Throwable ignored) { } } public static void onDeferredScopeCleanup(Object scope) { - if (!recording) { + Object window = ScopeDiagnostics.recordingWindow(); + if (window == null) { return; } try { - ScopeDiagnostics.recordDeferredScopeCleanup(scope); + ScopeDiagnostics.recordDeferredScopeCleanup(window, scope); } catch (Throwable ignored) { } } /** Records an out-of-order close when the internal stack can be inspected. */ public static void onScopeClosing(Object scope) { - if (!recording) { + Object window = ScopeDiagnostics.recordingWindow(); + if (window == null) { return; } try { if (isNotOnTop(scope)) { - ScopeDiagnostics.recordScopeCloseWrongThread(scope); + ScopeDiagnostics.recordScopeCloseWrongThread(window, scope); } } catch (Throwable ignored) { } diff --git a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeDiagnostics.java b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeDiagnostics.java index c95582b545f..e78a0bc0adb 100644 --- a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeDiagnostics.java +++ b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeDiagnostics.java @@ -28,7 +28,7 @@ public final class ScopeDiagnostics { Collections.newSetFromMap(new IdentityHashMap()); private long seq; private long scopeSeq; - private boolean recording; + private volatile Object recordingWindow; private StackFilter stackFilter = new StackFilter(DEFAULT_MAX_FRAMES); private final Listener listener = new Listener(); @@ -42,35 +42,34 @@ public static void startRecording() { /** Clears any prior data and starts recording, keeping up to {@code maxFrames} per stack. */ public static void startRecording(int maxFrames) { - ScopeContinuationProbe.disable(); + ScopeContinuationTransformer.install(); synchronized (INSTANCE.lifecycleLock) { - INSTANCE.recording = false; INSTANCE.clear(); INSTANCE.stackFilter = new StackFilter(maxFrames); - } - ScopeContinuationProbe.enable(); - synchronized (INSTANCE.lifecycleLock) { - INSTANCE.recording = true; + INSTANCE.recordingWindow = new Object(); } } - /** Stops recording (the probe goes inert). Recorded data remains queryable until reset. */ + /** Stops recording. Recorded data remains queryable until reset. */ public static void stop() { - ScopeContinuationProbe.disable(); synchronized (INSTANCE.lifecycleLock) { - INSTANCE.recording = false; + INSTANCE.recordingWindow = null; } } /** Discards all recorded data. */ public static void reset() { - ScopeContinuationProbe.disable(); synchronized (INSTANCE.lifecycleLock) { - INSTANCE.recording = false; + INSTANCE.recordingWindow = null; INSTANCE.clear(); } } + /** Each probe keeps this identity until admission under the lifecycle lock. */ + static Object recordingWindow() { + return INSTANCE.recordingWindow; + } + /** Returns an immutable snapshot of the events recorded so far. */ public static ScopeDiagnosticsReport report() { synchronized (INSTANCE.lifecycleLock) { @@ -82,7 +81,7 @@ public static ScopeDiagnosticsReport report() { public static void awaitQuiescence() { long deadline = System.nanoTime() + DEFAULT_QUIESCENCE_TIMEOUT_MILLIS * 1_000_000L; synchronized (INSTANCE.lifecycleLock) { - while (INSTANCE.recording && INSTANCE.snapshot().hasIncompleteLifecycles()) { + while (INSTANCE.recordingWindow != null && INSTANCE.snapshot().hasIncompleteLifecycles()) { long remaining = deadline - System.nanoTime(); if (remaining <= 0) { return; @@ -162,15 +161,21 @@ private ScopeEvent event(ScopeEvent.Type type, long nanos) { } static void recordCapture( - ContextContinuation id, DDTraceId traceId, long spanId, String spanName, byte source) { + Object window, + ContextContinuation id, + DDTraceId traceId, + long spanId, + String spanName, + byte source) { synchronized (INSTANCE.lifecycleLock) { - if (INSTANCE.recording) { + if (window != null && window == INSTANCE.recordingWindow) { INSTANCE.listener.onCapture(id, traceId, spanId, spanName, source); } } } static void recordActivate( + Object window, ContextContinuation id, DDTraceId traceId, long spanId, @@ -178,38 +183,45 @@ static void recordActivate( byte source, long nanos) { synchronized (INSTANCE.lifecycleLock) { - if (INSTANCE.recording) { + if (window != null && window == INSTANCE.recordingWindow) { INSTANCE.listener.onActivate(id, traceId, spanId, spanName, source, nanos); } } } - static void recordActivateFailed(ContextContinuation id) { + static void recordActivateFailed(Object window, ContextContinuation id) { synchronized (INSTANCE.lifecycleLock) { - if (INSTANCE.recording) { + if (window != null && window == INSTANCE.recordingWindow) { INSTANCE.listener.onActivateFailed(id); } } } static void recordResolve( - ContextContinuation id, boolean cancelled, long resolveNanos, boolean alreadyResolved) { + Object window, + ContextContinuation id, + boolean cancelled, + long resolveNanos, + boolean alreadyResolved) { synchronized (INSTANCE.lifecycleLock) { - if (INSTANCE.recording && (alreadyResolved || INSTANCE.resolved.add(id))) { + if (window != null + && window == INSTANCE.recordingWindow + && (alreadyResolved || INSTANCE.resolved.add(id))) { INSTANCE.listener.onResolve(id, cancelled, resolveNanos); } } } - static void recordRootWritten(DDTraceId traceId) { + static void recordRootWritten(Object window, DDTraceId traceId) { synchronized (INSTANCE.lifecycleLock) { - if (INSTANCE.recording) { + if (window != null && window == INSTANCE.recordingWindow) { INSTANCE.listener.onRootWritten(traceId); } } } static void recordScopeOpen( + Object window, Object scope, DDTraceId traceId, long spanId, @@ -217,31 +229,31 @@ static void recordScopeOpen( byte source, ContextContinuation continuation) { synchronized (INSTANCE.lifecycleLock) { - if (INSTANCE.recording) { + if (window != null && window == INSTANCE.recordingWindow) { INSTANCE.listener.onScopeOpen(scope, traceId, spanId, spanName, source, continuation); } } } - static void recordScopeClose(Object scope) { + static void recordScopeClose(Object window, Object scope) { synchronized (INSTANCE.lifecycleLock) { - if (INSTANCE.recording) { + if (window != null && window == INSTANCE.recordingWindow) { INSTANCE.listener.onScopeClose(scope); } } } - static void recordScopeCloseWrongThread(Object scope) { + static void recordScopeCloseWrongThread(Object window, Object scope) { synchronized (INSTANCE.lifecycleLock) { - if (INSTANCE.recording) { + if (window != null && window == INSTANCE.recordingWindow) { INSTANCE.listener.onScopeCloseWrongThread(scope); } } } - static void recordDeferredScopeCleanup(Object scope) { + static void recordDeferredScopeCleanup(Object window, Object scope) { synchronized (INSTANCE.lifecycleLock) { - if (INSTANCE.recording) { + if (window != null && window == INSTANCE.recordingWindow) { INSTANCE.listener.onDeferredScopeCleanup(scope); } } @@ -277,9 +289,8 @@ void onActivate( void onActivateFailed(ContextContinuation id) { try { ContinuationRecord record = records.get(id); - // only an activation of an already-resolved continuation is a real failure; a plain - // rollback (e.g. cancelled before any capture was recorded) is benign and ignored - if (record != null && record.isResolved()) { + // Resolution advice can arrive after a concurrent failed resume. + if (record != null) { record.addFailedActivation(event(ScopeEvent.Type.ACTIVATE_FAILED)); } } catch (Throwable ignored) { diff --git a/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsAdmissionTest.java b/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsAdmissionTest.java new file mode 100644 index 00000000000..9b571a36842 --- /dev/null +++ b/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsAdmissionTest.java @@ -0,0 +1,85 @@ +package datadog.trace.agent.test.scopediag; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import datadog.context.Context; +import datadog.context.ContextContinuation; +import datadog.trace.api.DDTraceId; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +class ScopeDiagnosticsAdmissionTest { + @AfterEach + void reset() { + ScopeDiagnostics.reset(); + } + + @Test + void failedActivationBeforeResolutionAdviceIsRetained() { + ScopeDiagnostics.startRecording(); + Object window = ScopeDiagnostics.recordingWindow(); + ContextContinuation continuation = Context.root().capture(); + ScopeDiagnostics.recordCapture(window, continuation, DDTraceId.from(1), 2, "op", (byte) 0); + + // The tracer has resolved, but its exit advice has not reached the recorder yet. + ScopeDiagnostics.recordActivateFailed(window, continuation); + assertEquals(1, ScopeDiagnostics.report().records().get(0).failedActivations().size()); + assertEquals(0, ScopeDiagnostics.report().activateAfterResolveCount()); + ScopeDiagnostics.recordResolve(window, continuation, true, System.nanoTime(), false); + + ScopeDiagnosticsReport report = ScopeDiagnostics.report(); + assertEquals(0, report.leakCount()); + assertEquals(1, report.activateAfterResolveCount()); + assertTrue(report.hasProblems()); + } + + @Test + void failedActivationWithoutAnObservedContinuationIsIgnored() { + ScopeDiagnostics.startRecording(); + ScopeDiagnostics.recordActivateFailed( + ScopeDiagnostics.recordingWindow(), Context.root().capture()); + assertTrue(ScopeDiagnostics.report().records().isEmpty()); + } + + @Test + void delayedEventsCannotEnterTheNextRecordingWindow() { + ScopeDiagnostics.startRecording(); + Object previousWindow = ScopeDiagnostics.recordingWindow(); + ScopeDiagnostics.stop(); + ScopeDiagnostics.reset(); + ScopeDiagnostics.startRecording(); + Object currentWindow = ScopeDiagnostics.recordingWindow(); + ContextContinuation continuation = Context.root().capture(); + Object scope = new Object(); + DDTraceId trace = DDTraceId.from(1); + + // Deliver callbacks that read their window before stop but arrive after the next start. + ScopeDiagnostics.recordCapture(previousWindow, continuation, trace, 2, "old", (byte) 0); + ScopeDiagnostics.recordActivate(previousWindow, continuation, trace, 2, "old", (byte) 0, 1); + ScopeDiagnostics.recordActivateFailed(previousWindow, continuation); + ScopeDiagnostics.recordResolve(previousWindow, continuation, true, 2, false); + ScopeDiagnostics.recordRootWritten(previousWindow, trace); + ScopeDiagnostics.recordScopeOpen( + previousWindow, scope, trace, 2, "old", (byte) 0, continuation); + ScopeDiagnostics.recordScopeClose(previousWindow, scope); + ScopeDiagnostics.recordScopeCloseWrongThread(previousWindow, scope); + ScopeDiagnostics.recordDeferredScopeCleanup(previousWindow, scope); + + assertTrue(ScopeDiagnostics.report().records().isEmpty()); + assertTrue(ScopeDiagnostics.report().scopeRecords().isEmpty()); + + // Old resolution/deferred-cleanup/write events must not poison fresh records either. + ScopeDiagnostics.recordCapture(currentWindow, continuation, trace, 2, "new", (byte) 0); + ScopeDiagnostics.recordScopeOpen(currentWindow, scope, trace, 2, "new", (byte) 0, continuation); + ScopeDiagnostics.recordResolve(currentWindow, continuation, true, Long.MAX_VALUE, false); + assertEquals(0, ScopeDiagnostics.report().deferredCleanupScopeCount()); + ScopeDiagnostics.recordScopeClose(currentWindow, scope); + ScopeDiagnosticsReport report = ScopeDiagnostics.report(); + assertEquals(1, report.records().size()); + assertEquals(0, report.leakCount()); + assertEquals(0, report.lateCount()); + assertFalse(report.hasProblems()); + } +} From 9549280e8ffa5bdb53b1f592b94ce1074f6c717f Mon Sep 17 00:00:00 2001 From: Andrea Marziali Date: Tue, 22 Sep 2026 08:04:50 +0200 Subject: [PATCH 12/35] Adapt continuation diagnostics to ContextScope --- .../trace/agent/test/scopediag/ScopeContinuationProbe.java | 4 ++-- .../agent/test/scopediag/ScopeDiagnosticsIntegrationTest.java | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeContinuationProbe.java b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeContinuationProbe.java index b5418a142e9..cb426fb5572 100644 --- a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeContinuationProbe.java +++ b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeContinuationProbe.java @@ -1,8 +1,8 @@ package datadog.trace.agent.test.scopediag; import datadog.context.ContextContinuation; +import datadog.context.ContextScope; import datadog.trace.api.DDTraceId; -import datadog.trace.bootstrap.instrumentation.api.AgentScope; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; import datadog.trace.bootstrap.instrumentation.api.NoopScope; import java.lang.reflect.Field; @@ -112,7 +112,7 @@ public static void onScopeOpen(Object scope) { return; } try { - AgentSpan span = ((AgentScope) scope).span(); + AgentSpan span = AgentSpan.fromContext(((ContextScope) scope).context()); DDTraceId traceId = span != null ? span.getTraceId() : DDTraceId.ZERO; long spanId = span != null ? span.getSpanId() : 0L; String name = span != null ? spanName(span) : null; diff --git a/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsIntegrationTest.java b/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsIntegrationTest.java index 288a5ba580a..8cef045d5c0 100644 --- a/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsIntegrationTest.java +++ b/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsIntegrationTest.java @@ -8,7 +8,6 @@ import datadog.context.Context; import datadog.context.ContextContinuation; import datadog.context.ContextScope; -import datadog.trace.bootstrap.instrumentation.api.AgentScope; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; import datadog.trace.common.writer.ListWriter; import datadog.trace.core.CoreTracer; @@ -60,7 +59,7 @@ void sameSpanReactivationIsNotFlaggedActivateAfterResolve() { ScopeDiagnostics.startRecording(); AgentSpan span = tracer.startSpan("test", "op"); - AgentScope active = tracer.activateSpan(span); + ContextScope active = tracer.activateSpan(span); // Same-span reuse resolves the continuation before resume() returns. ContextContinuation continuation = tracer.capture(span); ContextScope reused = continuation.resume(); From beed56821b29c427ae1cfcda1321426c5cc181a2 Mon Sep 17 00:00:00 2001 From: Andrea Marziali Date: Tue, 22 Sep 2026 09:27:59 +0200 Subject: [PATCH 13/35] avoid release race --- .../test/scopediag/ContinuationAdvice.java | 21 ++-- .../scopediag/ScopeContinuationProbe.java | 65 ++++++++-- .../test/scopediag/ScopeDiagnostics.java | 13 +- .../test/scopediag/ScopeResolutionTest.java | 116 ++++++++++++++++++ 4 files changed, 190 insertions(+), 25 deletions(-) create mode 100644 dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeResolutionTest.java diff --git a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ContinuationAdvice.java b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ContinuationAdvice.java index cf430b5ed22..cbfa8c7256c 100644 --- a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ContinuationAdvice.java +++ b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ContinuationAdvice.java @@ -29,22 +29,19 @@ public static void exit( /** Timestamps entry because resolution may write the trace before the method returns. */ public static final class Cancel { - @Advice.OnMethodEnter - public static int enter( - @Advice.FieldValue("count") int count, - @Advice.Local("ddResolveNanos") long ddResolveNanos) { - ddResolveNanos = System.nanoTime(); - return count; + @Advice.OnMethodEnter(suppress = Throwable.class) + public static ScopeContinuationProbe.ResolveAttempt enter( + @Advice.This Object self, + @Advice.Origin("#m") String method, + @Advice.FieldValue("count") int count) { + return ScopeContinuationProbe.onResolveEnter(self, method, count); } - @Advice.OnMethodExit(suppress = Throwable.class) + @Advice.OnMethodExit(onThrowable = Throwable.class, suppress = Throwable.class) public static void exit( - @Advice.This Object self, - @Advice.Origin("#m") String method, - @Advice.Enter int countBefore, - @Advice.Local("ddResolveNanos") long ddResolveNanos, + @Advice.Enter ScopeContinuationProbe.ResolveAttempt attempt, @Advice.FieldValue("count") int countAfter) { - ScopeContinuationProbe.onResolve(self, method, countBefore, countAfter, ddResolveNanos); + ScopeContinuationProbe.onResolveExit(attempt, countAfter); } } } diff --git a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeContinuationProbe.java b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeContinuationProbe.java index cb426fb5572..7fb6ec378d3 100644 --- a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeContinuationProbe.java +++ b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeContinuationProbe.java @@ -16,6 +16,27 @@ public final class ScopeContinuationProbe { */ static final int CANCELLED = Integer.MIN_VALUE >> 1; + private static final ThreadLocal resolving = new ThreadLocal<>(); + + /** Distinguishes an explicit release from a scope close's internal release call. */ + public static final class ResolveAttempt { + private final Object window; + private final ContextContinuation continuation; + private final boolean release; + private final int countBefore; + private final long nanos = System.nanoTime(); + private final ResolveAttempt previous; + + private ResolveAttempt( + Object window, ContextContinuation continuation, boolean release, int countBefore) { + this.window = window; + this.continuation = continuation; + this.release = release; + this.countBefore = countBefore; + this.previous = resolving.get(); + } + } + private static volatile Field sourceField; private static volatile Field scopeSourceField; @@ -75,23 +96,49 @@ public static void onActivate(Object self, Object returnedScope, long activateNa } } - public static void onResolve( - Object self, String method, int countBefore, int countAfter, long resolveNanos) { + public static ResolveAttempt onResolveEnter(Object self, String method, int countBefore) { Object window = ScopeDiagnostics.recordingWindow(); if (window == null) { - return; + return null; } - if (countBefore != CANCELLED && countAfter != CANCELLED) { + ResolveAttempt attempt = + new ResolveAttempt( + window, (ContextContinuation) self, "release".equals(method), countBefore); + resolving.set(attempt); + return attempt; + } + + public static void onResolveExit(ResolveAttempt attempt, int countAfter) { + if (attempt == null) { return; } - // release discards; cancelFromContinuedScopeClose finishes. Its slow path delegates to release, - // so a multi-activation finish can appear as a cancellation. - boolean cancelled = "release".equals(method); try { - ContextContinuation continuation = (ContextContinuation) self; + if (attempt.countBefore != CANCELLED && countAfter != CANCELLED) { + return; + } + // Scope cleanup's slow path calls release(): only the outer close owns that event. + if (attempt.release) { + for (ResolveAttempt parent = attempt.previous; parent != null; parent = parent.previous) { + if (!parent.release + && parent.window == attempt.window + && parent.continuation == attempt.continuation) { + return; + } + } + } ScopeDiagnostics.recordResolve( - window, continuation, cancelled, resolveNanos, countBefore == CANCELLED); + attempt.window, + attempt.continuation, + attempt.release, + attempt.nanos, + attempt.countBefore == CANCELLED); } catch (Throwable ignored) { + } finally { + if (attempt.previous == null) { + resolving.remove(); + } else { + resolving.set(attempt.previous); + } } } diff --git a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeDiagnostics.java b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeDiagnostics.java index e78a0bc0adb..5262a0a8d7b 100644 --- a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeDiagnostics.java +++ b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeDiagnostics.java @@ -26,6 +26,8 @@ public final class ScopeDiagnostics { private final Map rootWrittenNanos = new HashMap<>(); private final Set resolved = Collections.newSetFromMap(new IdentityHashMap()); + private final Set released = + Collections.newSetFromMap(new IdentityHashMap()); private long seq; private long scopeSeq; private volatile Object recordingWindow; @@ -132,6 +134,7 @@ private void clear() { deferredCleanupScopes.clear(); rootWrittenNanos.clear(); resolved.clear(); + released.clear(); seq = 0; scopeSeq = 0; } @@ -204,10 +207,12 @@ static void recordResolve( long resolveNanos, boolean alreadyResolved) { synchronized (INSTANCE.lifecycleLock) { - if (window != null - && window == INSTANCE.recordingWindow - && (alreadyResolved || INSTANCE.resolved.add(id))) { - INSTANCE.listener.onResolve(id, cancelled, resolveNanos); + if (window != null && window == INSTANCE.recordingWindow) { + boolean firstResolution = INSTANCE.resolved.add(id); + boolean repeatedRelease = cancelled && !INSTANCE.released.add(id); + if (alreadyResolved || firstResolution || repeatedRelease) { + INSTANCE.listener.onResolve(id, cancelled, resolveNanos); + } } } } diff --git a/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeResolutionTest.java b/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeResolutionTest.java new file mode 100644 index 00000000000..4468a0f7219 --- /dev/null +++ b/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeResolutionTest.java @@ -0,0 +1,116 @@ +package datadog.trace.agent.test.scopediag; + +import static datadog.trace.agent.test.scopediag.ScopeContinuationProbe.CANCELLED; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; + +import datadog.context.Context; +import datadog.context.ContextContinuation; +import datadog.trace.api.DDTraceId; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** Replays advice observations deterministically, including overlapping method invocations. */ +class ScopeResolutionTest { + private ContextContinuation continuation; + + @BeforeEach + void start() { + ScopeDiagnostics.startRecording(); + continuation = Context.root().capture(); + ScopeDiagnostics.recordCapture( + ScopeDiagnostics.recordingWindow(), continuation, DDTraceId.from(1), 2, "op", (byte) 0); + } + + @AfterEach + void stop() { + ScopeDiagnostics.stop(); + ScopeDiagnostics.reset(); + } + + @Test + void overlappingReleasesAreReportedEvenWhenBothEnteredBeforeResolution() { + Object window = ScopeDiagnostics.recordingWindow(); + // Both release entries observed zero; both exits observed CANCELLED. + ScopeDiagnostics.recordResolve(window, continuation, true, 1, false); + ScopeDiagnostics.recordResolve(window, continuation, true, 2, false); + assertEquals(1, ScopeDiagnostics.report().doubleCount()); + assertEquals(0, ScopeDiagnostics.report().leakCount()); + } + + @Test + void probeRetainsOverlappingReleaseAttempts() { + ScopeContinuationProbe.ResolveAttempt first = enter("release", 0); + ScopeContinuationProbe.ResolveAttempt second = enter("release", 0); + ScopeContinuationProbe.onResolveExit(second, CANCELLED); + ScopeContinuationProbe.onResolveExit(first, CANCELLED); + assertEquals(1, ScopeDiagnostics.report().doubleCount()); + } + + @Test + void nestedReleaseBelongsToTheScopeClose() { + ScopeContinuationProbe.ResolveAttempt close = enter("cancelFromContinuedScopeClose", 2); + ScopeContinuationProbe.ResolveAttempt release = enter("release", 0); + ScopeContinuationProbe.onResolveExit(release, CANCELLED); + ScopeContinuationProbe.onResolveExit(close, CANCELLED); + assertResolvedOnce(); + assertEquals(ContinuationStatus.FINISHED, ScopeDiagnostics.report().records().get(0).status()); + } + + @Test + void overlappingLegitimateScopeClosesResolveOnce() { + Object window = ScopeDiagnostics.recordingWindow(); + // Separate resumed scopes may both observe CANCELLED by the time their exit advice runs. + ScopeDiagnostics.recordResolve(window, continuation, false, 1, false); + ScopeDiagnostics.recordResolve(window, continuation, false, 2, false); + assertResolvedOnce(); + } + + @Test + void holdReleaseAndLastScopeCloseCanReportInEitherOrder() { + Object window = ScopeDiagnostics.recordingWindow(); + ScopeDiagnostics.recordResolve(window, continuation, true, 1, false); + ScopeDiagnostics.recordResolve(window, continuation, false, 2, false); + assertResolvedOnce(); + + ScopeDiagnostics.reset(); + start(); + window = ScopeDiagnostics.recordingWindow(); + ScopeDiagnostics.recordResolve(window, continuation, false, 2, false); + ScopeDiagnostics.recordResolve(window, continuation, true, 1, false); + assertResolvedOnce(); + } + + @Test + void scopeCloseDuringNestedReleaseDoesNotProduceADuplicate() { + ScopeContinuationProbe.ResolveAttempt close = enter("cancelFromContinuedScopeClose", 2); + ScopeContinuationProbe.ResolveAttempt release = enter("release", 0); + // Another scope's exit advice arrives before the nested release's exit advice. + ScopeDiagnostics.recordResolve( + ScopeDiagnostics.recordingWindow(), continuation, false, System.nanoTime(), false); + ScopeContinuationProbe.onResolveExit(release, CANCELLED); + ScopeContinuationProbe.onResolveExit(close, CANCELLED); + assertResolvedOnce(); + } + + @Test + void resolutionEnteredInAnOldWindowCannotResolveANewCapture() { + ScopeContinuationProbe.ResolveAttempt release = enter("release", 0); + ScopeDiagnostics.reset(); + start(); + ScopeContinuationProbe.onResolveExit(release, CANCELLED); + assertEquals(1, ScopeDiagnostics.report().leakCount()); + ScopeContinuationProbe.onResolveExit(enter("release", 0), CANCELLED); + assertResolvedOnce(); + } + + private ScopeContinuationProbe.ResolveAttempt enter(String method, int count) { + return ScopeContinuationProbe.onResolveEnter(continuation, method, count); + } + + private void assertResolvedOnce() { + assertEquals(0, ScopeDiagnostics.report().leakCount()); + assertFalse(ScopeDiagnostics.report().hasProblems()); + } +} From 13ee8f85227dc6f9ed952ceddade018f93e6189e Mon Sep 17 00:00:00 2001 From: Andrea Marziali Date: Tue, 22 Sep 2026 09:58:54 +0200 Subject: [PATCH 14/35] Duplicate scope cleanup now reports double finish --- .../test/scopediag/ContinuationRecord.java | 10 +++- .../scopediag/ScopeContinuationProbe.java | 4 +- .../test/scopediag/ScopeDiagnostics.java | 12 ++-- .../test/scopediag/ScopeResolutionTest.java | 55 +++++++++++++++++++ 4 files changed, 73 insertions(+), 8 deletions(-) diff --git a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ContinuationRecord.java b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ContinuationRecord.java index 9eab87ee408..3eb8623bb0f 100644 --- a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ContinuationRecord.java +++ b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ContinuationRecord.java @@ -20,6 +20,7 @@ public final class ContinuationRecord { private final List resumes = new ArrayList<>(1); private final List failedActivations = new ArrayList<>(0); private ScopeEvent terminal; + private boolean duplicateTerminalAttempt; private final List extraTerminals = new ArrayList<>(0); private final List scopeRecordSeqs = new ArrayList<>(1); @@ -50,6 +51,12 @@ synchronized void addFailedActivation(ScopeEvent event) { /** First terminal sets {@link #terminal}; any subsequent terminal is a double-finish signal. */ synchronized void setTerminalOrExtra(ScopeEvent event) { + setTerminalOrExtra(event, false); + } + + synchronized void setTerminalOrExtra(ScopeEvent event, boolean duplicateAttempt) { + // Counter underflow can prove duplicate cleanup before the first close's advice arrives. + duplicateTerminalAttempt |= duplicateAttempt; if (terminal == null) { terminal = event; } else { @@ -84,6 +91,7 @@ synchronized ContinuationRecord snapshot() { copy.extraTerminals.add(event.snapshot()); } copy.scopeRecordSeqs.addAll(scopeRecordSeqs); + copy.duplicateTerminalAttempt = duplicateTerminalAttempt; return copy; } @@ -130,7 +138,7 @@ public synchronized EnumSet failures(Long rootWrittenNanos) { if (terminal == null) { failures.add(Failure.LEAKED); } - if (!extraTerminals.isEmpty()) { + if (duplicateTerminalAttempt || !extraTerminals.isEmpty()) { failures.add(Failure.DOUBLE_FINISH); } if ((terminal != null && !failedActivations.isEmpty()) || resumedAfterTerminal()) { diff --git a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeContinuationProbe.java b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeContinuationProbe.java index 7fb6ec378d3..8eb97804b96 100644 --- a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeContinuationProbe.java +++ b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeContinuationProbe.java @@ -113,7 +113,7 @@ public static void onResolveExit(ResolveAttempt attempt, int countAfter) { return; } try { - if (attempt.countBefore != CANCELLED && countAfter != CANCELLED) { + if (attempt.countBefore > CANCELLED && countAfter > CANCELLED) { return; } // Scope cleanup's slow path calls release(): only the outer close owns that event. @@ -131,7 +131,7 @@ public static void onResolveExit(ResolveAttempt attempt, int countAfter) { attempt.continuation, attempt.release, attempt.nanos, - attempt.countBefore == CANCELLED); + attempt.countBefore <= CANCELLED || countAfter < CANCELLED); } catch (Throwable ignored) { } finally { if (attempt.previous == null) { diff --git a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeDiagnostics.java b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeDiagnostics.java index 5262a0a8d7b..ecc9eb3e28f 100644 --- a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeDiagnostics.java +++ b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeDiagnostics.java @@ -205,13 +205,14 @@ static void recordResolve( ContextContinuation id, boolean cancelled, long resolveNanos, - boolean alreadyResolved) { + boolean duplicateAttempt) { synchronized (INSTANCE.lifecycleLock) { if (window != null && window == INSTANCE.recordingWindow) { boolean firstResolution = INSTANCE.resolved.add(id); boolean repeatedRelease = cancelled && !INSTANCE.released.add(id); - if (alreadyResolved || firstResolution || repeatedRelease) { - INSTANCE.listener.onResolve(id, cancelled, resolveNanos); + if (duplicateAttempt || firstResolution || repeatedRelease) { + INSTANCE.listener.onResolve( + id, cancelled, resolveNanos, duplicateAttempt || repeatedRelease); } } } @@ -302,12 +303,13 @@ void onActivateFailed(ContextContinuation id) { } } - void onResolve(ContextContinuation id, boolean cancelled, long resolveNanos) { + void onResolve( + ContextContinuation id, boolean cancelled, long resolveNanos, boolean duplicateAttempt) { try { ScopeEvent.Type type = cancelled ? ScopeEvent.Type.RESOLVE_CANCEL : ScopeEvent.Type.RESOLVE_FINISH; recordFor(id, DDTraceId.ZERO, 0, null, (byte) -1) - .setTerminalOrExtra(event(type, resolveNanos)); + .setTerminalOrExtra(event(type, resolveNanos), duplicateAttempt); } catch (Throwable ignored) { } } diff --git a/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeResolutionTest.java b/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeResolutionTest.java index 4468a0f7219..e129095d371 100644 --- a/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeResolutionTest.java +++ b/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeResolutionTest.java @@ -105,10 +105,65 @@ void resolutionEnteredInAnOldWindowCannotResolveANewCapture() { assertResolvedOnce(); } + @Test + void duplicateScopeClosesWithBothExitsBelowSentinelAreNotReportedAsLeaked() { + ScopeContinuationProbe.ResolveAttempt first = enter("cancelFromContinuedScopeClose", 1); + ScopeContinuationProbe.ResolveAttempt second = enter("cancelFromContinuedScopeClose", 1); + ScopeContinuationProbe.onResolveExit(second, CANCELLED - 1); + ScopeContinuationProbe.onResolveExit(first, CANCELLED - 1); + assertDuplicateWithoutLeak(); + } + + @Test + void duplicateScopeCloseAfterSuccessfulCloseIsReported() { + ScopeContinuationProbe.ResolveAttempt first = enter("cancelFromContinuedScopeClose", 1); + ScopeContinuationProbe.ResolveAttempt second = enter("cancelFromContinuedScopeClose", 1); + ScopeContinuationProbe.onResolveExit(second, CANCELLED); + ScopeContinuationProbe.onResolveExit(first, CANCELLED - 1); + assertDuplicateWithoutLeak(); + } + + @Test + void underflowBeforeSuccessfulCloseAdviceIsAlreadyDuplicateEvidence() { + ScopeContinuationProbe.ResolveAttempt first = enter("cancelFromContinuedScopeClose", 1); + ScopeContinuationProbe.ResolveAttempt second = enter("cancelFromContinuedScopeClose", 1); + ScopeContinuationProbe.onResolveExit(second, CANCELLED - 1); + try { + assertDuplicateWithoutLeak(); + } finally { + // The successful close sampled CANCELLED, but its recorder callback was delayed. + ScopeContinuationProbe.onResolveExit(first, CANCELLED); + } + assertDuplicateWithoutLeak(); + } + + @Test + void furtherCloseAfterCounterUnderflowRemainsAReportedDuplicate() { + ScopeContinuationProbe.onResolveExit(enter("cancelFromContinuedScopeClose", 1), CANCELLED); + ScopeContinuationProbe.onResolveExit( + enter("cancelFromContinuedScopeClose", CANCELLED - 1), CANCELLED - 2); + assertDuplicateWithoutLeak(); + } + + @Test + void legitimateScopeClosesObservingTheSameTerminalStateAreNotDuplicates() { + ScopeContinuationProbe.ResolveAttempt first = enter("cancelFromContinuedScopeClose", 2); + ScopeContinuationProbe.ResolveAttempt second = enter("cancelFromContinuedScopeClose", 1); + ScopeContinuationProbe.onResolveExit(second, CANCELLED); + ScopeContinuationProbe.onResolveExit(first, CANCELLED); + assertResolvedOnce(); + } + private ScopeContinuationProbe.ResolveAttempt enter(String method, int count) { return ScopeContinuationProbe.onResolveEnter(continuation, method, count); } + private void assertDuplicateWithoutLeak() { + ScopeDiagnosticsReport report = ScopeDiagnostics.report(); + assertEquals(0, report.leakCount()); + assertEquals(1, report.doubleCount()); + } + private void assertResolvedOnce() { assertEquals(0, ScopeDiagnostics.report().leakCount()); assertFalse(ScopeDiagnostics.report().hasProblems()); From 46ceb31a8bb530407963a7e7c75b4fdbf31bc32d Mon Sep 17 00:00:00 2001 From: Andrea Marziali Date: Tue, 22 Sep 2026 10:24:46 +0200 Subject: [PATCH 15/35] spotbugs --- .../datadog/trace/agent/test/scopediag/ContinuationRecord.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ContinuationRecord.java b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ContinuationRecord.java index 3eb8623bb0f..c57b958a131 100644 --- a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ContinuationRecord.java +++ b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ContinuationRecord.java @@ -85,13 +85,12 @@ synchronized ContinuationRecord snapshot() { copy.failedActivations.add(event.snapshot()); } if (terminal != null) { - copy.setTerminalOrExtra(terminal.snapshot()); + copy.setTerminalOrExtra(terminal.snapshot(), duplicateTerminalAttempt); } for (ScopeEvent event : extraTerminals) { copy.extraTerminals.add(event.snapshot()); } copy.scopeRecordSeqs.addAll(scopeRecordSeqs); - copy.duplicateTerminalAttempt = duplicateTerminalAttempt; return copy; } From 48b0bd8bbeece009dc4538783d95c1ae58245ffc Mon Sep 17 00:00:00 2001 From: Andrea Marziali Date: Wed, 23 Sep 2026 10:02:25 +0200 Subject: [PATCH 16/35] harden javadoc --- .../scopediag/ScopeContinuationProbe.java | 19 ++++++++++- .../ScopeContinuationTransformer.java | 33 +++++++++++++++++++ 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeContinuationProbe.java b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeContinuationProbe.java index 8eb97804b96..bd81814753f 100644 --- a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeContinuationProbe.java +++ b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeContinuationProbe.java @@ -8,7 +8,24 @@ import java.lang.reflect.Field; import java.lang.reflect.Method; -/** Forwards test-only Byte Buddy advice events to {@link ScopeDiagnostics}. */ +/** + * Forwards test-only Byte Buddy advice events to {@link ScopeDiagnostics}. + * + *

Reflective access depends on {@code ScopeContinuation.source}, {@code + * ContinuableScope.source}, {@code ContinuingScope.continuation}, and the chain {@code + * ContinuableScope.scopeManager -> scopeStack() -> checkTop(scope)}. An ordinary scope has no + * continuation; a continuing scope must expose its owning continuation so the recorder can link + * their lifecycles. + * + *

Callbacks retain their recording-window identity so delayed events cannot enter a later test's + * recording. Resolution entry and exit must remain paired, including exceptional exits, to restore + * the thread-local nesting used to distinguish a scope close from its internal release. + * + *

Probe failures are suppressed to avoid disturbing tracer behavior. Missing reflective members + * can therefore lose events or metadata without failing the instrumented call. Changes to the core + * members require corresponding updates here and assertions that their events and metadata are + * actually recorded, as described by {@link ScopeContinuationTransformer}. + */ public final class ScopeContinuationProbe { /** * Mirrors {@code ScopeContinuation.CANCELLED}. Reaching this value marks a resolved continuation; diff --git a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeContinuationTransformer.java b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeContinuationTransformer.java index a5a97136a91..b5e1c6544e1 100644 --- a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeContinuationTransformer.java +++ b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeContinuationTransformer.java @@ -15,6 +15,39 @@ /** * Installs test-only advice with a separate {@link AgentBuilder} because the tracer ignores its own * core classes. Retransformation covers classes loaded before the diagnostic starts. + * + *

The advice relies on the following lifecycle contract in dd-trace-core. When moving or + * changing these operations, update their matchers and advice together: + * + *

    + *
  • {@code ScopeContinuation.register()} returns the registered continuation; its normal exit + * records capture before the caller can resume or release it. + *
  • {@code ScopeContinuation.resume()} returns a scope, or {@code NoopScope.INSTANCE} when + * activation fails. Entry supplies the activation timestamp because same-span reuse can + * resolve the continuation before this method returns. + *
  • {@code ScopeContinuation.release()} and {@code cancelFromContinuedScopeClose()} expose + * resolution through their {@code count} field. The probe compares entry and exit counts with + * {@code CANCELLED}; a scope close's nested release belongs to that close, not a second + * resolution. A hold or outstanding activation can keep the continuation unresolved. + *
  • {@code ScopeStack.push(ContinuableScope)} receives a scope whose context, source, and + * optional continuation are already available. Entry records a close-owned scope; swapping + * context alone must not create one. {@code ContinuableScope.onProperClose()} marks completed + * cleanup. + *
  • {@code ContinuableScope.close()} is inspected before it changes the stack. A scope that is + * not on top is recorded as an out-of-order or wrong-thread close attempt. + *
  • {@code ContinuableScopeManager.scheduleRootIterationScopeCleanup(ScopeStack, + * ContinuableScope)} transfers cleanup responsibility to the iteration cleaner. Its normal + * exit marks the scope as having deferred cleanup. + *
  • {@code PendingTrace.write(boolean)} exposes {@code rootSpanWritten} and {@code traceId}. A + * non-partial write that changes the flag from false to true records the root write on normal + * exit; partial writes and already-written roots do not produce another event. + *
+ * + *

Keep the reflective member checks in {@code ScopeContinuationProbeTest} and the real-tracer + * assertions in {@code ScopeDiagnosticsIntegrationTest} aligned with this contract. Member + * existence alone does not prove advice was applied or still observes the right lifecycle boundary. + * Tests for an observed event must assert its presence: an empty report can otherwise look + * leak-free. */ final class ScopeContinuationTransformer { private static volatile ResettableClassFileTransformer transformer; From d9925e64b64db425a0028118f3aab7fb250f6dec Mon Sep 17 00:00:00 2001 From: Andrea Marziali Date: Wed, 23 Sep 2026 10:20:51 +0200 Subject: [PATCH 17/35] harden tests --- .../scopediag/ScopeContinuationProbeTest.java | 70 ++++++----- .../ScopeDiagnosticsIntegrationTest.java | 110 +++++++++++++++++- 2 files changed, 149 insertions(+), 31 deletions(-) diff --git a/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeContinuationProbeTest.java b/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeContinuationProbeTest.java index 3752b466af1..0a5a0df4f7d 100644 --- a/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeContinuationProbeTest.java +++ b/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeContinuationProbeTest.java @@ -1,11 +1,14 @@ package datadog.trace.agent.test.scopediag; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertTrue; +import datadog.context.ContextScope; +import datadog.trace.api.DDTraceId; import java.lang.reflect.Field; import java.lang.reflect.Method; +import java.lang.reflect.Modifier; import org.junit.jupiter.api.Test; /** Verifies the tracer internals used by {@link ScopeContinuationProbe}. */ @@ -27,46 +30,46 @@ void cancelledSentinelMatchesProduction() throws Exception { @Test void continuationHooksExist() throws Exception { Class scopeContinuation = Class.forName("datadog.trace.core.scopemanager.ScopeContinuation"); - assertNotNull(scopeContinuation.getDeclaredMethod("register"), "register() (capture)"); - assertNotNull(scopeContinuation.getDeclaredMethod("resume"), "resume()"); - assertNotNull(scopeContinuation.getDeclaredMethod("release"), "release() (resolve)"); - assertNotNull( - scopeContinuation.getDeclaredMethod("cancelFromContinuedScopeClose"), - "cancelFromContinuedScopeClose() (resolve)"); - assertNotNull(findField(scopeContinuation, "count"), "ScopeContinuation.count"); - assertNotNull(findField(scopeContinuation, "source"), "ScopeContinuation.source"); + assertMethod(scopeContinuation, "register", scopeContinuation); + assertMethod(scopeContinuation, "resume", ContextScope.class); + assertMethod(scopeContinuation, "release", void.class); + assertMethod(scopeContinuation, "cancelFromContinuedScopeClose", void.class); + assertField(scopeContinuation, "count", int.class); + assertField(scopeContinuation, "source", byte.class); } @Test void rootWrittenHookExists() throws Exception { Class pendingTrace = Class.forName("datadog.trace.core.PendingTrace"); - assertNotNull( - pendingTrace.getDeclaredMethod("write", boolean.class), "PendingTrace.write(boolean)"); - assertNotNull(findField(pendingTrace, "rootSpanWritten"), "PendingTrace.rootSpanWritten"); - assertNotNull(findField(pendingTrace, "traceId"), "PendingTrace.traceId"); + assertMethod(pendingTrace, "write", int.class, boolean.class); + assertField(pendingTrace, "rootSpanWritten", boolean.class); + assertField(pendingTrace, "traceId", DDTraceId.class); } @Test void scopeLifecycleHooksExist() throws Exception { Class scope = Class.forName("datadog.trace.core.scopemanager.ContinuableScope"); - assertNotNull(scope.getDeclaredMethod("afterActivated"), "afterActivated() (scope open)"); - assertNotNull(scope.getDeclaredMethod("onProperClose"), "onProperClose() (scope close)"); - assertNotNull(scope.getDeclaredMethod("close"), "close() (wrong-thread check)"); - assertNotNull(findField(scope, "source"), "ContinuableScope.source"); - + Class stack = Class.forName("datadog.trace.core.scopemanager.ScopeStack"); + assertMethod(stack, "push", void.class, scope); + assertMethod(scope, "onProperClose", void.class); + assertMethod(scope, "close", void.class); + assertField(scope, "source", byte.class); Class continuing = Class.forName("datadog.trace.core.scopemanager.ContinuingScope"); - assertNotNull( - continuing.getDeclaredField("continuation"), "ContinuingScope.continuation (scope link)"); + Class continuation = Class.forName("datadog.trace.core.scopemanager.ScopeContinuation"); + assertField(continuing, "continuation", continuation); + Class manager = Class.forName("datadog.trace.core.scopemanager.ContinuableScopeManager"); + assertMethod(manager, "scheduleRootIterationScopeCleanup", void.class, stack, scope); } @Test void wrongThreadCheckChainExists() throws Exception { Class scope = Class.forName("datadog.trace.core.scopemanager.ContinuableScope"); - assertNotNull(findField(scope, "scopeManager"), "ContinuableScope.scopeManager"); + Class manager = Class.forName("datadog.trace.core.scopemanager.ContinuableScopeManager"); - assertNotNull(manager.getDeclaredMethod("scopeStack"), "ContinuableScopeManager.scopeStack()"); + assertField(scope, "scopeManager", manager); Class stack = Class.forName("datadog.trace.core.scopemanager.ScopeStack"); - assertTrue(hasMethod(stack, "checkTop", 1), "ScopeStack.checkTop(scope)"); + assertMethod(manager, "scopeStack", stack); + assertMethod(stack, "checkTop", boolean.class, scope); } private static Field findField(Class cls, String name) { @@ -79,12 +82,19 @@ private static Field findField(Class cls, String name) { return null; } - private static boolean hasMethod(Class cls, String name, int params) { - for (Method m : cls.getDeclaredMethods()) { - if (m.getName().equals(name) && m.getParameterCount() == params) { - return true; - } - } - return false; + private static void assertField(Class owner, String name, Class type) { + Field field = findField(owner, name); + assertNotNull(field, owner.getName() + "." + name); + assertEquals(type, field.getType(), field.toString()); + assertFalse(Modifier.isStatic(field.getModifiers()), field.toString()); + field.setAccessible(true); + } + + private static void assertMethod( + Class owner, String name, Class returnType, Class... arguments) throws Exception { + Method method = owner.getDeclaredMethod(name, arguments); + assertEquals(returnType, method.getReturnType(), method.toString()); + assertFalse(Modifier.isStatic(method.getModifiers()), method.toString()); + method.setAccessible(true); } } diff --git a/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsIntegrationTest.java b/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsIntegrationTest.java index 8cef045d5c0..ae3fd3c5d00 100644 --- a/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsIntegrationTest.java +++ b/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsIntegrationTest.java @@ -11,8 +11,11 @@ import datadog.trace.bootstrap.instrumentation.api.AgentSpan; import datadog.trace.common.writer.ListWriter; import datadog.trace.core.CoreTracer; +import datadog.trace.core.DDSpan; +import datadog.trace.core.PendingTrace; import java.util.List; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.FutureTask; import java.util.concurrent.TimeUnit; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; @@ -70,6 +73,12 @@ void sameSpanReactivationIsNotFlaggedActivateAfterResolve() { ScopeDiagnosticsReport report = ScopeDiagnostics.report(); assertEquals(1, report.records().size()); + ContinuationRecord record = report.records().get(0); + assertEquals(1, record.resumes().size(), "same-span resume must be observed"); + assertNotNull(record.terminal()); + assertTrue( + record.resumes().get(0).nanos <= record.terminal().nanos, + "resume must be timestamped before its nested resolution"); assertEquals( 0, report.activateAfterResolveCount(), @@ -155,7 +164,17 @@ void scopeLifetimeRecordedAndLinkedToContinuation() { assertTrue(linked.closed(), "scope close observed"); assertEquals(0, report.neverClosedScopeCount()); assertEquals(1, report.records().size()); - assertEquals(Long.valueOf(report.records().get(0).seq), linked.continuationSeq); + ContinuationRecord record = report.records().get(0); + assertNotNull(record.capture()); + assertFalse(record.orphan); + assertEquals(1, record.resumes().size()); + assertEquals(ContinuationStatus.FINISHED, record.status()); + assertEquals(span.getTraceId(), record.traceId); + assertEquals(span.getSpanId(), record.spanId); + assertEquals("op", record.spanName); + assertEquals(record.source, linked.source); + assertFalse(record.source == (byte) -1, "source must not fall back after reflection failure"); + assertEquals(Long.valueOf(record.seq), linked.continuationSeq); } @Test @@ -262,6 +281,95 @@ void eventsAfterStopAreRejected() { span.finish(); } + @Test + void resumeAfterRootWriteIsRecorded() { + tracer = CoreTracer.builder().writer(new ListWriter()).strictTraceWrites(false).build(); + ScopeDiagnostics.startRecording(); + AgentSpan span = tracer.startSpan("test", "op"); + ContextContinuation continuation = tracer.capture(span); + try { + span.finish(); + ((PendingTrace) ((DDSpan) span).spanContext().getTraceCollector()).write(); + assertEquals(0, ScopeDiagnostics.report().lateCount()); + try (ContextScope ignored = continuation.resume()) { + assertEquals(1, ScopeDiagnostics.report().lateCount()); + } + assertEquals(0, ScopeDiagnostics.report().leakCount()); + } finally { + continuation.release(); + } + } + + @Test + void resumeAfterReleaseRecordsFailedActivation() { + tracer = CoreTracer.builder().writer(new ListWriter()).strictTraceWrites(false).build(); + ScopeDiagnostics.startRecording(); + AgentSpan span = tracer.startSpan("test", "op"); + try { + ContextContinuation continuation = tracer.capture(span); + continuation.release(); + assertEquals(0, ScopeDiagnostics.report().activateAfterResolveCount()); + continuation.resume().close(); + ScopeDiagnosticsReport report = ScopeDiagnostics.report(); + assertEquals(1, report.records().size()); + assertEquals(1, report.records().get(0).failedActivations().size()); + assertTrue(report.records().get(0).resumes().isEmpty()); + assertEquals(1, report.activateAfterResolveCount()); + assertEquals(0, report.leakCount()); + } finally { + span.finish(); + } + } + + @Test + void outOfOrderScopeCloseIsRecorded() throws Exception { + assertMisplacedClose(false); + } + + @Test + void wrongThreadScopeCloseIsRecorded() throws Exception { + assertMisplacedClose(true); + } + + private void assertMisplacedClose(boolean anotherThread) throws Exception { + tracer = CoreTracer.builder().writer(new ListWriter()).strictTraceWrites(false).build(); + ScopeDiagnostics.startRecording(); + AgentSpan outerSpan = tracer.startSpan("test", "outer"); + AgentSpan innerSpan = tracer.startSpan("test", "inner"); + ContextScope outer = tracer.activateSpan(outerSpan); + ContextScope inner = tracer.activateSpan(innerSpan); + try { + assertEquals(0, ScopeDiagnostics.report().closeWrongThreadCount()); + String closingThread = Thread.currentThread().getName(); + if (anotherThread) { + FutureTask close = new FutureTask<>(outer::close, null); + Thread worker = new Thread(close, "scope-diagnostics-wrong-thread"); + worker.setDaemon(true); + worker.start(); + close.get(5, TimeUnit.SECONDS); + closingThread = worker.getName(); + } else { + outer.close(); + } + ScopeDiagnosticsReport report = ScopeDiagnostics.report(); + assertEquals(1, report.closeWrongThreadCount()); + ScopeRecord record = + report.scopeRecords().stream() + .filter(scope -> scope.spanId == outerSpan.getSpanId()) + .findFirst() + .orElseThrow(() -> new AssertionError("outer scope was not recorded")); + assertEquals(1, record.wrongThreadCloses().size()); + assertEquals(closingThread, record.wrongThreadCloses().get(0).threadName); + assertFalse(record.closed(), "owner stack has not unwound yet"); + } finally { + inner.close(); + innerSpan.finish(); + outerSpan.finish(); + } + assertEquals(0, ScopeDiagnostics.report().neverClosedScopeCount()); + assertEquals(1, ScopeDiagnostics.report().closeWrongThreadCount()); + } + private static ScopeRecord continuationScope(ScopeDiagnosticsReport report) { List scopes = report.scopeRecords(); for (ScopeRecord s : scopes) { From bdc81e270a31c7d85924adc53817376c18cad84b Mon Sep 17 00:00:00 2001 From: Andrea Marziali Date: Wed, 23 Sep 2026 12:32:44 +0200 Subject: [PATCH 18/35] Report explicit continuation resolution as released --- .../test/scopediag/ContinuationRecord.java | 4 ++-- .../test/scopediag/ContinuationStatus.java | 11 ++++++---- .../test/scopediag/ScopeDiagnostics.java | 10 ++++----- .../scopediag/ScopeDiagnosticsReport.java | 4 ++-- .../agent/test/scopediag/ScopeEvent.java | 2 +- .../ScopeDiagnosticsIntegrationTest.java | 22 +++++++++++++++++++ .../scopediag/ScopeDiagnosticsReportTest.java | 4 ++-- 7 files changed, 41 insertions(+), 16 deletions(-) diff --git a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ContinuationRecord.java b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ContinuationRecord.java index c57b958a131..0080ed39f93 100644 --- a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ContinuationRecord.java +++ b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ContinuationRecord.java @@ -124,8 +124,8 @@ public synchronized boolean isResolved() { public synchronized ContinuationStatus status() { if (terminal != null) { - return terminal.type == ScopeEvent.Type.RESOLVE_CANCEL - ? ContinuationStatus.CANCELLED + return terminal.type == ScopeEvent.Type.RESOLVE_RELEASE + ? ContinuationStatus.RELEASED : ContinuationStatus.FINISHED; } return ContinuationStatus.LEAKED; diff --git a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ContinuationStatus.java b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ContinuationStatus.java index e26bf691fa4..493cab1173e 100644 --- a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ContinuationStatus.java +++ b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ContinuationStatus.java @@ -1,11 +1,14 @@ package datadog.trace.agent.test.scopediag; -/** Derived continuation state used when rendering a timeline. */ +/** + * Derived continuation state used when rendering a timeline. Resolved states describe the first + * recorded terminal observation, not which racing operation completed the core transition. + */ public enum ContinuationStatus { - /** Resolved normally (all activations closed or a clean cancel with no outstanding work). */ + /** Resolution observed through scope cleanup. */ FINISHED, - /** Resolved via the cancel-with-outstanding-work path. */ - CANCELLED, + /** Resolution observed through an explicit release, including a clean release after a hold. */ + RELEASED, /** Captured (and possibly resumed) but never resolved within the recording window. */ LEAKED } diff --git a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeDiagnostics.java b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeDiagnostics.java index ecc9eb3e28f..c26b8739065 100644 --- a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeDiagnostics.java +++ b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeDiagnostics.java @@ -203,16 +203,16 @@ static void recordActivateFailed(Object window, ContextContinuation id) { static void recordResolve( Object window, ContextContinuation id, - boolean cancelled, + boolean release, long resolveNanos, boolean duplicateAttempt) { synchronized (INSTANCE.lifecycleLock) { if (window != null && window == INSTANCE.recordingWindow) { boolean firstResolution = INSTANCE.resolved.add(id); - boolean repeatedRelease = cancelled && !INSTANCE.released.add(id); + boolean repeatedRelease = release && !INSTANCE.released.add(id); if (duplicateAttempt || firstResolution || repeatedRelease) { INSTANCE.listener.onResolve( - id, cancelled, resolveNanos, duplicateAttempt || repeatedRelease); + id, release, resolveNanos, duplicateAttempt || repeatedRelease); } } } @@ -304,10 +304,10 @@ void onActivateFailed(ContextContinuation id) { } void onResolve( - ContextContinuation id, boolean cancelled, long resolveNanos, boolean duplicateAttempt) { + ContextContinuation id, boolean release, long resolveNanos, boolean duplicateAttempt) { try { ScopeEvent.Type type = - cancelled ? ScopeEvent.Type.RESOLVE_CANCEL : ScopeEvent.Type.RESOLVE_FINISH; + release ? ScopeEvent.Type.RESOLVE_RELEASE : ScopeEvent.Type.RESOLVE_FINISH; recordFor(id, DDTraceId.ZERO, 0, null, (byte) -1) .setTerminalOrExtra(event(type, resolveNanos), duplicateAttempt); } catch (Throwable ignored) { diff --git a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsReport.java b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsReport.java index 6bd0c3ee214..d25d9020853 100644 --- a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsReport.java +++ b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsReport.java @@ -262,7 +262,7 @@ public String renderTimeline() { if (terminal != null) { appendEvent( sb, - terminal.type == ScopeEvent.Type.RESOLVE_CANCEL ? "cancel " : "finish ", + terminal.type == ScopeEvent.Type.RESOLVE_RELEASE ? "release " : "finish ", terminal); } for (ScopeEvent extra : r.extraTerminals()) { @@ -275,7 +275,7 @@ public String renderTimeline() { } } if (terminal == null) { - sb.append(" LEAKED (never finished or cancelled)\n"); + sb.append(" LEAKED (never finished or released)\n"); } } diff --git a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeEvent.java b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeEvent.java index c1b77c461e5..2d20de2aacd 100644 --- a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeEvent.java +++ b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeEvent.java @@ -8,7 +8,7 @@ public enum Type { /** An {@code activate()} that returned the noop scope after the continuation was resolved. */ ACTIVATE_FAILED, RESOLVE_FINISH, - RESOLVE_CANCEL, + RESOLVE_RELEASE, SCOPE_OPEN, SCOPE_CLOSE, /** A scope was closed while not on top of its thread's stack. */ diff --git a/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsIntegrationTest.java b/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsIntegrationTest.java index ae3fd3c5d00..55b9adbb408 100644 --- a/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsIntegrationTest.java +++ b/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsIntegrationTest.java @@ -139,11 +139,33 @@ void heldContinuationScopeCloseAndReleaseResolveOnce() { span.finish(); ScopeDiagnosticsReport report = ScopeDiagnostics.report(); + assertEquals(1, report.records().size()); + assertEquals(ContinuationStatus.RELEASED, report.records().get(0).status()); + assertEquals(ScopeEvent.Type.RESOLVE_RELEASE, report.records().get(0).terminal().type); + assertTrue(report.renderTimeline().contains("RELEASED")); + assertTrue(report.renderTimeline().contains("release ")); + assertFalse(report.renderTimeline().contains("cancel")); assertEquals(0, report.doubleCount()); assertEquals(0, report.leakCount()); assertFalse(report.hasProblems()); } + @Test + void releaseWithoutResumeIsReportedAsReleased() { + tracer = CoreTracer.builder().writer(new ListWriter()).strictTraceWrites(false).build(); + ScopeDiagnostics.startRecording(); + + AgentSpan span = tracer.startSpan("test", "op"); + tracer.capture(span).release(); + span.finish(); + + ScopeDiagnosticsReport report = ScopeDiagnostics.report(); + assertEquals(1, report.records().size()); + assertEquals(ContinuationStatus.RELEASED, report.records().get(0).status()); + assertEquals(ScopeEvent.Type.RESOLVE_RELEASE, report.records().get(0).terminal().type); + assertFalse(report.hasProblems()); + } + @Test void scopeLifetimeRecordedAndLinkedToContinuation() { tracer = CoreTracer.builder().writer(new ListWriter()).strictTraceWrites(false).build(); diff --git a/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsReportTest.java b/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsReportTest.java index 5155d563e0b..0a39128a660 100644 --- a/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsReportTest.java +++ b/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsReportTest.java @@ -102,7 +102,7 @@ void multipleResolutionsAreFlaggedDouble() { @Test void activationAfterResolveIsFailure() { ContinuationRecord r = record(0, DDTraceId.from(14)); - r.setTerminalOrExtra(event(ScopeEvent.Type.RESOLVE_CANCEL, "pool-1", 2000)); + r.setTerminalOrExtra(event(ScopeEvent.Type.RESOLVE_RELEASE, "pool-1", 2000)); r.addResume(event(ScopeEvent.Type.ACTIVATE, "pool-2", 3000)); ScopeDiagnosticsReport report = report(list(r), map()); @@ -115,7 +115,7 @@ void activationAfterResolveIsFailure() { @Test void failedActivationIsActivateAfterResolve() { ContinuationRecord r = record(0, DDTraceId.from(141)); - r.setTerminalOrExtra(event(ScopeEvent.Type.RESOLVE_CANCEL, "pool-1", 2000)); + r.setTerminalOrExtra(event(ScopeEvent.Type.RESOLVE_RELEASE, "pool-1", 2000)); r.addFailedActivation(event(ScopeEvent.Type.ACTIVATE_FAILED, "pool-2", 3000)); ScopeDiagnosticsReport report = report(list(r), map()); From 09da15f585fdeae8a1719ba11bbcb4a4a69ba9b1 Mon Sep 17 00:00:00 2001 From: Andrea Marziali Date: Wed, 23 Sep 2026 12:34:03 +0200 Subject: [PATCH 19/35] Use earliest resume timestamp in continuation diagnostics --- .../test/scopediag/ContinuationRecord.java | 6 ++++- .../scopediag/ScopeDiagnosticsReportTest.java | 22 +++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ContinuationRecord.java b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ContinuationRecord.java index 0080ed39f93..c0f99fe79eb 100644 --- a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ContinuationRecord.java +++ b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ContinuationRecord.java @@ -181,7 +181,11 @@ public synchronized Long captureToFirstResumeNanos() { if (capture == null || resumes.isEmpty()) { return null; } - return resumes.get(0).nanos - capture.nanos; + long firstResumeNanos = Long.MAX_VALUE; + for (ScopeEvent resume : resumes) { + firstResumeNanos = Math.min(firstResumeNanos, resume.nanos); + } + return firstResumeNanos - capture.nanos; } /** Nanos between capture and the terminal resolution, or {@code null} if not both observed. */ diff --git a/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsReportTest.java b/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsReportTest.java index 0a39128a660..e09b84c58d7 100644 --- a/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsReportTest.java +++ b/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsReportTest.java @@ -2,6 +2,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; import datadog.trace.api.DDTraceId; @@ -26,6 +27,27 @@ private static ContinuationRecord record(long seq, DDTraceId trace) { seq, trace, 7L, "op", (byte) 0, false, event(ScopeEvent.Type.CAPTURE, "main", 1000)); } + @Test + void firstResumeTimingUsesEarliestTimestampRegardlessOfRecordingOrder() { + ContinuationRecord r = record(0, DDTraceId.from(10)); + r.addResume(event(ScopeEvent.Type.ACTIVATE, "pool-2", 3000)); + r.addResume(event(ScopeEvent.Type.ACTIVATE, "pool-1", 2000)); + + assertEquals(Long.valueOf(1000), r.captureToFirstResumeNanos()); + assertEquals(Long.valueOf(1000), r.snapshot().captureToFirstResumeNanos()); + } + + @Test + void firstResumeTimingRequiresCaptureAndResume() { + ContinuationRecord captured = record(0, DDTraceId.from(10)); + assertNull(captured.captureToFirstResumeNanos()); + + ContinuationRecord orphan = + new ContinuationRecord(1, DDTraceId.from(10), 7L, "op", (byte) 0, true, null); + orphan.addResume(event(ScopeEvent.Type.ACTIVATE, "pool-1", 2000)); + assertNull(orphan.captureToFirstResumeNanos()); + } + @Test void resolvedContinuationHasNoFailures() { ContinuationRecord r = record(0, DDTraceId.from(10)); From aedf44f086325dce4cc3c6fe847fd56e2f802720 Mon Sep 17 00:00:00 2001 From: Andrea Marziali Date: Wed, 23 Sep 2026 12:43:00 +0200 Subject: [PATCH 20/35] Avoid dropping similarly named classes --- .../agent/test/scopediag/StackFilter.java | 32 +++++++++++----- .../agent/test/scopediag/StackFilterTest.java | 38 +++++++++++++++++++ 2 files changed, 60 insertions(+), 10 deletions(-) diff --git a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/StackFilter.java b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/StackFilter.java index e2e735a850e..fbc39593188 100644 --- a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/StackFilter.java +++ b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/StackFilter.java @@ -10,15 +10,7 @@ final class StackFilter { "datadog.trace.core.", "datadog.trace.bootstrap.instrumentation.java.concurrent.", "datadog.trace.bootstrap.instrumentation.api.", - "datadog.trace.bootstrap.InstrumentationContext", - "java.lang.Thread.getStackTrace", - "java.util.concurrent.ThreadPoolExecutor", - "java.util.concurrent.ScheduledThreadPoolExecutor", - "java.util.concurrent.ForkJoinPool", - "java.util.concurrent.ForkJoinWorkerThread", "java.util.concurrent.Executors$", - "java.util.concurrent.FutureTask", - "java.util.concurrent.CompletableFuture", "jdk.internal.reflect.", "java.lang.reflect.", "sun.reflect.", @@ -28,6 +20,16 @@ final class StackFilter { "net.bytebuddy.", }; + private static final String[] DROP_CLASSES = { + "datadog.trace.bootstrap.InstrumentationContext", + "java.util.concurrent.ThreadPoolExecutor", + "java.util.concurrent.ScheduledThreadPoolExecutor", + "java.util.concurrent.ForkJoinPool", + "java.util.concurrent.ForkJoinWorkerThread", + "java.util.concurrent.FutureTask", + "java.util.concurrent.CompletableFuture", + }; + private final int maxFrames; StackFilter(int maxFrames) { @@ -56,9 +58,19 @@ StackTraceElement[] filter(StackTraceElement[] raw) { } private static boolean isDropped(StackTraceElement frame) { - String fqn = frame.getClassName() + "." + frame.getMethodName(); + String className = frame.getClassName(); + if (className.equals("java.lang.Thread") && frame.getMethodName().equals("getStackTrace")) { + return true; + } for (String prefix : DROP_PREFIXES) { - if (fqn.startsWith(prefix) || frame.getClassName().startsWith(prefix)) { + if (className.startsWith(prefix)) { + return true; + } + } + for (String droppedClass : DROP_CLASSES) { + if (className.equals(droppedClass) + || (className.startsWith(droppedClass) + && className.charAt(droppedClass.length()) == '$')) { return true; } } diff --git a/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/StackFilterTest.java b/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/StackFilterTest.java index 3853ab274fd..ecb93f7a53c 100644 --- a/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/StackFilterTest.java +++ b/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/StackFilterTest.java @@ -1,5 +1,6 @@ package datadog.trace.agent.test.scopediag; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -29,6 +30,43 @@ void dropsPlumbingAndKeepsAppFrames() { assertEquals("com.app.Main", filtered[1].getClassName()); } + @Test + void matchesPlumbingClassesAndNestedClassesButNotNamePrefixes() { + String[] plumbingClasses = { + "datadog.trace.bootstrap.InstrumentationContext", + "java.util.concurrent.ThreadPoolExecutor", + "java.util.concurrent.ScheduledThreadPoolExecutor", + "java.util.concurrent.ForkJoinPool", + "java.util.concurrent.ForkJoinWorkerThread", + "java.util.concurrent.FutureTask", + "java.util.concurrent.CompletableFuture", + }; + for (String className : plumbingClasses) { + StackTraceElement unrelated = frame(className + "Holder", "run"); + StackTraceElement unrelatedNested = frame(className + "Holder$Worker", "run"); + StackTraceElement[] raw = { + frame(className, "run"), frame(className + "$Worker", "run"), unrelated, unrelatedNested, + }; + + assertArrayEquals( + new StackTraceElement[] {unrelated, unrelatedNested}, + new StackFilter(6).filter(raw), + className); + } + } + + @Test + void matchesStackCaptureMethodExactly() { + StackTraceElement otherMethod = frame("java.lang.Thread", "getStackTraceHelper"); + StackTraceElement otherClass = frame("java.lang.ThreadHelper", "getStackTrace"); + StackTraceElement[] raw = { + frame("java.lang.Thread", "getStackTrace"), otherMethod, otherClass, + }; + + assertArrayEquals( + new StackTraceElement[] {otherMethod, otherClass}, new StackFilter(6).filter(raw)); + } + @Test void respectsMaxFrames() { StackTraceElement[] raw = { From 9b23b489693fd43b098aafe38a63405ea3e54979 Mon Sep 17 00:00:00 2001 From: Andrea Marziali Date: Wed, 23 Sep 2026 13:55:47 +0200 Subject: [PATCH 21/35] Avoid false activation-after-resolution findings for overlapping resumes --- .../test/scopediag/ContinuationRecord.java | 15 ++------------- .../scopediag/ScopeDiagnosticsReportTest.java | 11 +++++++---- .../test/scopediag/ScopeResolutionTest.java | 18 ++++++++++++++++++ 3 files changed, 27 insertions(+), 17 deletions(-) diff --git a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ContinuationRecord.java b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ContinuationRecord.java index c0f99fe79eb..3758977f646 100644 --- a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ContinuationRecord.java +++ b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ContinuationRecord.java @@ -140,7 +140,8 @@ public synchronized EnumSet failures(Long rootWrittenNanos) { if (duplicateTerminalAttempt || !extraTerminals.isEmpty()) { failures.add(Failure.DOUBLE_FINISH); } - if ((terminal != null && !failedActivations.isEmpty()) || resumedAfterTerminal()) { + // Cleanup entry timestamps do not order resolution against concurrent successful resumes. + if (terminal != null && !failedActivations.isEmpty()) { failures.add(Failure.ACTIVATE_AFTER_RESOLVE); } if (rootWrittenNanos != null @@ -150,18 +151,6 @@ public synchronized EnumSet failures(Long rootWrittenNanos) { return failures; } - private boolean resumedAfterTerminal() { - if (terminal == null) { - return false; - } - for (ScopeEvent r : resumes) { - if (r.nanos > terminal.nanos) { - return true; - } - } - return false; - } - /** {@code true} when capture and any resume/terminal happened on different threads. */ public synchronized boolean threadHandoff() { if (capture == null) { diff --git a/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsReportTest.java b/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsReportTest.java index e09b84c58d7..232a7d18b2e 100644 --- a/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsReportTest.java +++ b/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeDiagnosticsReportTest.java @@ -122,16 +122,19 @@ void multipleResolutionsAreFlaggedDouble() { } @Test - void activationAfterResolveIsFailure() { + void successfulResumeAfterCleanupEntryIsNotActivationAfterResolve() { ContinuationRecord r = record(0, DDTraceId.from(14)); - r.setTerminalOrExtra(event(ScopeEvent.Type.RESOLVE_RELEASE, "pool-1", 2000)); + r.addResume(event(ScopeEvent.Type.ACTIVATE, "pool-1", 1500)); r.addResume(event(ScopeEvent.Type.ACTIVATE, "pool-2", 3000)); + r.setTerminalOrExtra(event(ScopeEvent.Type.RESOLVE_FINISH, "pool-1", 2000)); ScopeDiagnosticsReport report = report(list(r), map()); - assertEquals(1, report.activateAfterResolveCount()); + assertEquals(2, report.records().get(0).resumes().size()); + assertEquals(ContinuationStatus.FINISHED, report.records().get(0).status()); + assertEquals(0, report.activateAfterResolveCount()); assertEquals(0, report.doubleCount()); - assertTrue(report.hasProblems()); + assertFalse(report.hasProblems()); } @Test diff --git a/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeResolutionTest.java b/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeResolutionTest.java index e129095d371..103d503b769 100644 --- a/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeResolutionTest.java +++ b/dd-java-agent/instrumentation-testing/src/test/java/datadog/trace/agent/test/scopediag/ScopeResolutionTest.java @@ -58,6 +58,24 @@ void nestedReleaseBelongsToTheScopeClose() { assertEquals(ContinuationStatus.FINISHED, ScopeDiagnostics.report().records().get(0).status()); } + @Test + void delayedSuccessfulResumeCallbackAfterCleanupIsNotRejected() { + Object window = ScopeDiagnostics.recordingWindow(); + ScopeContinuationProbe.ResolveAttempt close = enter("cancelFromContinuedScopeClose", 1); + ScopeContinuationProbe.onResolveExit(close, CANCELLED); + long cleanupEntryNanos = ScopeDiagnostics.report().records().get(0).terminal().nanos; + + // The resume succeeded during cleanup, but its exit callback arrives after cleanup's callback. + ScopeDiagnostics.recordActivate( + window, continuation, DDTraceId.from(1), 2, "op", (byte) 0, cleanupEntryNanos + 1); + + ScopeDiagnosticsReport report = ScopeDiagnostics.report(); + assertEquals(1, report.records().get(0).resumes().size()); + assertEquals(ContinuationStatus.FINISHED, report.records().get(0).status()); + assertEquals(0, report.activateAfterResolveCount()); + assertResolvedOnce(); + } + @Test void overlappingLegitimateScopeClosesResolveOnce() { Object window = ScopeDiagnostics.recordingWindow(); From 2a87051b289893907f8323e45d0dcab1215dead9 Mon Sep 17 00:00:00 2001 From: Andrea Marziali Date: Wed, 23 Sep 2026 14:21:49 +0200 Subject: [PATCH 22/35] Enforce continuation diagnostics on smoke tests --- .../instrumentation-testing/build.gradle | 27 ++ .../ScopeContinuationTransformer.java | 101 ++++++-- .../agent/test/scopediag/StackFilter.java | 1 + .../scopediag/SmokeDiagnosticsBridge.java | 63 +++++ .../test/smoke/ScopeDiagnosticsAgent.java | 213 ++++++++++++++++ dd-smoke-tests/build.gradle | 11 +- .../smoketest/QuarkusNativeSmokeTest.groovy | 5 + .../smoketest/SampleTraceSmokeTest.groovy | 5 + ...SpringBootNativeInstrumentationTest.groovy | 5 + .../smoketest/AbstractServerSmokeTest.groovy | 5 + .../smoketest/AbstractSmokeTest.groovy | 1 + .../datadog/smoketest/ProcessManager.groovy | 88 ++++++- .../datadog/smoketest/AbstractSmokeApp.java | 98 ++++++-- .../smoketest/ScopeDiagnosticsClient.java | 231 +++++++++++++++++ .../java/datadog/smoketest/SmokeCliApp.java | 1 + .../ScopeDiagnosticsLegacySmokeTest.groovy | 29 +++ .../smoketest/ScopeDiagnosticsAgentTest.java | 127 ++++++++++ .../smoketest/ScopeDiagnosticsAppTest.java | 49 ++++ .../smoketest/ScopeDiagnosticsClientTest.java | 236 ++++++++++++++++++ .../smoketest/ScopeDiagnosticsTestApp.java | 39 +++ docs/how_to_smoke_test.md | 29 +++ 21 files changed, 1322 insertions(+), 42 deletions(-) create mode 100644 dd-java-agent/instrumentation-testing/src/smokeAgent/java/datadog/trace/agent/test/scopediag/SmokeDiagnosticsBridge.java create mode 100644 dd-java-agent/instrumentation-testing/src/smokeAgent/java/datadog/trace/agent/test/smoke/ScopeDiagnosticsAgent.java create mode 100644 dd-smoke-tests/src/main/java/datadog/smoketest/ScopeDiagnosticsClient.java create mode 100644 dd-smoke-tests/src/test/groovy/datadog/smoketest/ScopeDiagnosticsLegacySmokeTest.groovy create mode 100644 dd-smoke-tests/src/test/java/datadog/smoketest/ScopeDiagnosticsAgentTest.java create mode 100644 dd-smoke-tests/src/test/java/datadog/smoketest/ScopeDiagnosticsAppTest.java create mode 100644 dd-smoke-tests/src/test/java/datadog/smoketest/ScopeDiagnosticsClientTest.java create mode 100644 dd-smoke-tests/src/test/java/datadog/smoketest/ScopeDiagnosticsTestApp.java diff --git a/dd-java-agent/instrumentation-testing/build.gradle b/dd-java-agent/instrumentation-testing/build.gradle index d26aa9176ea..9b6a194c17f 100644 --- a/dd-java-agent/instrumentation-testing/build.gradle +++ b/dd-java-agent/instrumentation-testing/build.gradle @@ -2,6 +2,33 @@ plugins { id 'dd-trace-java.module.testing-support' } +sourceSets { + smokeAgent { + compileClasspath += sourceSets.main.output + sourceSets.main.compileClasspath + } +} + +tasks.register('scopeDiagnosticsAgentJar', Jar) { + archiveClassifier = 'scope-diagnostics-agent' + manifest { + attributes 'Premain-Class': 'datadog.trace.agent.test.smoke.ScopeDiagnosticsAgent', + 'Can-Retransform-Classes': 'true', 'Can-Redefine-Classes': 'true' + } + from(sourceSets.smokeAgent.output) { + include 'datadog/trace/agent/test/smoke/**' + } + into('payload') { + from(sourceSets.main.output) { + include 'datadog/trace/agent/test/scopediag/**' + exclude '**/ScopeDiagnosticsExtension*' + } + from(sourceSets.smokeAgent.output) { + include 'datadog/trace/agent/test/scopediag/**' + } + rename '(.*)\\.class', '$1.classdata' + } +} + dependencies { api libs.bytebuddy api libs.bytebuddyagent diff --git a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeContinuationTransformer.java b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeContinuationTransformer.java index b5e1c6544e1..8ae746d2a2b 100644 --- a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeContinuationTransformer.java +++ b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/ScopeContinuationTransformer.java @@ -7,10 +7,18 @@ import static net.bytebuddy.matcher.ElementMatchers.takesArguments; import java.lang.instrument.Instrumentation; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.atomic.AtomicReference; import net.bytebuddy.agent.ByteBuddyAgent; import net.bytebuddy.agent.builder.AgentBuilder; import net.bytebuddy.agent.builder.ResettableClassFileTransformer; import net.bytebuddy.asm.Advice; +import net.bytebuddy.description.type.TypeDescription; +import net.bytebuddy.dynamic.ClassFileLocator; +import net.bytebuddy.dynamic.DynamicType; +import net.bytebuddy.utility.JavaModule; /** * Installs test-only advice with a separate {@link AgentBuilder} because the tracer ignores its own @@ -55,30 +63,72 @@ final class ScopeContinuationTransformer { private ScopeContinuationTransformer() {} static synchronized void install() { + if (transformer == null) { + install( + ByteBuddyAgent.getInstrumentation(), + "datadog.trace.core", + ClassFileLocator.ForClassLoader.of(ScopeContinuationTransformer.class.getClassLoader())); + } + } + + static synchronized void install( + Instrumentation instrumentation, String corePackage, ClassFileLocator locator) { if (transformer != null) { return; } try { // Related core types can otherwise load these targets reentrantly while they are transformed. ClassLoader loader = ScopeContinuationTransformer.class.getClassLoader(); - Class.forName("datadog.trace.core.scopemanager.ScopeContinuation", false, loader); - Class.forName("datadog.trace.core.scopemanager.ScopeStack", false, loader); + for (String target : + new String[] { + ".scopemanager.ScopeContinuation", + ".scopemanager.ScopeStack", + ".scopemanager.ContinuableScope", + ".scopemanager.ContinuableScopeManager", + ".PendingTrace" + }) { + Class.forName(corePackage + target, false, loader); + } } catch (ClassNotFoundException missingCoreTracer) { throw new IllegalStateException( "Scope continuation diagnostics require dd-trace-core", missingCoreTracer); } - Instrumentation instrumentation = ByteBuddyAgent.getInstrumentation(); + AtomicReference failure = new AtomicReference<>(); + Set transformed = Collections.synchronizedSet(new HashSet()); transformer = new AgentBuilder.Default() + .with( + new AgentBuilder.Listener.Adapter() { + @Override + public void onTransformation( + TypeDescription type, + ClassLoader loader, + JavaModule module, + boolean loaded, + DynamicType dynamicType) { + transformed.add(type.getName()); + } + + @Override + public void onError( + String typeName, + ClassLoader loader, + JavaModule module, + boolean loaded, + Throwable error) { + failure.compareAndSet( + null, new IllegalStateException("Cannot instrument " + typeName, error)); + } + }) .disableClassFormatChanges() .with(AgentBuilder.RedefinitionStrategy.RETRANSFORMATION) .with(AgentBuilder.TypeStrategy.Default.REDEFINE) - .type(named("datadog.trace.core.scopemanager.ScopeContinuation")) + .type(named(corePackage + ".scopemanager.ScopeContinuation")) .transform( (builder, type, classLoader, module, pd) -> builder .visit( - Advice.to(ContinuationAdvice.Register.class) + Advice.to(ContinuationAdvice.Register.class, locator) .on( isMethod() .and(named("register")) @@ -86,16 +136,17 @@ static synchronized void install() { .and( returns( named( - "datadog.trace.core.scopemanager.ScopeContinuation"))))) + corePackage + + ".scopemanager.ScopeContinuation"))))) .visit( - Advice.to(ContinuationAdvice.Activate.class) + Advice.to(ContinuationAdvice.Activate.class, locator) .on( isMethod() .and(named("resume")) .and(takesArguments(0)) .and(returns(named("datadog.context.ContextScope"))))) .visit( - Advice.to(ContinuationAdvice.Cancel.class) + Advice.to(ContinuationAdvice.Cancel.class, locator) .on( isMethod() .and( @@ -103,39 +154,39 @@ static synchronized void install() { .or(named("cancelFromContinuedScopeClose"))) .and(takesArguments(0)) .and(returns(void.class))))) - .type(named("datadog.trace.core.PendingTrace")) + .type(named(corePackage + ".PendingTrace")) .transform( (builder, type, classLoader, module, pd) -> builder.visit( - Advice.to(PendingTraceAdvice.Write.class) + Advice.to(PendingTraceAdvice.Write.class, locator) .on( isMethod() .and(named("write")) .and(takesArguments(boolean.class)) .and(returns(int.class))))) - .type(named("datadog.trace.core.scopemanager.ContinuableScope")) + .type(named(corePackage + ".scopemanager.ContinuableScope")) .transform( (builder, type, classLoader, module, pd) -> builder .visit( - Advice.to(ContinuableScopeAdvice.OnProperClose.class) + Advice.to(ContinuableScopeAdvice.OnProperClose.class, locator) .on( isMethod() .and(named("onProperClose")) .and(takesArguments(0)) .and(returns(void.class)))) .visit( - Advice.to(ContinuableScopeAdvice.Close.class) + Advice.to(ContinuableScopeAdvice.Close.class, locator) .on( isMethod() .and(named("close")) .and(takesArguments(0)) .and(returns(void.class))))) - .type(named("datadog.trace.core.scopemanager.ScopeStack")) + .type(named(corePackage + ".scopemanager.ScopeStack")) .transform( (builder, type, classLoader, module, pd) -> builder.visit( - Advice.to(ScopeStackAdvice.Push.class) + Advice.to(ScopeStackAdvice.Push.class, locator) .on( isMethod() .and(named("push")) @@ -143,27 +194,33 @@ static synchronized void install() { .and( takesArgument( 0, - named( - "datadog.trace.core.scopemanager.ContinuableScope"))) + named(corePackage + ".scopemanager.ContinuableScope"))) .and(returns(void.class))))) - .type(named("datadog.trace.core.scopemanager.ContinuableScopeManager")) + .type(named(corePackage + ".scopemanager.ContinuableScopeManager")) .transform( (builder, type, classLoader, module, pd) -> builder.visit( - Advice.to(ContinuableScopeManagerAdvice.ScheduleRootIterationCleanup.class) + Advice.to( + ContinuableScopeManagerAdvice.ScheduleRootIterationCleanup.class, + locator) .on( isMethod() .and(named("scheduleRootIterationScopeCleanup")) .and(takesArguments(2)) .and( takesArgument( - 0, named("datadog.trace.core.scopemanager.ScopeStack"))) + 0, named(corePackage + ".scopemanager.ScopeStack"))) .and( takesArgument( 1, - named( - "datadog.trace.core.scopemanager.ContinuableScope"))) + named(corePackage + ".scopemanager.ContinuableScope"))) .and(returns(void.class))))) .installOn(instrumentation); + if (failure.get() != null || transformed.size() != 5) { + transformer.reset(instrumentation, AgentBuilder.RedefinitionStrategy.RETRANSFORMATION); + transformer = null; + throw new IllegalStateException( + "Scope diagnostics installation failed; transformed " + transformed, failure.get()); + } } } diff --git a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/StackFilter.java b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/StackFilter.java index fbc39593188..aed6d84d850 100644 --- a/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/StackFilter.java +++ b/dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/scopediag/StackFilter.java @@ -8,6 +8,7 @@ final class StackFilter { private static final String[] DROP_PREFIXES = { "datadog.trace.agent.test.scopediag.", "datadog.trace.core.", + "datadog.trace.agent.core.", "datadog.trace.bootstrap.instrumentation.java.concurrent.", "datadog.trace.bootstrap.instrumentation.api.", "java.util.concurrent.Executors$", diff --git a/dd-java-agent/instrumentation-testing/src/smokeAgent/java/datadog/trace/agent/test/scopediag/SmokeDiagnosticsBridge.java b/dd-java-agent/instrumentation-testing/src/smokeAgent/java/datadog/trace/agent/test/scopediag/SmokeDiagnosticsBridge.java new file mode 100644 index 00000000000..d1e5d57095b --- /dev/null +++ b/dd-java-agent/instrumentation-testing/src/smokeAgent/java/datadog/trace/agent/test/scopediag/SmokeDiagnosticsBridge.java @@ -0,0 +1,63 @@ +package datadog.trace.agent.test.scopediag; + +import java.lang.instrument.Instrumentation; +import java.util.Map; +import java.util.Properties; +import net.bytebuddy.dynamic.ClassFileLocator; + +/** Runs in the tracer's loader so all recorded objects retain their actual type identity. */ +public final class SmokeDiagnosticsBridge { + private static Properties completed; + private static boolean recording; + + private SmokeDiagnosticsBridge() {} + + public static void install(Instrumentation instrumentation, Map classes) { + ScopeContinuationTransformer.install( + instrumentation, "datadog.trace.agent.core", new ClassFileLocator.Simple(classes)); + } + + public static synchronized void start() { + if (recording) { + throw new IllegalStateException("Scope diagnostics are already recording"); + } + completed = null; + ScopeDiagnostics.startRecording(); + recording = true; + } + + public static synchronized Properties finish() { + if (completed != null) { + return completed; + } + if (!recording) { + throw new IllegalStateException("Scope diagnostics have not started recording"); + } + ScopeDiagnostics.awaitQuiescence(); + ScopeDiagnostics.stop(); + recording = false; + ScopeDiagnosticsReport report = ScopeDiagnostics.report(); + Properties result = new Properties(); + result.setProperty("status", report.hasProblems() ? "problems" : "ok"); + result.setProperty("detail", report.renderSummary()); + result.setProperty("timeline", report.renderTimeline()); + long events = 0; + for (ContinuationRecord record : report.records()) { + events += + (record.capture() == null ? 0 : 1) + + record.resumes().size() + + record.failedActivations().size() + + (record.terminal() == null ? 0 : 1) + + record.extraTerminals().size(); + } + for (ScopeRecord record : report.scopeRecords()) { + events += + (record.open() == null ? 0 : 1) + + (record.close() == null ? 0 : 1) + + record.wrongThreadCloses().size(); + } + result.setProperty("eventCount", Long.toString(events)); + completed = result; + return result; + } +} diff --git a/dd-java-agent/instrumentation-testing/src/smokeAgent/java/datadog/trace/agent/test/smoke/ScopeDiagnosticsAgent.java b/dd-java-agent/instrumentation-testing/src/smokeAgent/java/datadog/trace/agent/test/smoke/ScopeDiagnosticsAgent.java new file mode 100644 index 00000000000..44dc5609d45 --- /dev/null +++ b/dd-java-agent/instrumentation-testing/src/smokeAgent/java/datadog/trace/agent/test/smoke/ScopeDiagnosticsAgent.java @@ -0,0 +1,213 @@ +package datadog.trace.agent.test.smoke; + +import java.io.ByteArrayOutputStream; +import java.io.InputStream; +import java.io.OutputStream; +import java.lang.instrument.Instrumentation; +import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; +import java.util.Enumeration; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Properties; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; + +/** File-controlled companion agent used only by smoke-test child processes. */ +public final class ScopeDiagnosticsAgent { + private ScopeDiagnosticsAgent() {} + + public static void premain(String arguments, Instrumentation instrumentation) throws Exception { + if (arguments == null || arguments.isEmpty()) { + throw new IllegalArgumentException("Scope diagnostics require a control directory"); + } + Path directory = Paths.get(arguments).toAbsolutePath(); + try { + Properties config = read(directory.resolve("config")); + String mode = config.getProperty("mode"); + if (!"server".equals(mode) && !"cli".equals(mode)) { + throw new IllegalArgumentException("Unknown scope diagnostics mode: " + mode); + } + Class agent = Class.forName("datadog.trace.bootstrap.Agent", false, null); + Field field = agent.getDeclaredField("AGENT_CLASSLOADER"); + field.setAccessible(true); + ClassLoader loader = (ClassLoader) field.get(null); + if (loader == null + || !loader.getClass().getName().equals("datadog.trace.bootstrap.DatadogClassLoader")) { + throw new IllegalStateException("The Datadog agent must precede the diagnostics companion"); + } + Map payload = payload(); + Class injector = loader.loadClass("datadog.instrument.classinject.ClassInjector"); + injector + .getMethod("injectClasses", Map.class, ClassLoader.class) + .invoke(null, payload, loader); + Class bridge = + loader.loadClass("datadog.trace.agent.test.scopediag.SmokeDiagnosticsBridge"); + bridge + .getMethod("install", Instrumentation.class, Map.class) + .invoke(null, instrumentation, payload); + Method start = bridge.getMethod("start"); + Method finish = bridge.getMethod("finish"); + if ("cli".equals(mode)) { + start.invoke(null); + Runtime.getRuntime() + .addShutdownHook( + new Thread( + () -> { + try { + publish( + directory.resolve("final"), + response(0, (Properties) finish.invoke(null))); + } catch (Throwable failure) { + publishFailure(directory, "final", 0, failure); + } + }, + "scope-diagnostics-shutdown")); + } + Thread controller = + new Thread( + () -> control(directory, start, finish, "cli".equals(mode)), + "scope-diagnostics-control"); + controller.setDaemon(true); + controller.start(); + Properties ready = new Properties(); + ready.setProperty("protocol", "1"); + publish(directory.resolve("ready"), ready); + } catch (Throwable failure) { + publishFailure(directory, "error", 0, failure); + throw new IllegalStateException( + "Cannot install smoke-test scope diagnostics", unwrap(failure)); + } + } + + private static void control(Path directory, Method start, Method finish, boolean cli) { + long previous = Long.MIN_VALUE; + while (!Thread.currentThread().isInterrupted()) { + try { + Path commandFile = directory.resolve("command"); + if (Files.exists(commandFile)) { + Properties command = read(commandFile); + long seq = Long.parseLong(command.getProperty("seq")); + if (seq != previous) { + previous = seq; + try { + Properties result; + String action = command.getProperty("action"); + if ("start".equals(action)) { + if (cli) { + throw new IllegalStateException("CLI diagnostics record one process lifetime"); + } + start.invoke(null); + result = new Properties(); + result.setProperty("status", "ok"); + result.setProperty("detail", "Recording started"); + result.setProperty("timeline", ""); + result.setProperty("eventCount", "0"); + } else if ("finish".equals(action)) { + result = (Properties) finish.invoke(null); + } else { + throw new IllegalArgumentException("Unknown diagnostic command: " + action); + } + publish(directory.resolve("response-" + seq), response(seq, result)); + } catch (Throwable failure) { + publishFailure(directory, "response-" + seq, seq, failure); + } + } + } + Thread.sleep(20); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + } catch (Throwable failure) { + publishFailure(directory, "error", previous, failure); + return; + } + } + } + + private static Map payload() throws Exception { + Path ownJar = + Paths.get( + ScopeDiagnosticsAgent.class + .getProtectionDomain() + .getCodeSource() + .getLocation() + .toURI()); + Map classes = new LinkedHashMap<>(); + try (JarFile jar = new JarFile(ownJar.toFile())) { + Enumeration entries = jar.entries(); + while (entries.hasMoreElements()) { + JarEntry entry = entries.nextElement(); + String name = entry.getName(); + if (!name.startsWith("payload/") || !name.endsWith(".classdata")) { + continue; + } + try (InputStream input = jar.getInputStream(entry)) { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + byte[] buffer = new byte[8192]; + int count; + while ((count = input.read(buffer)) != -1) { + bytes.write(buffer, 0, count); + } + classes.put(name.substring(8, name.length() - 10).replace('/', '.'), bytes.toByteArray()); + } + } + } + if (classes.isEmpty()) { + throw new IllegalStateException("Missing diagnostic payload"); + } + return classes; + } + + private static Properties read(Path path) throws Exception { + Properties properties = new Properties(); + try (InputStream input = Files.newInputStream(path)) { + properties.load(input); + } + return properties; + } + + private static Properties response(long seq, Properties data) { + Properties result = new Properties(); + result.putAll(data); + result.setProperty("protocol", "1"); + result.setProperty("seq", Long.toString(seq)); + return result; + } + + private static void publish(Path target, Properties data) throws Exception { + Path temporary = Files.createTempFile(target.getParent(), ".scope-diagnostics-", ".tmp"); + try { + try (OutputStream output = Files.newOutputStream(temporary)) { + data.store(output, "Smoke-test scope diagnostics"); + } + Files.move( + temporary, target, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + } finally { + Files.deleteIfExists(temporary); + } + } + + private static void publishFailure(Path directory, String file, long seq, Throwable failure) { + Properties result = new Properties(); + result.setProperty("status", "error"); + result.setProperty("detail", unwrap(failure).toString()); + try { + publish(directory.resolve(file), response(seq, result)); + } catch (Exception writingFailure) { + System.err.println("Cannot publish scope diagnostic failure: " + writingFailure); + failure.printStackTrace(System.err); + } + } + + private static Throwable unwrap(Throwable failure) { + while (failure instanceof InvocationTargetException && failure.getCause() != null) { + failure = failure.getCause(); + } + return failure; + } +} diff --git a/dd-smoke-tests/build.gradle b/dd-smoke-tests/build.gradle index cb067249d46..4869c2e8e22 100644 --- a/dd-smoke-tests/build.gradle +++ b/dd-smoke-tests/build.gradle @@ -36,21 +36,26 @@ allprojects { } } -subprojects { Project subProj -> +allprojects { Project subProj -> // Don't need javadoc task run for internal projects. subProj.tasks.withType(Javadoc).configureEach { enabled = false } subProj.evaluationDependsOn ':dd-java-agent' + subProj.evaluationDependsOn ':dd-java-agent:instrumentation-testing' subProj.tasks.withType(Test).configureEach { subTask -> def agentTaskRef = project(':dd-java-agent').tasks.named('shadowJar', ShadowJar) - dependsOn agentTaskRef + def diagnosticsTaskRef = project(':dd-java-agent:instrumentation-testing').tasks.named('scopeDiagnosticsAgentJar', Jar) + dependsOn agentTaskRef, diagnosticsTaskRef // Tests depend on this to know where to run things and what agent jar to use jvmArgs "-Ddatadog.smoketest.builddir=${buildDir}" jvmArgumentProviders.add(new CommandLineArgumentProvider() { @Override Iterable asArguments() { - return agentTaskRef.map { ["-Ddatadog.smoketest.agent.shadowJar.path=${it.archiveFile.get()}"] }.get() + return [ + "-Ddatadog.smoketest.agent.shadowJar.path=${agentTaskRef.get().archiveFile.get()}", + "-Ddatadog.smoketest.scopeDiagnostics.agent.path=${diagnosticsTaskRef.get().archiveFile.get()}" + ] } }) } diff --git a/dd-smoke-tests/quarkus-native/src/test/groovy/datadog/smoketest/QuarkusNativeSmokeTest.groovy b/dd-smoke-tests/quarkus-native/src/test/groovy/datadog/smoketest/QuarkusNativeSmokeTest.groovy index 665f1f51e01..053f2971976 100644 --- a/dd-smoke-tests/quarkus-native/src/test/groovy/datadog/smoketest/QuarkusNativeSmokeTest.groovy +++ b/dd-smoke-tests/quarkus-native/src/test/groovy/datadog/smoketest/QuarkusNativeSmokeTest.groovy @@ -9,6 +9,11 @@ import java.util.regex.Pattern abstract class QuarkusNativeSmokeTest extends AbstractServerSmokeTest { + @Override + protected String skipScopeContinuationCheckReason() { + 'Native images cannot load the JVM diagnostic companion agent or retransform tracer classes.' + } + @Override ProcessBuilder createProcessBuilder() { String quarkusNativeExecutable = System.getProperty('datadog.smoketest.quarkus.native.executable') diff --git a/dd-smoke-tests/sample-trace/src/test/groovy/datadog/smoketest/SampleTraceSmokeTest.groovy b/dd-smoke-tests/sample-trace/src/test/groovy/datadog/smoketest/SampleTraceSmokeTest.groovy index 791f9aed1c2..e904e1e403f 100644 --- a/dd-smoke-tests/sample-trace/src/test/groovy/datadog/smoketest/SampleTraceSmokeTest.groovy +++ b/dd-smoke-tests/sample-trace/src/test/groovy/datadog/smoketest/SampleTraceSmokeTest.groovy @@ -5,6 +5,11 @@ import datadog.trace.test.util.Flaky class SampleTraceSmokeTest extends AbstractSmokeTest { + @Override + protected String skipScopeContinuationCheckReason() { + 'Exercises the agent JAR as a CLI without premain; the companion requires an installed Java agent.' + } + @Override ProcessBuilder createProcessBuilder() { List command = new ArrayList<>() diff --git a/dd-smoke-tests/spring-boot-3.0-native/src/test/groovy/SpringBootNativeInstrumentationTest.groovy b/dd-smoke-tests/spring-boot-3.0-native/src/test/groovy/SpringBootNativeInstrumentationTest.groovy index 80c5b3a2443..01aed969b7b 100644 --- a/dd-smoke-tests/spring-boot-3.0-native/src/test/groovy/SpringBootNativeInstrumentationTest.groovy +++ b/dd-smoke-tests/spring-boot-3.0-native/src/test/groovy/SpringBootNativeInstrumentationTest.groovy @@ -29,6 +29,11 @@ import java.util.concurrent.TimeoutException import java.util.concurrent.atomic.AtomicInteger class SpringBootNativeInstrumentationTest extends AbstractServerSmokeTest { + @Override + protected String skipScopeContinuationCheckReason() { + 'Native images cannot load the JVM diagnostic companion agent or retransform tracer classes.' + } + @Shared @TempDir def testJfrDir diff --git a/dd-smoke-tests/src/main/groovy/datadog/smoketest/AbstractServerSmokeTest.groovy b/dd-smoke-tests/src/main/groovy/datadog/smoketest/AbstractServerSmokeTest.groovy index bb1b90304d4..68b225a5cbd 100644 --- a/dd-smoke-tests/src/main/groovy/datadog/smoketest/AbstractServerSmokeTest.groovy +++ b/dd-smoke-tests/src/main/groovy/datadog/smoketest/AbstractServerSmokeTest.groovy @@ -14,6 +14,11 @@ import static org.junit.jupiter.api.Assumptions.assumeTrue abstract class AbstractServerSmokeTest extends AbstractSmokeTest { + @Override + protected boolean scopeDiagnosticsProcessLifetime() { + false + } + @Shared protected int[] httpPorts = (0.. scopeDiagnostics = [] + + /** Return a documented incompatibility; disabling log checks does not disable diagnostics. */ + protected String skipScopeContinuationCheckReason() { + null + } + + /** Plain smoke applications may finish before the first feature starts. */ + protected boolean scopeDiagnosticsProcessLifetime() { + true + } + public static final PROFILING_START_DELAY_SECONDS = 1 public static final int PROFILING_RECORDING_UPLOAD_PERIOD_SECONDS = 5 public static final String SERVICE_NAME = "smoke-test-java-app" @@ -69,8 +84,11 @@ abstract class ProcessManager extends Specification { def logFilePath = logFilePaths.length > 0 ? logFilePaths[0] : null def setup() { - testedProcesses.each { - assert it.alive: "Process $it is not available on test beginning" + testedProcesses.eachWithIndex { process, idx -> + if (!scopeDiagnosticsProcessLifetime()) { + assert process.alive: "Process $process is not available on test beginning" + scopeDiagnostics[idx]?.start(process) + } } synchronized (outputThreads.testLogMessages) { @@ -78,6 +96,32 @@ abstract class ProcessManager extends Specification { } } + def cleanup() { + if (!scopeDiagnosticsProcessLifetime()) { + checkScopeDiagnostics() + } + } + + private void checkScopeDiagnostics() { + Throwable firstFailure = null + scopeDiagnostics.eachWithIndex { diagnostic, idx -> + if (diagnostic != null && testedProcesses[idx] != null) { + try { + diagnostic.finish(testedProcesses[idx]) + } catch (Throwable failure) { + if (firstFailure == null) { + firstFailure = failure + } else { + firstFailure.addSuppressed(failure) + } + } + } + } + if (firstFailure != null) { + throw firstFailure + } + } + @Shared @AutoCleanup OutputThreads outputThreads = new OutputThreads() @@ -92,6 +136,7 @@ abstract class ProcessManager extends Specification { (0.. ProcessBuilder processBuilder = createProcessBuilder(idx) + configureScopeDiagnostics(processBuilder, idx) Map env = processBuilder.environment() env.put("JAVA_HOME", System.getProperty("java.home")) @@ -104,6 +149,7 @@ abstract class ProcessManager extends Specification { testedProcesses[idx] = p outputThreads.captureOutput(p, new File(logFilePaths[idx])) + scopeDiagnostics[idx]?.awaitReady(p) } testedProcess = numberOfProcesses == 1 ? testedProcesses[0] : null @@ -127,6 +173,13 @@ abstract class ProcessManager extends Specification { def cleanupSpec() { Throwable firstFailure = null + if (scopeDiagnosticsProcessLifetime()) { + try { + checkScopeDiagnostics() + } catch (Throwable failure) { + firstFailure = failure + } + } testedProcesses.each { tp -> if (tp == null) { return // closure continue — skip null slots @@ -168,6 +221,37 @@ abstract class ProcessManager extends Specification { } } + protected final void configureScopeDiagnostics(ProcessBuilder builder, int index) { + String reason = skipScopeContinuationCheckReason() + if (reason != null && reason.trim().isEmpty()) { + throw new IllegalArgumentException('Disabling scope continuation diagnostics requires a reason') + } + ScopeDiagnosticsClient diagnostic = reason == null + ? new ScopeDiagnosticsClient(Paths.get(buildDirectory, 'reports'), "${getClass().simpleName}-${index}", scopeDiagnosticsProcessLifetime()) + : null + scopeDiagnostics.add(diagnostic) + String argument = diagnostic == null ? '' : diagnostic.javaAgentArgument() + boolean found = false + builder.command().replaceAll { String value -> + if (value.contains(SCOPE_DIAGNOSTICS_ARGUMENT)) { + found = true + return value.replace(SCOPE_DIAGNOSTICS_ARGUMENT, argument) + } + value + } + builder.command().removeAll { it.isEmpty() } + builder.environment().replaceAll { String key, String value -> + if (value.contains(SCOPE_DIAGNOSTICS_ARGUMENT)) { + found = true + return value.replace(SCOPE_DIAGNOSTICS_ARGUMENT, argument) + } + value + } + if (diagnostic != null && !found) { + throw new IllegalStateException('Smoke launch must include defaultJavaProperties, or document an incompatible launch with skipScopeContinuationCheckReason()') + } + } + def getProfilingUrl() { if (profilingPort == -1) { profilingPort = PortUtils.randomOpenPort() diff --git a/dd-smoke-tests/src/main/java/datadog/smoketest/AbstractSmokeApp.java b/dd-smoke-tests/src/main/java/datadog/smoketest/AbstractSmokeApp.java index 508d8d7e8b3..f649e2031eb 100644 --- a/dd-smoke-tests/src/main/java/datadog/smoketest/AbstractSmokeApp.java +++ b/dd-smoke-tests/src/main/java/datadog/smoketest/AbstractSmokeApp.java @@ -99,11 +99,13 @@ public abstract class AbstractSmokeApp private final boolean checkTelemetry; private final boolean applyMemoryTuning; private final boolean debugLogs; + private final boolean checkScopeContinuations; private final OutputThreads outputThreads = new OutputThreads(); private Process process; private File logFile; private boolean telemetryChecked; + private ScopeDiagnosticsClient scopeDiagnostics; protected AbstractSmokeApp(Builder builder) { this.name = builder.name; @@ -123,6 +125,7 @@ protected AbstractSmokeApp(Builder builder) { this.checkTelemetry = builder.checkTelemetry; this.applyMemoryTuning = builder.applyMemoryTuning; this.debugLogs = builder.debugLogs; + this.checkScopeContinuations = builder.scopeContinuationSkipReason == null; this.errorLogFilter = builder.errorLogFilter != null ? builder.errorLogFilter @@ -233,17 +236,33 @@ protected final void registerPlaceholder(String token, Supplier valueSup public final void beforeAll(ExtensionContext context) throws Exception { this.backend.start(); launch(); + if (this.scopeDiagnostics != null) { + this.scopeDiagnostics.awaitReady(this.process); + } onStarted(); } @Override public final void beforeEach(ExtensionContext context) { onBeforeEach(); + if (this.scopeDiagnostics != null) { + this.scopeDiagnostics.start(this.process); + } } @Override public final void afterEach(ExtensionContext context) { - onAfterEach(); + Throwable failure = null; + try { + onAfterEach(); + } catch (Throwable problem) { + failure = problem; + } + // A CLI's process-wide recording must survive methods while it is still running. + if (this.scopeDiagnostics != null + && (!(this instanceof SmokeCliApp) || !this.process.isAlive())) { + failure = attempt(failure, () -> this.scopeDiagnostics.finish(this.process)); + } // Check telemetry once, here, while the app and backend are still up as afterAll is too late // (the app is killed, and the per-method session clear may have wiped a once-only app-started). // Only for an agent-instrumented app on an owned backend (a shared session mixes apps). @@ -252,26 +271,57 @@ public final void afterEach(ExtensionContext context) { && !this.backend.isShared() && !this.telemetryChecked) { this.telemetryChecked = true; - assertTelemetryReceived(); + failure = attempt(failure, this::assertTelemetryReceived); } + rethrow(failure); } @Override public final void afterAll(ExtensionContext context) { + Throwable failure = null; + if (this.scopeDiagnostics != null && this.process != null) { + failure = attempt(failure, () -> this.scopeDiagnostics.verifyCompleted(this.process)); + } + failure = attempt(failure, this::stopProcess); + // Flush captured output before scanning logs, even when diagnostics failed. + failure = attempt(failure, this.outputThreads::close); + if (!this.backend.isShared()) { + failure = attempt(failure, this.backend::close); + } + if (this.checkErrorLogs) { + failure = attempt(failure, this::assertNoErrorLogs); + } + rethrow(failure); + } + + private static Throwable attempt(Throwable first, Runnable action) { try { - stopProcess(); - } finally { - // Join the output threads first so the log file is fully flushed before we scan it. - this.outputThreads.close(); - try { - if (!this.backend.isShared()) { - this.backend.close(); - } - } finally { - if (this.checkErrorLogs) { - assertNoErrorLogs(); - } + action.run(); + } catch (Throwable problem) { + if (first == null) { + return problem; } + first.addSuppressed(problem); + } + return first; + } + + private static void rethrow(Throwable failure) { + if (failure instanceof Error) { + throw (Error) failure; + } + if (failure instanceof RuntimeException) { + throw (RuntimeException) failure; + } + if (failure != null) { + throw new IllegalStateException(failure); + } + } + + /** Validates the companion's completed CLI report after an explicit process-exit assertion. */ + protected final void verifyScopeDiagnostics() { + if (this.scopeDiagnostics != null) { + this.scopeDiagnostics.verifyCompleted(this.process); } } @@ -291,6 +341,12 @@ protected void onBeforeEach() {} protected void onAfterEach() {} private void launch() throws IOException { + this.logFile = resolveLogFile(); + if (this.checkScopeContinuations && this.agentJar != null) { + this.scopeDiagnostics = + new ScopeDiagnosticsClient( + this.logFile.getParentFile().toPath(), this.name, this instanceof SmokeCliApp); + } List command = javaCommand(); appendMemoryTuningArguments(command); appendLogsArguments(command); @@ -308,7 +364,6 @@ private void launch() throws IOException { env.putAll(this.extraEnv); processBuilder.redirectErrorStream(true); - this.logFile = resolveLogFile(); this.process = processBuilder.start(); this.outputThreads.captureOutput(this.process, this.logFile); } @@ -346,6 +401,9 @@ private void appendLogsArguments(List command) { private void appendAgentArguments(List command) { if (this.agentJar != null) { command.add("-javaagent:" + this.agentJar); + if (this.scopeDiagnostics != null) { + command.add(this.scopeDiagnostics.javaAgentArgument()); + } command.add("-Ddd.agent.host=" + this.backend.url().getHost()); command.add("-Ddd.trace.agent.port=" + this.backend.port()); command.add("-Ddd.service.name=" + SERVICE_NAME); @@ -530,6 +588,7 @@ public abstract static class Builder= deadline) { + throw failure("Timed out waiting for diagnostic " + file); + } + Thread.sleep(10); + } catch (IOException e) { + throw failure("Cannot read diagnostic " + file + ": " + e); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw failure("Interrupted waiting for diagnostic " + file); + } + } + } + + private void validateResponse(Properties response, long id) { + validateProtocol(response); + if (!Long.toString(id).equals(response.getProperty("seq"))) { + throw failure("Diagnostic response has missing or incorrect sequence: " + response); + } + String status = response.getProperty("status"); + if (!"ok".equals(status) && !"problems".equals(status) && !"error".equals(status)) { + throw failure("Diagnostic response has invalid status: " + response); + } + if (!"error".equals(status)) { + try { + if (Long.parseLong(response.getProperty("eventCount")) < 0 + || response.getProperty("detail") == null + || response.getProperty("timeline") == null) { + throw new IllegalArgumentException(); + } + } catch (IllegalArgumentException e) { + throw failure("Diagnostic response is incomplete: " + response); + } + } + if (!"ok".equals(status)) { + throw failure( + response.getProperty("detail", status) + "\n" + response.getProperty("timeline", "")); + } + } + + private void validateProtocol(Properties properties) { + if (!"1".equals(properties.getProperty("protocol"))) { + throw failure("Missing or unsupported diagnostic protocol: " + properties); + } + } + + private Properties read(String name) throws IOException { + Properties properties = new Properties(); + try (InputStream input = Files.newInputStream(directory.resolve(name))) { + properties.load(input); + } + return properties; + } + + private void write(String name, Properties properties) throws IOException { + Path temporary = Files.createTempFile(directory, name, ".tmp"); + try { + try (OutputStream output = Files.newOutputStream(temporary)) { + properties.store(output, null); + } + Files.move(temporary, directory.resolve(name), ATOMIC_MOVE, REPLACE_EXISTING); + } finally { + Files.deleteIfExists(temporary); + } + } + + private AssertionError failure(String message) { + return new AssertionError(message + "\nScope diagnostics: " + directory); + } + + private static Path configuredAgent() { + String agent = System.getProperty(AGENT_PROPERTY); + if (agent == null || agent.trim().isEmpty()) { + throw new IllegalStateException("Missing companion agent property: " + AGENT_PROPERTY); + } + return Paths.get(agent); + } +} diff --git a/dd-smoke-tests/src/main/java/datadog/smoketest/SmokeCliApp.java b/dd-smoke-tests/src/main/java/datadog/smoketest/SmokeCliApp.java index 6c2ac7509f3..bd715199e5a 100644 --- a/dd-smoke-tests/src/main/java/datadog/smoketest/SmokeCliApp.java +++ b/dd-smoke-tests/src/main/java/datadog/smoketest/SmokeCliApp.java @@ -64,6 +64,7 @@ public void assertCompletesWithValue(long timeout, TimeUnit unit, int expectedEx throw new AssertionError( "App '" + name() + "' exited with " + actual + " but expected " + expectedExitValue); } + verifyScopeDiagnostics(); } /** Fluent builder for a {@link SmokeCliApp}; obtain via {@link SmokeCliApp#named(String)}. */ diff --git a/dd-smoke-tests/src/test/groovy/datadog/smoketest/ScopeDiagnosticsLegacySmokeTest.groovy b/dd-smoke-tests/src/test/groovy/datadog/smoketest/ScopeDiagnosticsLegacySmokeTest.groovy new file mode 100644 index 00000000000..dce33fbc6c1 --- /dev/null +++ b/dd-smoke-tests/src/test/groovy/datadog/smoketest/ScopeDiagnosticsLegacySmokeTest.groovy @@ -0,0 +1,29 @@ +package datadog.smoketest + +import java.nio.file.Paths +import java.util.concurrent.TimeUnit + +/** Exercises companion injection through the legacy smoke-test launch properties. */ +class ScopeDiagnosticsLegacySmokeTest extends AbstractSmokeTest { + @Override + ProcessBuilder createProcessBuilder() { + String classes = Paths.get(ScopeDiagnosticsTestApp.protectionDomain.codeSource.location.toURI()).toString() + new ProcessBuilder([ + javaPath(), + *defaultJavaProperties, + '-Ddd.profiling.enabled=false', + '-Ddd.telemetry.enabled=false', + '-Ddd.remote_config.enabled=false', + '-cp', + classes, + ScopeDiagnosticsTestApp.name, + 'resolved' + ] as String[]) + } + + def 'completed CLI is checked even when it exits before the feature'() { + expect: + testedProcess.waitFor(30, TimeUnit.SECONDS) + testedProcess.exitValue() == 0 + } +} diff --git a/dd-smoke-tests/src/test/java/datadog/smoketest/ScopeDiagnosticsAgentTest.java b/dd-smoke-tests/src/test/java/datadog/smoketest/ScopeDiagnosticsAgentTest.java new file mode 100644 index 00000000000..0b0ded53dd9 --- /dev/null +++ b/dd-smoke-tests/src/test/java/datadog/smoketest/ScopeDiagnosticsAgentTest.java @@ -0,0 +1,127 @@ +package datadog.smoketest; + +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStreamWriter; +import java.io.Writer; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Properties; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.api.io.TempDir; + +@Timeout(60) +class ScopeDiagnosticsAgentTest { + @TempDir Path reports; + + @Test + void observesActualShadedTracerInQuickCli() throws Exception { + ScopeDiagnosticsClient diagnostics = new ScopeDiagnosticsClient(reports, "resolved", true); + Process child = launch(diagnostics, "resolved"); + try { + diagnostics.awaitReady(child); + assertTrue(child.waitFor(30, SECONDS)); + assertEquals(0, child.exitValue()); + diagnostics.verifyCompleted(child); + Properties result = result(diagnostics.directory().resolve("final")); + assertTrue(Integer.parseInt(result.getProperty("eventCount")) > 0); + assertTrue(result.getProperty("timeline").contains("diagnostic-resolved")); + } finally { + child.destroyForcibly(); + } + } + + @Test + void leakedContinuationFailsWithCaptureSite() throws Exception { + ScopeDiagnosticsClient diagnostics = new ScopeDiagnosticsClient(reports, "leaked", true); + Process child = launch(diagnostics, "leak"); + try { + diagnostics.awaitReady(child); + assertTrue(child.waitFor(30, SECONDS)); + AssertionError failure = + assertThrows(AssertionError.class, () -> diagnostics.verifyCompleted(child)); + assertTrue(failure.getMessage().contains("LEAKED"), failure.getMessage()); + Properties result = result(diagnostics.directory().resolve("final")); + assertTrue( + result.getProperty("timeline").contains("ScopeDiagnosticsTestApp"), + result.getProperty("timeline")); + } finally { + child.destroyForcibly(); + } + } + + @Test + void serverWindowsObserveWorkWithoutRetainingPreviousLeak() throws Exception { + ScopeDiagnosticsClient diagnostics = new ScopeDiagnosticsClient(reports, "server", false); + Process child = launch(diagnostics, "server"); + try (Writer input = new OutputStreamWriter(child.getOutputStream(), StandardCharsets.UTF_8); + BufferedReader output = + new BufferedReader( + new InputStreamReader(child.getInputStream(), StandardCharsets.UTF_8))) { + diagnostics.awaitReady(child); + diagnostics.start(child); + exercise(input, output, "leak"); + assertThrows(AssertionError.class, () -> diagnostics.finish(child)); + diagnostics.start(child); + exercise(input, output, "resolved"); + diagnostics.finish(child); + Properties second = result(diagnostics.directory().resolve("response-4")); + assertTrue(Integer.parseInt(second.getProperty("eventCount")) > 0); + assertTrue(second.getProperty("timeline").contains("diagnostic-resolved")); + input.write("exit\n"); + input.flush(); + assertTrue(child.waitFor(30, SECONDS)); + } finally { + child.destroyForcibly(); + } + } + + private Process launch(ScopeDiagnosticsClient diagnostics, String mode) throws Exception { + String classes = + Paths.get( + ScopeDiagnosticsTestApp.class + .getProtectionDomain() + .getCodeSource() + .getLocation() + .toURI()) + .toString(); + return new ProcessBuilder( + Paths.get(System.getProperty("java.home"), "bin", "java").toString(), + "-javaagent:" + System.getProperty("datadog.smoketest.agent.shadowJar.path"), + diagnostics.javaAgentArgument(), + "-Ddd.trace.enabled=true", + "-Ddd.telemetry.enabled=false", + "-Ddd.remote_config.enabled=false", + "-Ddd.trace.startup.logs=false", + "-cp", + classes, + ScopeDiagnosticsTestApp.class.getName(), + mode) + .redirectError(reports.resolve("child-" + mode + ".log").toFile()) + .start(); + } + + private static void exercise(Writer input, BufferedReader output, String action) + throws Exception { + input.write(action + "\n"); + input.flush(); + assertEquals("DONE", output.readLine()); + } + + private static Properties result(Path path) throws Exception { + Properties properties = new Properties(); + try (InputStream input = Files.newInputStream(path)) { + properties.load(input); + } + return properties; + } +} diff --git a/dd-smoke-tests/src/test/java/datadog/smoketest/ScopeDiagnosticsAppTest.java b/dd-smoke-tests/src/test/java/datadog/smoketest/ScopeDiagnosticsAppTest.java new file mode 100644 index 00000000000..63586145976 --- /dev/null +++ b/dd-smoke-tests/src/test/java/datadog/smoketest/ScopeDiagnosticsAppTest.java @@ -0,0 +1,49 @@ +package datadog.smoketest; + +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import datadog.smoketest.backend.AgentBackend; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +@Timeout(60) +class ScopeDiagnosticsAppTest { + @Test + void cliHarnessChecksResolvedContinuationByDefault() throws Exception { + SmokeCliApp app = app("resolved"); + try { + app.beforeAll(null); + app.beforeEach(null); + app.assertCompletesWithValue(30, SECONDS, 0); + app.afterEach(null); + } finally { + app.afterAll(null); + } + } + + @Test + void cliHarnessFailsOnLeakedContinuationByDefault() throws Exception { + SmokeCliApp app = app("leak"); + try { + app.beforeAll(null); + app.beforeEach(null); + AssertionError failure = + assertThrows(AssertionError.class, () -> app.assertCompletesWithValue(30, SECONDS, 0)); + assertTrue(failure.getMessage().contains("LEAKED"), failure.getMessage()); + } finally { + assertThrows(AssertionError.class, () -> app.afterAll(null)); + } + } + + private static SmokeCliApp app(String mode) { + return SmokeCliApp.named("scope-" + mode) + .mainClass(ScopeDiagnosticsTestApp.class) + .args(mode) + .backend(AgentBackend.mockAgent()) + .jvmArgs("-Ddd.telemetry.enabled=false", "-Ddd.remote_config.enabled=false") + .skipTelemetryCheck() + .build(); + } +} diff --git a/dd-smoke-tests/src/test/java/datadog/smoketest/ScopeDiagnosticsClientTest.java b/dd-smoke-tests/src/test/java/datadog/smoketest/ScopeDiagnosticsClientTest.java new file mode 100644 index 00000000000..e2afbf2cabc --- /dev/null +++ b/dd-smoke-tests/src/test/java/datadog/smoketest/ScopeDiagnosticsClientTest.java @@ -0,0 +1,236 @@ +package datadog.smoketest; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Properties; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class ScopeDiagnosticsClientTest { + @TempDir Path temporary; + + @Test + void serverWindowsUseDistinctSequences() throws Exception { + ScopeDiagnosticsClient client = client(false); + write(client, "ready", "protocol", "1"); + client.awaitReady(new StubProcess(true)); + response(client, "response-1", 1, "ok"); + client.start(new StubProcess(true)); + assertThrows(AssertionError.class, () -> client.start(new StubProcess(true))); + response(client, "response-2", 2, "ok"); + client.finish(new StubProcess(true)); + response(client, "response-3", 3, "ok"); + client.start(new StubProcess(true)); + assertEquals("3", read(client, "command").getProperty("seq")); + assertEquals("start", read(client, "command").getProperty("action")); + } + + @Test + void quickCliExitRetainsItsFinalReport() throws Exception { + ScopeDiagnosticsClient client = client(true); + write(client, "ready", "protocol", "1"); + response(client, "final", 0, "ok"); + StubProcess exited = new StubProcess(false); + client.awaitReady(exited); + client.start(exited); + client.verifyCompleted(exited); + assertDoesNotThrow(() -> client.verifyCompleted(exited)); + assertTrue(Files.notExists(client.directory().resolve("command"))); + } + + @Test + void liveCliCanBeCheckedBeforeDestruction() throws Exception { + ScopeDiagnosticsClient client = client(true); + response(client, "response-1", 1, "ok"); + client.finish(new StubProcess(true)); + assertEquals("finish", read(client, "command").getProperty("action")); + assertDoesNotThrow(() -> client.verifyCompleted(new StubProcess(false))); + } + + @Test + void cliExitRacingFinishUsesFinalReport() throws Exception { + ScopeDiagnosticsClient client = client(true); + response(client, "final", 0, "ok"); + client.finish(new StubProcess(true, false)); + assertEquals("finish", read(client, "command").getProperty("action")); + assertDoesNotThrow(() -> client.verifyCompleted(new StubProcess(false))); + } + + @Test + void cliExitCannotHideAnInvalidFinishResponse() throws Exception { + ScopeDiagnosticsClient client = client(true); + response(client, "final", 0, "ok"); + response(client, "response-1", 0, "ok"); + AssertionError error = + assertThrows(AssertionError.class, () -> client.finish(new StubProcess(true, false))); + assertTrue(error.getMessage().contains("incorrect sequence")); + } + + @Test + void cliExitCannotHideReportedProblems() throws Exception { + ScopeDiagnosticsClient client = client(true); + response(client, "final", 0, "ok"); + response(client, "response-1", 1, "problems"); + AssertionError error = + assertThrows(AssertionError.class, () -> client.finish(new StubProcess(true, false))); + assertTrue(error.getMessage().contains("test timeline")); + } + + @Test + void cliExitFallbackStillValidatesFinalSequence() throws Exception { + ScopeDiagnosticsClient client = client(true); + response(client, "final", 1, "ok"); + AssertionError error = + assertThrows(AssertionError.class, () -> client.finish(new StubProcess(true, false))); + assertTrue(error.getMessage().contains("incorrect sequence")); + } + + @Test + void absentFinalReportCannotPass() throws Exception { + ScopeDiagnosticsClient client = client(true); + AssertionError error = + assertThrows(AssertionError.class, () -> client.verifyCompleted(new StubProcess(false))); + assertTrue(error.getMessage().contains("without diagnostic final")); + } + + @Test + void invalidProtocolAndIncompleteResponseCannotPass() throws Exception { + ScopeDiagnosticsClient client = client(true); + write(client, "ready", "protocol", "2"); + assertThrows(AssertionError.class, () -> client.awaitReady(new StubProcess(false))); + write(client, "final", "protocol", "1", "seq", "0", "status", "ok"); + assertThrows(AssertionError.class, () -> client.verifyCompleted(new StubProcess(false))); + response(client, "final", 12, "ok"); + assertThrows(AssertionError.class, () -> client.verifyCompleted(new StubProcess(false))); + } + + @Test + void failuresCarryTimelineAndInstallationDetails() throws Exception { + ScopeDiagnosticsClient client = client(true); + response(client, "final", 0, "problems"); + AssertionError error = + assertThrows(AssertionError.class, () -> client.verifyCompleted(new StubProcess(false))); + assertTrue(error.getMessage().contains("test timeline")); + write(client, "error", "detail", "installation failed"); + error = assertThrows(AssertionError.class, () -> client.awaitReady(new StubProcess(false))); + assertTrue(error.getMessage().contains("installation failed")); + } + + @Test + void unresponsiveAgentTimesOut() throws Exception { + ScopeDiagnosticsClient client = client(false); + AssertionError error = + assertThrows(AssertionError.class, () -> client.awaitReady(new StubProcess(true))); + assertTrue(error.getMessage().contains("Timed out")); + } + + @Test + void optOutRequiresAnExplanation() { + assertThrows( + IllegalArgumentException.class, + () -> SmokeCliApp.named("cli").skipScopeContinuationCheck(" ")); + assertThrows( + IllegalArgumentException.class, + () -> SmokeServerApp.named("server").skipScopeContinuationCheck(null)); + assertDoesNotThrow( + () -> SmokeCliApp.named("cli").skipScopeContinuationCheck("Expected Runtime.halt")); + } + + private ScopeDiagnosticsClient client(boolean cli) throws IOException { + Path jar = temporary.resolve("companion.jar"); + if (Files.notExists(jar)) { + Files.createFile(jar); + } + return new ScopeDiagnosticsClient(temporary, "test app", cli, jar, 25); + } + + private static void response(ScopeDiagnosticsClient client, String file, int seq, String status) + throws IOException { + write( + client, + file, + "protocol", + "1", + "seq", + Integer.toString(seq), + "status", + status, + "detail", + "test summary", + "timeline", + "test timeline", + "eventCount", + "0"); + } + + private static void write(ScopeDiagnosticsClient client, String file, String... entries) + throws IOException { + Properties properties = new Properties(); + for (int i = 0; i < entries.length; i += 2) { + properties.setProperty(entries[i], entries[i + 1]); + } + try (OutputStream output = Files.newOutputStream(client.directory().resolve(file))) { + properties.store(output, null); + } + } + + private static Properties read(ScopeDiagnosticsClient client, String file) throws IOException { + Properties properties = new Properties(); + try (InputStream input = Files.newInputStream(client.directory().resolve(file))) { + properties.load(input); + } + return properties; + } + + private static final class StubProcess extends Process { + private final boolean[] alive; + private int checks; + + StubProcess(boolean... alive) { + this.alive = alive; + } + + @Override + public boolean isAlive() { + return alive[Math.min(checks++, alive.length - 1)]; + } + + @Override + public OutputStream getOutputStream() { + return new ByteArrayOutputStream(); + } + + @Override + public InputStream getInputStream() { + return new ByteArrayInputStream(new byte[0]); + } + + @Override + public InputStream getErrorStream() { + return getInputStream(); + } + + @Override + public int waitFor() { + return 0; + } + + @Override + public int exitValue() { + return 0; + } + + @Override + public void destroy() {} + } +} diff --git a/dd-smoke-tests/src/test/java/datadog/smoketest/ScopeDiagnosticsTestApp.java b/dd-smoke-tests/src/test/java/datadog/smoketest/ScopeDiagnosticsTestApp.java new file mode 100644 index 00000000000..7b347802d7e --- /dev/null +++ b/dd-smoke-tests/src/test/java/datadog/smoketest/ScopeDiagnosticsTestApp.java @@ -0,0 +1,39 @@ +package datadog.smoketest; + +import datadog.context.ContextContinuation; +import datadog.context.ContextScope; +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.bootstrap.instrumentation.api.AgentTracer; +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; + +/** Small child application that uses the actual packaged agent, not a second test tracer. */ +public final class ScopeDiagnosticsTestApp { + private ScopeDiagnosticsTestApp() {} + + public static void main(String[] args) throws Exception { + if ("server".equals(args[0])) { + BufferedReader input = + new BufferedReader(new InputStreamReader(System.in, StandardCharsets.UTF_8)); + String action; + while ((action = input.readLine()) != null && !"exit".equals(action)) { + exercise(action); + System.out.println("DONE"); + System.out.flush(); + } + } else { + exercise(args[0]); + } + } + + private static void exercise(String action) { + AgentSpan span = AgentTracer.startSpan("smoke-diagnostic-test", "diagnostic-" + action); + ContextContinuation continuation = AgentTracer.get().capture(span); + if (!"leak".equals(action)) { + ContextScope scope = continuation.resume(); + scope.close(); + } + span.finish(); + } +} diff --git a/docs/how_to_smoke_test.md b/docs/how_to_smoke_test.md index 445b2565f43..5d881c81795 100644 --- a/docs/how_to_smoke_test.md +++ b/docs/how_to_smoke_test.md @@ -27,6 +27,35 @@ A smoke test is also how you check what an instrumentation does to a running app > The two are complements, not alternatives. > Use an instrumentation test to pin the hooks and the trace shape, and a smoke test to check the instrumentation works in an application where every other instrumentation is loaded too. +## Scope and continuation diagnostics + +The shared JUnit and Spock smoke harnesses enable scope/continuation diagnostics by default. +Gradle adds a **test-only companion agent**, after the built Datadog agent, to install the same +lifecycle probes used by instrumentation tests into the application's tracer class loader. +The production agent artifact and its strict-write configuration are unchanged. + +- `SmokeServerApp` and `AbstractServerSmokeTest` record each test's work, including per-test teardown, + then allow a bounded cleanup grace period before checking. Server startup and suite shutdown are + outside these windows. +- `SmokeCliApp` and other legacy `AbstractSmokeTest` applications record from before `main` until + exit, or until the harness checks them before stopping a still-running process. This includes + first-use static initialization. Concurrent application shutdown hooks are not guaranteed to finish + before the diagnostic snapshot. + +Leaks and invalid lifecycle operations fail the test with a summary and capture/resume/cleanup +timeline. Reports are kept under `build/reports/scope-diagnostics-*`. Missing or malformed reports +and failed probe installation fail explicitly; they are not clean results. +Legacy launchers must preserve `defaultJavaProperties`, including when passing them through an +environment variable to a shell wrapper. Each child receives its own diagnostic endpoint. + +Fix the lifecycle or test ownership problem first. A genuinely intermittent failure may be marked +`@Flaky` with a tracked explanation. A proven diagnostic incompatibility can use +`.skipScopeContinuationCheck("specific reason")` on the JUnit builder, or override +`skipScopeContinuationCheckReason()` in Spock. Blank reasons are rejected. Native-image applications +cannot run this JVM companion; tests intentionally crashing or aborting before it can report also +need an explicit exception. JUnit `.noAgent()` launches do not install diagnostics. +Disabling error-log checks does not disable this check, and relaxing strict trace writes is not a fix. + ## Your first smoke test A complete smoke test for a small web application serving two endpoints, with one test method per endpoint: From cd271b1713303bc42d4c7255a9df5c618b47665f Mon Sep 17 00:00:00 2001 From: Andrea Marziali Date: Wed, 23 Sep 2026 16:48:39 +0200 Subject: [PATCH 23/35] Address kafka timeouts --- .../kafka/iast/IastConfiguration.java | 26 ++++++++++++++++--- .../smoketest/kafka/iast/IastController.java | 13 +++++++++- .../test/groovy/IastKafka2SmokeTest.groovy | 15 +++++++++++ .../kafka/iast/IastConfiguration.java | 26 ++++++++++++++++--- .../smoketest/kafka/iast/IastController.java | 13 +++++++++- .../test/groovy/IastKafka3SmokeTest.groovy | 15 +++++++++++ 6 files changed, 98 insertions(+), 10 deletions(-) diff --git a/dd-smoke-tests/kafka-2/application/src/main/java/datadog/smoketest/kafka/iast/IastConfiguration.java b/dd-smoke-tests/kafka-2/application/src/main/java/datadog/smoketest/kafka/iast/IastConfiguration.java index f70dbd50b30..a86c138946a 100644 --- a/dd-smoke-tests/kafka-2/application/src/main/java/datadog/smoketest/kafka/iast/IastConfiguration.java +++ b/dd-smoke-tests/kafka-2/application/src/main/java/datadog/smoketest/kafka/iast/IastConfiguration.java @@ -27,6 +27,7 @@ import org.springframework.kafka.requestreply.ReplyingKafkaTemplate; import org.springframework.kafka.support.serializer.JsonDeserializer; import org.springframework.kafka.support.serializer.JsonSerializer; +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler; @Configuration public class IastConfiguration { @@ -250,22 +251,39 @@ public DefaultKafkaProducerFactory iastReplyJsonProducer() @Bean public ReplyingKafkaTemplate iastStringTemplate() { - return new ReplyingKafkaTemplate<>(iastStringProducer(), iastReplyStringContainer()); + ReplyingKafkaTemplate template = + new ReplyingKafkaTemplate<>(iastStringProducer(), iastReplyStringContainer()); + template.setTaskScheduler(iastReplyTimeoutScheduler()); + return template; } @Bean public ReplyingKafkaTemplate iastByteArrayTemplate() { - return new ReplyingKafkaTemplate<>(iastByteArrayProducer(), iastReplyByteArrayContainer()); + ReplyingKafkaTemplate template = + new ReplyingKafkaTemplate<>(iastByteArrayProducer(), iastReplyByteArrayContainer()); + template.setTaskScheduler(iastReplyTimeoutScheduler()); + return template; } @Bean public ReplyingKafkaTemplate iastByteBufferTemplate() { - return new ReplyingKafkaTemplate<>(iastByteBufferProducer(), iastReplyByteBufferContainer()); + ReplyingKafkaTemplate template = + new ReplyingKafkaTemplate<>(iastByteBufferProducer(), iastReplyByteBufferContainer()); + template.setTaskScheduler(iastReplyTimeoutScheduler()); + return template; } @Bean public ReplyingKafkaTemplate iastJsonTemplate() { - return new ReplyingKafkaTemplate<>(iastJsonProducer(), iastReplyJsonContainer()); + ReplyingKafkaTemplate template = + new ReplyingKafkaTemplate<>(iastJsonProducer(), iastReplyJsonContainer()); + template.setTaskScheduler(iastReplyTimeoutScheduler()); + return template; + } + + @Bean + public ThreadPoolTaskScheduler iastReplyTimeoutScheduler() { + return new ThreadPoolTaskScheduler(); } @Bean diff --git a/dd-smoke-tests/kafka-2/application/src/main/java/datadog/smoketest/kafka/iast/IastController.java b/dd-smoke-tests/kafka-2/application/src/main/java/datadog/smoketest/kafka/iast/IastController.java index 2d2031ebce4..ced5de9e08c 100644 --- a/dd-smoke-tests/kafka-2/application/src/main/java/datadog/smoketest/kafka/iast/IastController.java +++ b/dd-smoke-tests/kafka-2/application/src/main/java/datadog/smoketest/kafka/iast/IastController.java @@ -6,6 +6,7 @@ import static datadog.smoketest.kafka.iast.IastConfiguration.STRING_TOPIC; import java.nio.ByteBuffer; +import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.function.Function; import org.apache.kafka.clients.consumer.ConsumerRecord; @@ -18,6 +19,7 @@ import org.springframework.kafka.requestreply.ReplyingKafkaTemplate; import org.springframework.kafka.requestreply.RequestReplyFuture; import org.springframework.messaging.handler.annotation.SendTo; +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler; import org.springframework.stereotype.Controller; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RequestParam; @@ -31,6 +33,7 @@ public class IastController { private final ReplyingKafkaTemplate byteArrayTemplate; private final ReplyingKafkaTemplate byteBufferTemplate; private final ReplyingKafkaTemplate jsonTemplate; + private final ThreadPoolTaskScheduler replyTimeoutScheduler; public IastController( @Qualifier("iastStringTemplate") @@ -40,11 +43,19 @@ public IastController( @Qualifier("iastByteBufferTemplate") final ReplyingKafkaTemplate byteBufferTemplate, @Qualifier("iastJsonTemplate") - final ReplyingKafkaTemplate jsonTemplate) { + final ReplyingKafkaTemplate jsonTemplate, + @Qualifier("iastReplyTimeoutScheduler") final ThreadPoolTaskScheduler replyTimeoutScheduler) { this.stringTemplate = stringTemplate; this.byteArrayTemplate = byteArrayTemplate; this.byteBufferTemplate = byteBufferTemplate; this.jsonTemplate = jsonTemplate; + this.replyTimeoutScheduler = replyTimeoutScheduler; + } + + @GetMapping("/iast/kafka/timeouts/idle") + public ResponseEntity replyTimeoutsIdle() { + ScheduledThreadPoolExecutor executor = replyTimeoutScheduler.getScheduledThreadPoolExecutor(); + return ResponseEntity.ok(executor.getQueue().isEmpty() && executor.getActiveCount() == 0); } @GetMapping("/iast/health") diff --git a/dd-smoke-tests/kafka-2/src/test/groovy/IastKafka2SmokeTest.groovy b/dd-smoke-tests/kafka-2/src/test/groovy/IastKafka2SmokeTest.groovy index 9c3b9a46685..8da2722b889 100644 --- a/dd-smoke-tests/kafka-2/src/test/groovy/IastKafka2SmokeTest.groovy +++ b/dd-smoke-tests/kafka-2/src/test/groovy/IastKafka2SmokeTest.groovy @@ -3,6 +3,7 @@ import datadog.trace.agent.test.utils.OkHttpUtils import okhttp3.Request import org.springframework.kafka.test.EmbeddedKafkaBroker import spock.lang.Shared +import spock.util.concurrent.PollingConditions import static datadog.trace.api.config.IastConfig.IAST_CONTEXT_MODE import static datadog.trace.api.config.IastConfig.IAST_DEBUG_ENABLED @@ -62,6 +63,20 @@ class IastKafka2SmokeTest extends AbstractIastServerSmokeTest { throw new IllegalStateException('Server not properly initialized') } + def cleanup() { + // Successful replies leave their timeout scheduled; let it finish before checking continuations. + new PollingConditions(timeout: 10).eventually { + final request = new Request.Builder().url("http://localhost:${httpPort}/iast/kafka/timeouts/idle").get().build() + final response = client.newCall(request).execute() + try { + assert response.successful + assert response.body().string() == 'true' + } finally { + response.close() + } + } + } + void 'test kafka #endpoint key source'() { setup: final type = "${endpoint}_source_key" diff --git a/dd-smoke-tests/kafka-3/application/src/main/java/datadog/smoketest/kafka/iast/IastConfiguration.java b/dd-smoke-tests/kafka-3/application/src/main/java/datadog/smoketest/kafka/iast/IastConfiguration.java index f70dbd50b30..a86c138946a 100644 --- a/dd-smoke-tests/kafka-3/application/src/main/java/datadog/smoketest/kafka/iast/IastConfiguration.java +++ b/dd-smoke-tests/kafka-3/application/src/main/java/datadog/smoketest/kafka/iast/IastConfiguration.java @@ -27,6 +27,7 @@ import org.springframework.kafka.requestreply.ReplyingKafkaTemplate; import org.springframework.kafka.support.serializer.JsonDeserializer; import org.springframework.kafka.support.serializer.JsonSerializer; +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler; @Configuration public class IastConfiguration { @@ -250,22 +251,39 @@ public DefaultKafkaProducerFactory iastReplyJsonProducer() @Bean public ReplyingKafkaTemplate iastStringTemplate() { - return new ReplyingKafkaTemplate<>(iastStringProducer(), iastReplyStringContainer()); + ReplyingKafkaTemplate template = + new ReplyingKafkaTemplate<>(iastStringProducer(), iastReplyStringContainer()); + template.setTaskScheduler(iastReplyTimeoutScheduler()); + return template; } @Bean public ReplyingKafkaTemplate iastByteArrayTemplate() { - return new ReplyingKafkaTemplate<>(iastByteArrayProducer(), iastReplyByteArrayContainer()); + ReplyingKafkaTemplate template = + new ReplyingKafkaTemplate<>(iastByteArrayProducer(), iastReplyByteArrayContainer()); + template.setTaskScheduler(iastReplyTimeoutScheduler()); + return template; } @Bean public ReplyingKafkaTemplate iastByteBufferTemplate() { - return new ReplyingKafkaTemplate<>(iastByteBufferProducer(), iastReplyByteBufferContainer()); + ReplyingKafkaTemplate template = + new ReplyingKafkaTemplate<>(iastByteBufferProducer(), iastReplyByteBufferContainer()); + template.setTaskScheduler(iastReplyTimeoutScheduler()); + return template; } @Bean public ReplyingKafkaTemplate iastJsonTemplate() { - return new ReplyingKafkaTemplate<>(iastJsonProducer(), iastReplyJsonContainer()); + ReplyingKafkaTemplate template = + new ReplyingKafkaTemplate<>(iastJsonProducer(), iastReplyJsonContainer()); + template.setTaskScheduler(iastReplyTimeoutScheduler()); + return template; + } + + @Bean + public ThreadPoolTaskScheduler iastReplyTimeoutScheduler() { + return new ThreadPoolTaskScheduler(); } @Bean diff --git a/dd-smoke-tests/kafka-3/application/src/main/java/datadog/smoketest/kafka/iast/IastController.java b/dd-smoke-tests/kafka-3/application/src/main/java/datadog/smoketest/kafka/iast/IastController.java index 3defbd5863f..6897d60e954 100644 --- a/dd-smoke-tests/kafka-3/application/src/main/java/datadog/smoketest/kafka/iast/IastController.java +++ b/dd-smoke-tests/kafka-3/application/src/main/java/datadog/smoketest/kafka/iast/IastController.java @@ -6,6 +6,7 @@ import static datadog.smoketest.kafka.iast.IastConfiguration.STRING_TOPIC; import java.nio.ByteBuffer; +import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.function.Function; import org.apache.kafka.clients.consumer.ConsumerRecord; @@ -18,6 +19,7 @@ import org.springframework.kafka.requestreply.ReplyingKafkaTemplate; import org.springframework.kafka.requestreply.RequestReplyFuture; import org.springframework.messaging.handler.annotation.SendTo; +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler; import org.springframework.stereotype.Controller; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RequestParam; @@ -31,6 +33,7 @@ public class IastController { private final ReplyingKafkaTemplate byteArrayTemplate; private final ReplyingKafkaTemplate byteBufferTemplate; private final ReplyingKafkaTemplate jsonTemplate; + private final ThreadPoolTaskScheduler replyTimeoutScheduler; public IastController( @Qualifier("iastStringTemplate") @@ -40,11 +43,19 @@ public IastController( @Qualifier("iastByteBufferTemplate") final ReplyingKafkaTemplate byteBufferTemplate, @Qualifier("iastJsonTemplate") - final ReplyingKafkaTemplate jsonTemplate) { + final ReplyingKafkaTemplate jsonTemplate, + @Qualifier("iastReplyTimeoutScheduler") final ThreadPoolTaskScheduler replyTimeoutScheduler) { this.stringTemplate = stringTemplate; this.byteArrayTemplate = byteArrayTemplate; this.byteBufferTemplate = byteBufferTemplate; this.jsonTemplate = jsonTemplate; + this.replyTimeoutScheduler = replyTimeoutScheduler; + } + + @GetMapping("/iast/kafka/timeouts/idle") + public ResponseEntity replyTimeoutsIdle() { + ScheduledThreadPoolExecutor executor = replyTimeoutScheduler.getScheduledThreadPoolExecutor(); + return ResponseEntity.ok(executor.getQueue().isEmpty() && executor.getActiveCount() == 0); } @GetMapping("/iast/health") diff --git a/dd-smoke-tests/kafka-3/src/test/groovy/IastKafka3SmokeTest.groovy b/dd-smoke-tests/kafka-3/src/test/groovy/IastKafka3SmokeTest.groovy index dd4f3a75d93..907d777cd4f 100644 --- a/dd-smoke-tests/kafka-3/src/test/groovy/IastKafka3SmokeTest.groovy +++ b/dd-smoke-tests/kafka-3/src/test/groovy/IastKafka3SmokeTest.groovy @@ -3,6 +3,7 @@ import datadog.trace.agent.test.utils.OkHttpUtils import okhttp3.Request import org.springframework.kafka.test.EmbeddedKafkaBroker import spock.lang.Shared +import spock.util.concurrent.PollingConditions import static datadog.trace.api.config.IastConfig.IAST_CONTEXT_MODE import static datadog.trace.api.config.IastConfig.IAST_DEBUG_ENABLED @@ -62,6 +63,20 @@ class IastKafka3SmokeTest extends AbstractIastServerSmokeTest { throw new IllegalStateException('Server not properly initialized') } + def cleanup() { + // Successful replies leave their timeout scheduled; let it finish before checking continuations. + new PollingConditions(timeout: 10).eventually { + final request = new Request.Builder().url("http://localhost:${httpPort}/iast/kafka/timeouts/idle").get().build() + final response = client.newCall(request).execute() + try { + assert response.successful + assert response.body().string() == 'true' + } finally { + response.close() + } + } + } + void 'test kafka #endpoint key source'() { setup: final type = "${endpoint}_source_key" From 4aa2326eddec92b13c7ce8132056112aefa27856 Mon Sep 17 00:00:00 2001 From: Andrea Marziali Date: Wed, 23 Sep 2026 17:01:59 +0200 Subject: [PATCH 24/35] Improve correctness of httpclient smoke tests --- .../smoketest/springboot/controller/SsrfController.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/dd-smoke-tests/iast-util/src/main/java/datadog/smoketest/springboot/controller/SsrfController.java b/dd-smoke-tests/iast-util/src/main/java/datadog/smoketest/springboot/controller/SsrfController.java index 4b180706aa1..4fa38ed881f 100644 --- a/dd-smoke-tests/iast-util/src/main/java/datadog/smoketest/springboot/controller/SsrfController.java +++ b/dd-smoke-tests/iast-util/src/main/java/datadog/smoketest/springboot/controller/SsrfController.java @@ -5,6 +5,7 @@ import java.net.HttpURLConnection; import java.net.URI; import java.net.URL; +import java.util.concurrent.TimeUnit; import org.apache.commons.httpclient.HttpClient; import org.apache.commons.httpclient.HttpMethod; import org.apache.commons.httpclient.methods.GetMethod; @@ -17,6 +18,7 @@ import org.apache.http.impl.nio.client.HttpAsyncClients; import org.apache.http.message.BasicHttpRequest; import org.apache.http.nio.client.methods.HttpAsyncMethods; +import org.apache.http.nio.protocol.BasicAsyncResponseConsumer; import org.apache.http.nio.protocol.HttpAsyncRequestProducer; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestMapping; @@ -153,13 +155,13 @@ public String apacheHttpAsyncClient( try { if (host != null) { final HttpHost httpHost = new HttpHost(host); - client.execute(httpHost, new HttpGet("/"), null); + client.execute(httpHost, new HttpGet("/"), null).get(10, TimeUnit.SECONDS); } else if (url != null) { final HttpGet request = new HttpGet(url); - client.execute(request, null); + client.execute(request, null).get(10, TimeUnit.SECONDS); } else if (urlProducer != null) { final HttpAsyncRequestProducer producer = HttpAsyncMethods.create(new HttpGet(urlProducer)); - client.execute(producer, null, null); + client.execute(producer, new BasicAsyncResponseConsumer(), null).get(10, TimeUnit.SECONDS); } } catch (Exception e) { } finally { From 42f056e879469a780793ced1108aff76ddaa65ce Mon Sep 17 00:00:00 2001 From: Andrea Marziali Date: Wed, 23 Sep 2026 18:16:06 +0200 Subject: [PATCH 25/35] Fix smoke diagnostic compatibility with custom launchers --- .../MultipleAgentGuardrailsTest.groovy | 7 +++ .../smoketest/OpenTracingSmokeTest.groovy | 5 ++ .../SpringBootOpenLibertySmokeTest.groovy | 17 ++++++ ...otOpenLibertySmokeVulnerabilityTest.groovy | 17 ++++++ .../SpringBootOpenLibertySmokeTest.groovy | 17 ++++++ ...otOpenLibertySmokeVulnerabilityTest.groovy | 17 ++++++ .../datadog/smoketest/ProcessManager.groovy | 12 +++- .../ProcessManagerScopeDiagnosticsTest.java | 57 +++++++++++++++++++ docs/how_to_smoke_test.md | 5 ++ 9 files changed, 151 insertions(+), 3 deletions(-) create mode 100644 dd-smoke-tests/src/test/java/datadog/smoketest/ProcessManagerScopeDiagnosticsTest.java diff --git a/dd-smoke-tests/lib-injection/src/test/groovy/datadog/smoketest/MultipleAgentGuardrailsTest.groovy b/dd-smoke-tests/lib-injection/src/test/groovy/datadog/smoketest/MultipleAgentGuardrailsTest.groovy index c0e9350e468..fc1cafc2acb 100644 --- a/dd-smoke-tests/lib-injection/src/test/groovy/datadog/smoketest/MultipleAgentGuardrailsTest.groovy +++ b/dd-smoke-tests/lib-injection/src/test/groovy/datadog/smoketest/MultipleAgentGuardrailsTest.groovy @@ -9,6 +9,13 @@ abstract class MultipleAgentGuardrailsTest extends AbstractSmokeTest { static final String LIB_INJECTION_ENABLED_FLAG = 'DD_INJECTION_ENABLED' static final String LIB_INJECTION_FORCE_FLAG = 'DD_INJECT_FORCE' + @Override + protected String skipScopeContinuationCheckReason() { + isLibInjectionEnabled() && !isLibInjectionForced() + ? 'The diagnostic companion is an additional agent and changes the injection guardrail being tested' + : null + } + @Override ProcessBuilder createProcessBuilder() { def jarPath = System.getProperty('datadog.smoketest.shadowJar.path') diff --git a/dd-smoke-tests/opentracing/src/test/groovy/datadog/smoketest/OpenTracingSmokeTest.groovy b/dd-smoke-tests/opentracing/src/test/groovy/datadog/smoketest/OpenTracingSmokeTest.groovy index 6cc100ff707..796e4419b2e 100644 --- a/dd-smoke-tests/opentracing/src/test/groovy/datadog/smoketest/OpenTracingSmokeTest.groovy +++ b/dd-smoke-tests/opentracing/src/test/groovy/datadog/smoketest/OpenTracingSmokeTest.groovy @@ -30,6 +30,11 @@ abstract class OpenTracingSmokeTest extends AbstractSmokeTest { } class OTWithoutAgentTest extends OpenTracingSmokeTest { + @Override + protected String skipScopeContinuationCheckReason() { + 'Exercises the standalone OpenTracing tracer without the agent required by the diagnostic companion' + } + @Override ProcessBuilder createProcessBuilder() { List command = baseCommand() diff --git a/dd-smoke-tests/springboot-openliberty-20/src/test/groovy/datadog/smoketest/SpringBootOpenLibertySmokeTest.groovy b/dd-smoke-tests/springboot-openliberty-20/src/test/groovy/datadog/smoketest/SpringBootOpenLibertySmokeTest.groovy index 0caf3fe7da9..186d7a7e923 100644 --- a/dd-smoke-tests/springboot-openliberty-20/src/test/groovy/datadog/smoketest/SpringBootOpenLibertySmokeTest.groovy +++ b/dd-smoke-tests/springboot-openliberty-20/src/test/groovy/datadog/smoketest/SpringBootOpenLibertySmokeTest.groovy @@ -56,6 +56,23 @@ class SpringBootOpenLibertySmokeTest extends AbstractServerSmokeTest { return processBuilder } + @Override + protected boolean replaceScopeDiagnosticsArgument(ProcessBuilder builder, String argument) { + // Liberty reads the child server's JVM arguments from inside the runnable JAR. + def appUri = URI.create("jar:${Paths.get(builder.command().get(2)).toUri()}") + try (def fs = FileSystems.newFileSystem(appUri, [:])) { + def optionsFile = fs.getPath('wlp', 'usr', 'servers', 'defaultServer', 'jvm.options') + def options = Files.readAllLines(optionsFile) + boolean found = options.any { + it.contains(SCOPE_DIAGNOSTICS_ARGUMENT) + } + Files.write(optionsFile, options.collect { + it.replace(SCOPE_DIAGNOSTICS_ARGUMENT, argument) + }) + return found + } + } + Path copyApplicationJar() { def applicationJar = Paths.get(openLibertyShadowJar) def randomId = System.nanoTime() diff --git a/dd-smoke-tests/springboot-openliberty-20/src/test/groovy/datadog/smoketest/SpringBootOpenLibertySmokeVulnerabilityTest.groovy b/dd-smoke-tests/springboot-openliberty-20/src/test/groovy/datadog/smoketest/SpringBootOpenLibertySmokeVulnerabilityTest.groovy index e1d65888734..29c73177b1a 100644 --- a/dd-smoke-tests/springboot-openliberty-20/src/test/groovy/datadog/smoketest/SpringBootOpenLibertySmokeVulnerabilityTest.groovy +++ b/dd-smoke-tests/springboot-openliberty-20/src/test/groovy/datadog/smoketest/SpringBootOpenLibertySmokeVulnerabilityTest.groovy @@ -53,6 +53,23 @@ class SpringBootOpenLibertySmokeVulnerabilityTest extends AbstractServerSmokeTes return processBuilder } + @Override + protected boolean replaceScopeDiagnosticsArgument(ProcessBuilder builder, String argument) { + // Liberty reads the child server's JVM arguments from inside the runnable JAR. + def appUri = URI.create("jar:${Paths.get(builder.command().get(2)).toUri()}") + try (def fs = FileSystems.newFileSystem(appUri, [:])) { + def optionsFile = fs.getPath('wlp', 'usr', 'servers', 'defaultServer', 'jvm.options') + def options = Files.readAllLines(optionsFile) + boolean found = options.any { + it.contains(SCOPE_DIAGNOSTICS_ARGUMENT) + } + Files.write(optionsFile, options.collect { + it.replace(SCOPE_DIAGNOSTICS_ARGUMENT, argument) + }) + return found + } + } + Path copyApplicationJar() { def applicationJar = Paths.get(openLibertyShadowJar) def randomId = System.nanoTime() diff --git a/dd-smoke-tests/springboot-openliberty-23/src/test/groovy/datadog/smoketest/SpringBootOpenLibertySmokeTest.groovy b/dd-smoke-tests/springboot-openliberty-23/src/test/groovy/datadog/smoketest/SpringBootOpenLibertySmokeTest.groovy index d7844da8f75..b5aa6946389 100644 --- a/dd-smoke-tests/springboot-openliberty-23/src/test/groovy/datadog/smoketest/SpringBootOpenLibertySmokeTest.groovy +++ b/dd-smoke-tests/springboot-openliberty-23/src/test/groovy/datadog/smoketest/SpringBootOpenLibertySmokeTest.groovy @@ -56,6 +56,23 @@ class SpringBootOpenLibertySmokeTest extends AbstractServerSmokeTest { return processBuilder } + @Override + protected boolean replaceScopeDiagnosticsArgument(ProcessBuilder builder, String argument) { + // Liberty reads the child server's JVM arguments from inside the runnable JAR. + def appUri = URI.create("jar:${Paths.get(builder.command().get(2)).toUri()}") + try (def fs = FileSystems.newFileSystem(appUri, [:])) { + def optionsFile = fs.getPath('wlp', 'usr', 'servers', 'defaultServer', 'jvm.options') + def options = Files.readAllLines(optionsFile) + boolean found = options.any { + it.contains(SCOPE_DIAGNOSTICS_ARGUMENT) + } + Files.write(optionsFile, options.collect { + it.replace(SCOPE_DIAGNOSTICS_ARGUMENT, argument) + }) + return found + } + } + Path copyApplicationJar() { def applicationJar = Paths.get(openLibertyShadowJar) def randomId = System.nanoTime() diff --git a/dd-smoke-tests/springboot-openliberty-23/src/test/groovy/datadog/smoketest/SpringBootOpenLibertySmokeVulnerabilityTest.groovy b/dd-smoke-tests/springboot-openliberty-23/src/test/groovy/datadog/smoketest/SpringBootOpenLibertySmokeVulnerabilityTest.groovy index e16770c0d2d..0a72c2d7b84 100644 --- a/dd-smoke-tests/springboot-openliberty-23/src/test/groovy/datadog/smoketest/SpringBootOpenLibertySmokeVulnerabilityTest.groovy +++ b/dd-smoke-tests/springboot-openliberty-23/src/test/groovy/datadog/smoketest/SpringBootOpenLibertySmokeVulnerabilityTest.groovy @@ -52,6 +52,23 @@ class SpringBootOpenLibertySmokeVulnerabilityTest extends AbstractServerSmokeTes return processBuilder } + @Override + protected boolean replaceScopeDiagnosticsArgument(ProcessBuilder builder, String argument) { + // Liberty reads the child server's JVM arguments from inside the runnable JAR. + def appUri = URI.create("jar:${Paths.get(builder.command().get(2)).toUri()}") + try (def fs = FileSystems.newFileSystem(appUri, [:])) { + def optionsFile = fs.getPath('wlp', 'usr', 'servers', 'defaultServer', 'jvm.options') + def options = Files.readAllLines(optionsFile) + boolean found = options.any { + it.contains(SCOPE_DIAGNOSTICS_ARGUMENT) + } + Files.write(optionsFile, options.collect { + it.replace(SCOPE_DIAGNOSTICS_ARGUMENT, argument) + }) + return found + } + } + Path copyApplicationJar() { def applicationJar = Paths.get(openLibertyShadowJar) def randomId = System.nanoTime() diff --git a/dd-smoke-tests/src/main/groovy/datadog/smoketest/ProcessManager.groovy b/dd-smoke-tests/src/main/groovy/datadog/smoketest/ProcessManager.groovy index 74d9e16171b..37acdb39c92 100644 --- a/dd-smoke-tests/src/main/groovy/datadog/smoketest/ProcessManager.groovy +++ b/dd-smoke-tests/src/main/groovy/datadog/smoketest/ProcessManager.groovy @@ -231,6 +231,14 @@ abstract class ProcessManager extends Specification { : null scopeDiagnostics.add(diagnostic) String argument = diagnostic == null ? '' : diagnostic.javaAgentArgument() + boolean found = replaceScopeDiagnosticsArgument(builder, argument) + if (diagnostic != null && !found) { + throw new IllegalStateException('Smoke launch must include defaultJavaProperties, or document an incompatible launch with skipScopeContinuationCheckReason()') + } + } + + /** Override for launches that store JVM arguments outside the command and environment. */ + protected boolean replaceScopeDiagnosticsArgument(ProcessBuilder builder, String argument) { boolean found = false builder.command().replaceAll { String value -> if (value.contains(SCOPE_DIAGNOSTICS_ARGUMENT)) { @@ -247,9 +255,7 @@ abstract class ProcessManager extends Specification { } value } - if (diagnostic != null && !found) { - throw new IllegalStateException('Smoke launch must include defaultJavaProperties, or document an incompatible launch with skipScopeContinuationCheckReason()') - } + return found } def getProfilingUrl() { diff --git a/dd-smoke-tests/src/test/java/datadog/smoketest/ProcessManagerScopeDiagnosticsTest.java b/dd-smoke-tests/src/test/java/datadog/smoketest/ProcessManagerScopeDiagnosticsTest.java new file mode 100644 index 00000000000..53a7e564f65 --- /dev/null +++ b/dd-smoke-tests/src/test/java/datadog/smoketest/ProcessManagerScopeDiagnosticsTest.java @@ -0,0 +1,57 @@ +package datadog.smoketest; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.Arrays; +import org.junit.jupiter.api.Test; + +class ProcessManagerScopeDiagnosticsTest { + private final ProcessManager manager = + new ProcessManager() { + @Override + protected void setupTracesConsumer() {} + }; + + @Test + void replacesCommandPlaceholderWithoutChangingAgentOrder() { + ProcessBuilder builder = + new ProcessBuilder( + "java", "-javaagent:tracer.jar", ProcessManager.SCOPE_DIAGNOSTICS_ARGUMENT, "App"); + + assertTrue(manager.replaceScopeDiagnosticsArgument(builder, "-javaagent:diagnostics.jar")); + assertEquals( + Arrays.asList("java", "-javaagent:tracer.jar", "-javaagent:diagnostics.jar", "App"), + builder.command()); + } + + @Test + void replacesEnvironmentPlaceholder() { + ProcessBuilder builder = new ProcessBuilder("launcher"); + builder + .environment() + .put("JAVA_OPTS", "-javaagent:tracer.jar " + ProcessManager.SCOPE_DIAGNOSTICS_ARGUMENT); + + assertTrue(manager.replaceScopeDiagnosticsArgument(builder, "-javaagent:diagnostics.jar")); + assertEquals( + "-javaagent:tracer.jar -javaagent:diagnostics.jar", builder.environment().get("JAVA_OPTS")); + } + + @Test + void removesCommandPlaceholderForDocumentedOptOut() { + ProcessBuilder builder = + new ProcessBuilder("java", ProcessManager.SCOPE_DIAGNOSTICS_ARGUMENT, "App"); + + assertTrue(manager.replaceScopeDiagnosticsArgument(builder, "")); + assertEquals(Arrays.asList("java", "App"), builder.command()); + } + + @Test + void reportsMissingPlaceholder() { + ProcessBuilder builder = new ProcessBuilder("java", "App"); + + assertFalse(manager.replaceScopeDiagnosticsArgument(builder, "-javaagent:diagnostics.jar")); + assertEquals(Arrays.asList("java", "App"), builder.command()); + } +} diff --git a/docs/how_to_smoke_test.md b/docs/how_to_smoke_test.md index 5d881c81795..d084d3195b3 100644 --- a/docs/how_to_smoke_test.md +++ b/docs/how_to_smoke_test.md @@ -47,6 +47,9 @@ timeline. Reports are kept under `build/reports/scope-diagnostics-*`. Missing or and failed probe installation fail explicitly; they are not clean results. Legacy launchers must preserve `defaultJavaProperties`, including when passing them through an environment variable to a shell wrapper. Each child receives its own diagnostic endpoint. +Launchers that embed JVM options in a file, such as OpenLiberty's `jvm.options`, override +`replaceScopeDiagnosticsArgument()` to replace the placeholder in that file and report whether it +was found. Keep the companion after the Datadog agent in the child JVM's options. Fix the lifecycle or test ownership problem first. A genuinely intermittent failure may be marked `@Flaky` with a tracked explanation. A proven diagnostic incompatibility can use @@ -54,6 +57,8 @@ Fix the lifecycle or test ownership problem first. A genuinely intermittent fail `skipScopeContinuationCheckReason()` in Spock. Blank reasons are rejected. Native-image applications cannot run this JVM companion; tests intentionally crashing or aborting before it can report also need an explicit exception. JUnit `.noAgent()` launches do not install diagnostics. +Legacy tests deliberately running without the agent need a documented opt-out, as do injection +guardrail tests where the companion would itself count as an extra agent and change the scenario. Disabling error-log checks does not disable this check, and relaxing strict trace writes is not a fix. ## Your first smoke test From 806dd538533156ed44807a308e0b315f5882c66a Mon Sep 17 00:00:00 2001 From: Andrea Marziali Date: Wed, 23 Sep 2026 18:16:17 +0200 Subject: [PATCH 26/35] Allow fallback error reporting in smoke diagnostic agent --- .../datadog/trace/agent/test/smoke/ScopeDiagnosticsAgent.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dd-java-agent/instrumentation-testing/src/smokeAgent/java/datadog/trace/agent/test/smoke/ScopeDiagnosticsAgent.java b/dd-java-agent/instrumentation-testing/src/smokeAgent/java/datadog/trace/agent/test/smoke/ScopeDiagnosticsAgent.java index 44dc5609d45..03efdb9f9bc 100644 --- a/dd-java-agent/instrumentation-testing/src/smokeAgent/java/datadog/trace/agent/test/smoke/ScopeDiagnosticsAgent.java +++ b/dd-java-agent/instrumentation-testing/src/smokeAgent/java/datadog/trace/agent/test/smoke/ScopeDiagnosticsAgent.java @@ -1,5 +1,6 @@ package datadog.trace.agent.test.smoke; +import de.thetaphi.forbiddenapis.SuppressForbidden; import java.io.ByteArrayOutputStream; import java.io.InputStream; import java.io.OutputStream; @@ -192,6 +193,7 @@ private static void publish(Path target, Properties data) throws Exception { } } + @SuppressForbidden // Last-resort stderr when writing the diagnostic report fails. private static void publishFailure(Path directory, String file, long seq, Throwable failure) { Properties result = new Properties(); result.setProperty("status", "error"); From d9eeb4f60321828428f0f39ca9962052dc0fd489 Mon Sep 17 00:00:00 2001 From: Andrea Marziali Date: Wed, 23 Sep 2026 15:47:56 +0200 Subject: [PATCH 27/35] Finish HTTP client spans and close scopes when AppSec blocks requests --- .../apachehttpclient/HelperMethods.java | 16 +++- .../ApacheHttpClientBlockingForkedTest.groovy | 80 ++++++++++++++++++ .../CommonsHttpClientInstrumentation.java | 13 ++- ...CommonsHttpClientBlockingForkedTest.groovy | 70 ++++++++++++++++ .../httpclient/SendAdvice.java | 13 ++- .../httpclient/SendAsyncAdvice.java | 13 ++- .../JavaHttpClientBlockingForkedTest.groovy | 82 +++++++++++++++++++ 7 files changed, 282 insertions(+), 5 deletions(-) create mode 100644 dd-java-agent/instrumentation/apache-httpclient/apache-httpclient-4.0/src/test/groovy/ApacheHttpClientBlockingForkedTest.groovy create mode 100644 dd-java-agent/instrumentation/commons-httpclient-2.0/src/test/groovy/CommonsHttpClientBlockingForkedTest.groovy create mode 100644 dd-java-agent/instrumentation/java/java-net/java-net-11.0/src/test/groovy/datadog/trace/instrumentation/httpclient/JavaHttpClientBlockingForkedTest.groovy diff --git a/dd-java-agent/instrumentation/apache-httpclient/apache-httpclient-4.0/src/main/java/datadog/trace/instrumentation/apachehttpclient/HelperMethods.java b/dd-java-agent/instrumentation/apache-httpclient/apache-httpclient-4.0/src/main/java/datadog/trace/instrumentation/apachehttpclient/HelperMethods.java index b4f77d405c6..7a9d74de25d 100644 --- a/dd-java-agent/instrumentation/apache-httpclient/apache-httpclient-4.0/src/main/java/datadog/trace/instrumentation/apachehttpclient/HelperMethods.java +++ b/dd-java-agent/instrumentation/apache-httpclient/apache-httpclient-4.0/src/main/java/datadog/trace/instrumentation/apachehttpclient/HelperMethods.java @@ -9,6 +9,7 @@ import static datadog.trace.instrumentation.apachehttpclient.ApacheHttpClientDecorator.HTTP_REQUEST; import static datadog.trace.instrumentation.apachehttpclient.HttpHeadersInjectAdapter.SETTER; +import datadog.appsec.api.blocking.BlockingException; import datadog.context.ContextScope; import datadog.trace.bootstrap.CallDepthThreadLocalMap; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; @@ -40,8 +41,19 @@ private static ContextScope activateHttpSpan(final HttpUriRequest request) { final AgentSpan span = startSpan(APACHE_HTTP_CLIENT.toString(), HTTP_REQUEST); final ContextScope scope = activateSpan(span); - DECORATE.afterStart(span); - DECORATE.onRequest(span, request); + try { + DECORATE.afterStart(span); + DECORATE.onRequest(span, request); + } catch (BlockingException e) { + try { + DECORATE.onError(span, e); + DECORATE.beforeFinish(span); + } finally { + scope.close(); + span.finish(); + } + throw e; + } return scope; } diff --git a/dd-java-agent/instrumentation/apache-httpclient/apache-httpclient-4.0/src/test/groovy/ApacheHttpClientBlockingForkedTest.groovy b/dd-java-agent/instrumentation/apache-httpclient/apache-httpclient-4.0/src/test/groovy/ApacheHttpClientBlockingForkedTest.groovy new file mode 100644 index 00000000000..78c316bfb36 --- /dev/null +++ b/dd-java-agent/instrumentation/apache-httpclient/apache-httpclient-4.0/src/test/groovy/ApacheHttpClientBlockingForkedTest.groovy @@ -0,0 +1,80 @@ +import datadog.appsec.api.blocking.BlockingContentType +import datadog.appsec.api.blocking.BlockingException +import datadog.trace.agent.test.InstrumentationSpecification +import datadog.trace.api.gateway.Flow +import datadog.trace.api.gateway.RequestContextSlot +import datadog.trace.bootstrap.CallDepthThreadLocalMap +import datadog.trace.bootstrap.instrumentation.api.AgentTracer +import datadog.trace.bootstrap.instrumentation.api.TagContext +import org.apache.http.HttpHost +import org.apache.http.client.HttpClient +import org.apache.http.client.methods.HttpGet +import org.apache.http.impl.client.DefaultHttpClient +import org.apache.http.message.BasicHttpRequest + +import java.util.function.BiFunction + +import static datadog.trace.api.gateway.Events.EVENTS + +/** Forked so AppSec configuration is applied before instrumentation is installed. */ +class ApacheHttpClientBlockingForkedTest extends InstrumentationSpecification { + @Override + protected void configurePreAgent() { + super.configurePreAgent() + injectSysConfig('appsec.enabled', 'true') + injectSysConfig('appsec.rasp.enabled', 'true') + } + + def 'blocking restores the parent and finishes each client span'() { + given: + def subscription = AgentTracer.get().getSubscriptionService(RequestContextSlot.APPSEC) + def blockedSpans = [] + def flow = Stub(Flow) { + getAction() >> new Flow.Action.RequestBlockingAction(403, BlockingContentType.JSON) + } + subscription.registerCallback(EVENTS.httpClientRequest(), { ctx, request -> + blockedSpans.add(AgentTracer.activeSpan()) + flow + } as BiFunction) + def parent = TEST_TRACER.startSpan('test', 'parent', + new TagContext().withRequestContextDataAppSec(new Object())) + def parentScope = AgentTracer.activateSpan(parent) + def client = new DefaultHttpClient() + + when: + 2.times { + try { + if (hostRequest) { + client.execute(new HttpHost('localhost', 1), new BasicHttpRequest('GET', '/blocked')) + } else { + client.execute(new HttpGet('http://localhost:1/blocked')) + } + assert false: 'The request must be blocked before reaching the client' + } catch (BlockingException expected) { + assert AgentTracer.activeSpan().is(parent) + assert CallDepthThreadLocalMap.getCallDepth(HttpClient) == 0 + assert blockedSpans.last().finished + } + } + parentScope.close() + parentScope = null + parent.finish() + + then: + blockedSpans.size() == 2 + blockedSpans.every { it.parentId == parent.spanId } + TEST_WRITER.waitForTraces(1) + TEST_WRITER.size() == 1 + TEST_WRITER[0].size() == 3 + + cleanup: + parentScope?.close() + if (parent != null && !parent.finished) { + parent.finish() + } + subscription.reset() + + where: + hostRequest << [false, true] + } +} diff --git a/dd-java-agent/instrumentation/commons-httpclient-2.0/src/main/java/datadog/trace/instrumentation/commonshttpclient/CommonsHttpClientInstrumentation.java b/dd-java-agent/instrumentation/commons-httpclient-2.0/src/main/java/datadog/trace/instrumentation/commonshttpclient/CommonsHttpClientInstrumentation.java index f7a64539c31..322ffc65b95 100644 --- a/dd-java-agent/instrumentation/commons-httpclient-2.0/src/main/java/datadog/trace/instrumentation/commonshttpclient/CommonsHttpClientInstrumentation.java +++ b/dd-java-agent/instrumentation/commons-httpclient-2.0/src/main/java/datadog/trace/instrumentation/commonshttpclient/CommonsHttpClientInstrumentation.java @@ -60,6 +60,7 @@ public static class ExecAdvice { @Advice.OnMethodEnter(suppress = Throwable.class) public static ContextScope methodEnter(@Advice.Argument(1) final HttpMethod httpMethod) { + ContextScope scope = null; try { final int callDepth = CallDepthThreadLocalMap.incrementCallDepth(HttpClient.class); if (callDepth > 0) { @@ -67,7 +68,7 @@ public static ContextScope methodEnter(@Advice.Argument(1) final HttpMethod http } final AgentSpan span = startSpan("commons-http-client", HTTP_REQUEST); - final ContextScope scope = activateSpan(span); + scope = activateSpan(span); DECORATE.afterStart(span); DECORATE.onRequest(span, httpMethod); @@ -75,6 +76,16 @@ public static ContextScope methodEnter(@Advice.Argument(1) final HttpMethod http return scope; } catch (BlockingException e) { CallDepthThreadLocalMap.reset(HttpClient.class); + if (scope != null) { + final AgentSpan span = spanFromScope(scope); + try { + DECORATE.onError(span, e); + DECORATE.beforeFinish(span); + } finally { + scope.close(); + span.finish(); + } + } // re-throw blocking exceptions throw e; } diff --git a/dd-java-agent/instrumentation/commons-httpclient-2.0/src/test/groovy/CommonsHttpClientBlockingForkedTest.groovy b/dd-java-agent/instrumentation/commons-httpclient-2.0/src/test/groovy/CommonsHttpClientBlockingForkedTest.groovy new file mode 100644 index 00000000000..f33b6a2932b --- /dev/null +++ b/dd-java-agent/instrumentation/commons-httpclient-2.0/src/test/groovy/CommonsHttpClientBlockingForkedTest.groovy @@ -0,0 +1,70 @@ +import datadog.appsec.api.blocking.BlockingContentType +import datadog.appsec.api.blocking.BlockingException +import datadog.trace.agent.test.InstrumentationSpecification +import datadog.trace.api.gateway.Flow +import datadog.trace.api.gateway.RequestContextSlot +import datadog.trace.bootstrap.CallDepthThreadLocalMap +import datadog.trace.bootstrap.instrumentation.api.AgentTracer +import datadog.trace.bootstrap.instrumentation.api.TagContext +import org.apache.commons.httpclient.HttpClient +import org.apache.commons.httpclient.methods.GetMethod + +import java.util.function.BiFunction + +import static datadog.trace.api.gateway.Events.EVENTS + +/** Forked so AppSec configuration is applied before instrumentation is installed. */ +class CommonsHttpClientBlockingForkedTest extends InstrumentationSpecification { + @Override + protected void configurePreAgent() { + super.configurePreAgent() + injectSysConfig('appsec.enabled', 'true') + injectSysConfig('appsec.rasp.enabled', 'true') + } + + def 'blocking restores the parent and finishes each client span'() { + given: + def subscription = AgentTracer.get().getSubscriptionService(RequestContextSlot.APPSEC) + def blockedSpans = [] + def flow = Stub(Flow) { + getAction() >> new Flow.Action.RequestBlockingAction(403, BlockingContentType.JSON) + } + subscription.registerCallback(EVENTS.httpClientRequest(), { ctx, request -> + blockedSpans.add(AgentTracer.activeSpan()) + flow + } as BiFunction) + def parent = TEST_TRACER.startSpan('test', 'parent', + new TagContext().withRequestContextDataAppSec(new Object())) + def parentScope = AgentTracer.activateSpan(parent) + def client = new HttpClient() + + when: + 2.times { + try { + client.executeMethod(new GetMethod('http://localhost:1/blocked')) + assert false: 'The request must be blocked before reaching the client' + } catch (BlockingException expected) { + assert AgentTracer.activeSpan().is(parent) + assert CallDepthThreadLocalMap.getCallDepth(HttpClient) == 0 + assert blockedSpans.last().finished + } + } + parentScope.close() + parentScope = null + parent.finish() + + then: + blockedSpans.size() == 2 + blockedSpans.every { it.parentId == parent.spanId } + TEST_WRITER.waitForTraces(1) + TEST_WRITER.size() == 1 + TEST_WRITER[0].size() == 3 + + cleanup: + parentScope?.close() + if (parent != null && !parent.finished) { + parent.finish() + } + subscription.reset() + } +} diff --git a/dd-java-agent/instrumentation/java/java-net/java-net-11.0/src/main/java11/datadog/trace/instrumentation/httpclient/SendAdvice.java b/dd-java-agent/instrumentation/java/java-net/java-net-11.0/src/main/java11/datadog/trace/instrumentation/httpclient/SendAdvice.java index 389d0a9ff6f..c57ecda7e43 100644 --- a/dd-java-agent/instrumentation/java/java-net/java-net-11.0/src/main/java11/datadog/trace/instrumentation/httpclient/SendAdvice.java +++ b/dd-java-agent/instrumentation/java/java-net/java-net-11.0/src/main/java11/datadog/trace/instrumentation/httpclient/SendAdvice.java @@ -20,6 +20,7 @@ public class SendAdvice { @Advice.OnMethodEnter(suppress = Throwable.class) public static ContextScope methodEnter( @Advice.Argument(value = 0) final HttpRequest httpRequest) { + ContextScope scope = null; try { if (DECORATE.isAgentRequest(httpRequest)) { return null; @@ -34,7 +35,7 @@ public static ContextScope methodEnter( } DECORATE.allowContextInjection(); final AgentSpan span = startSpan(INSTRUMENTATION_NAME, OPERATION_NAME); - final ContextScope scope = activateSpan(span); + scope = activateSpan(span); DECORATE.afterStart(span); DECORATE.onRequest(span, httpRequest); @@ -44,6 +45,16 @@ public static ContextScope methodEnter( } catch (BlockingException e) { CallDepthThreadLocalMap.reset(HttpClient.class); DECORATE.blockContextInjection(); + if (scope != null) { + final AgentSpan span = spanFromScope(scope); + try { + DECORATE.onError(span, e); + DECORATE.beforeFinish(span); + } finally { + scope.close(); + span.finish(); + } + } // re-throw blocking exceptions throw e; } diff --git a/dd-java-agent/instrumentation/java/java-net/java-net-11.0/src/main/java11/datadog/trace/instrumentation/httpclient/SendAsyncAdvice.java b/dd-java-agent/instrumentation/java/java-net/java-net-11.0/src/main/java11/datadog/trace/instrumentation/httpclient/SendAsyncAdvice.java index f9838510417..333596606e9 100644 --- a/dd-java-agent/instrumentation/java/java-net/java-net-11.0/src/main/java11/datadog/trace/instrumentation/httpclient/SendAsyncAdvice.java +++ b/dd-java-agent/instrumentation/java/java-net/java-net-11.0/src/main/java11/datadog/trace/instrumentation/httpclient/SendAsyncAdvice.java @@ -22,6 +22,7 @@ public class SendAsyncAdvice { public static ContextScope methodEnter( @Advice.Argument(value = 0) final HttpRequest httpRequest, @Advice.Argument(value = 1, readOnly = false) HttpResponse.BodyHandler bodyHandler) { + ContextScope scope = null; try { if (DECORATE.isAgentRequest(httpRequest)) { return null; @@ -36,7 +37,7 @@ public static ContextScope methodEnter( } DECORATE.allowContextInjection(); final AgentSpan span = startSpan(INSTRUMENTATION_NAME, OPERATION_NAME); - final ContextScope scope = activateSpan(span); + scope = activateSpan(span); if (bodyHandler != null) { // Pass span directly — BodyHandlerWrapper captures the continuation lazily in apply(), // only once response headers arrive. This avoids leaking a continuation when the @@ -52,6 +53,16 @@ public static ContextScope methodEnter( } catch (BlockingException e) { CallDepthThreadLocalMap.reset(HttpClient.class); DECORATE.blockContextInjection(); + if (scope != null) { + final AgentSpan span = spanFromScope(scope); + try { + DECORATE.onError(span, e); + DECORATE.beforeFinish(span); + } finally { + scope.close(); + span.finish(); + } + } // re-throw blocking exceptions throw e; } diff --git a/dd-java-agent/instrumentation/java/java-net/java-net-11.0/src/test/groovy/datadog/trace/instrumentation/httpclient/JavaHttpClientBlockingForkedTest.groovy b/dd-java-agent/instrumentation/java/java-net/java-net-11.0/src/test/groovy/datadog/trace/instrumentation/httpclient/JavaHttpClientBlockingForkedTest.groovy new file mode 100644 index 00000000000..b372062a774 --- /dev/null +++ b/dd-java-agent/instrumentation/java/java-net/java-net-11.0/src/test/groovy/datadog/trace/instrumentation/httpclient/JavaHttpClientBlockingForkedTest.groovy @@ -0,0 +1,82 @@ +package datadog.trace.instrumentation.httpclient + +import datadog.appsec.api.blocking.BlockingContentType +import datadog.appsec.api.blocking.BlockingException +import datadog.trace.agent.test.InstrumentationSpecification +import datadog.trace.api.gateway.Flow +import datadog.trace.api.gateway.RequestContextSlot +import datadog.trace.bootstrap.CallDepthThreadLocalMap +import datadog.trace.bootstrap.instrumentation.api.AgentTracer +import datadog.trace.bootstrap.instrumentation.api.TagContext +import java.net.http.HttpClient +import java.net.http.HttpRequest +import java.net.http.HttpResponse + +import java.util.function.BiFunction + +import static datadog.trace.api.gateway.Events.EVENTS + +/** Forked so AppSec configuration is applied before instrumentation is installed. */ +class JavaHttpClientBlockingForkedTest extends InstrumentationSpecification { + @Override + protected void configurePreAgent() { + super.configurePreAgent() + injectSysConfig('appsec.enabled', 'true') + injectSysConfig('appsec.rasp.enabled', 'true') + } + + def 'blocking restores the parent and finishes each client span'() { + given: + def subscription = AgentTracer.get().getSubscriptionService(RequestContextSlot.APPSEC) + def blockedSpans = [] + def flow = Stub(Flow) { + getAction() >> new Flow.Action.RequestBlockingAction(403, BlockingContentType.JSON) + } + subscription.registerCallback(EVENTS.httpClientRequest(), { ctx, clientRequest -> + blockedSpans.add(AgentTracer.activeSpan()) + flow + } as BiFunction) + def parent = TEST_TRACER.startSpan('test', 'parent', + new TagContext().withRequestContextDataAppSec(new Object())) + def parentScope = AgentTracer.activateSpan(parent) + def client = HttpClient.newHttpClient() + def request = HttpRequest.newBuilder(URI.create('http://localhost:1/blocked')).build() + + when: + 2.times { + try { + if (async) { + client.sendAsync(request, HttpResponse.BodyHandlers.discarding()) + } else { + client.send(request, HttpResponse.BodyHandlers.discarding()) + } + assert false: 'The request must be blocked before reaching the client' + } catch (BlockingException expected) { + assert AgentTracer.activeSpan().is(parent) + assert CallDepthThreadLocalMap.getCallDepth(HttpClient) == 0 + assert !JavaNetClientDecorator.DECORATE.isContextInjectionAllowed() + assert blockedSpans.last().finished + } + } + parentScope.close() + parentScope = null + parent.finish() + + then: + blockedSpans.size() == 2 + blockedSpans.every { it.parentId == parent.spanId } + TEST_WRITER.waitForTraces(1) + TEST_WRITER.size() == 1 + TEST_WRITER[0].size() == 3 + + cleanup: + parentScope?.close() + if (parent != null && !parent.finished) { + parent.finish() + } + subscription.reset() + + where: + async << [false, true] + } +} From a6f987b002fea50f6888320b7ed0cf8c54ce21f6 Mon Sep 17 00:00:00 2001 From: Andrea Marziali Date: Wed, 23 Sep 2026 16:17:58 +0200 Subject: [PATCH 28/35] Prevent AI Guard HTTP pool workers from retaining request context --- dd-java-agent/agent-aiguard/build.gradle | 1 + .../com/datadog/aiguard/AIGuardInternal.java | 14 +- .../AIGuardContextPropagationTest.java | 150 ++++++++++++++++++ 3 files changed, 164 insertions(+), 1 deletion(-) create mode 100644 dd-java-agent/agent-aiguard/src/test/java/com/datadog/aiguard/AIGuardContextPropagationTest.java diff --git a/dd-java-agent/agent-aiguard/build.gradle b/dd-java-agent/agent-aiguard/build.gradle index 7047ed3d34c..5cd5900dc56 100644 --- a/dd-java-agent/agent-aiguard/build.gradle +++ b/dd-java-agent/agent-aiguard/build.gradle @@ -22,6 +22,7 @@ dependencies { implementation project(':communication') testImplementation project(':utils:test-utils') + testImplementation project(':dd-trace-core') testImplementation libs.bundles.junit5 testImplementation libs.bundles.mockito testImplementation('org.skyscreamer:jsonassert:1.5.3') diff --git a/dd-java-agent/agent-aiguard/src/main/java/com/datadog/aiguard/AIGuardInternal.java b/dd-java-agent/agent-aiguard/src/main/java/com/datadog/aiguard/AIGuardInternal.java index 60b51ec0a4b..735c7108cc8 100644 --- a/dd-java-agent/agent-aiguard/src/main/java/com/datadog/aiguard/AIGuardInternal.java +++ b/dd-java-agent/agent-aiguard/src/main/java/com/datadog/aiguard/AIGuardInternal.java @@ -364,7 +364,7 @@ public Evaluation evaluate(final List messages, final Options options) .url(url) .method("POST", new MoshiJsonRequestBody(moshi, messages, meta)); headers.forEach(request::header); - try (final Response response = client.newCall(request.build()).execute()) { + try (final Response response = executeRequest(tracer, request.build())) { final Map result; try { result = parseResponseBody(response); @@ -455,6 +455,18 @@ public Evaluation evaluate(final List messages, final Options options) } } + private Response executeRequest(final AgentTracer.TracerAPI tracer, final Request request) + throws IOException { + final boolean asyncPropagation = tracer.isAsyncPropagationEnabled(); + try { + // The synchronous HTTP call must not pass its context to pool maintenance threads. + tracer.setAsyncPropagationEnabled(false); + return client.newCall(request).execute(); + } finally { + tracer.setAsyncPropagationEnabled(asyncPropagation); + } + } + @SuppressWarnings("unchecked") private Map parseResponseBody(final Response response) throws IOException { final ResponseBody body = response.body(); diff --git a/dd-java-agent/agent-aiguard/src/test/java/com/datadog/aiguard/AIGuardContextPropagationTest.java b/dd-java-agent/agent-aiguard/src/test/java/com/datadog/aiguard/AIGuardContextPropagationTest.java new file mode 100644 index 00000000000..aaaec185202 --- /dev/null +++ b/dd-java-agent/agent-aiguard/src/test/java/com/datadog/aiguard/AIGuardContextPropagationTest.java @@ -0,0 +1,150 @@ +package com.datadog.aiguard; + +import static java.util.Collections.emptyMap; +import static java.util.Collections.singletonList; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import datadog.context.ContextScope; +import datadog.trace.api.aiguard.AIGuard; +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.bootstrap.instrumentation.api.AgentTracer; +import datadog.trace.common.writer.ListWriter; +import datadog.trace.core.CoreTracer; +import datadog.trace.core.DDSpan; +import java.io.IOException; +import java.util.List; +import java.util.Properties; +import okhttp3.Call; +import okhttp3.HttpUrl; +import okhttp3.MediaType; +import okhttp3.OkHttpClient; +import okhttp3.Protocol; +import okhttp3.Request; +import okhttp3.Response; +import okhttp3.ResponseBody; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; + +class AIGuardContextPropagationTest { + private AgentTracer.TracerAPI originalTracer; + private CoreTracer tracer; + private ListWriter writer; + + @BeforeEach + void setUp() { + originalTracer = AgentTracer.get(); + writer = new ListWriter(); + } + + @AfterEach + void tearDown() { + AgentTracer.forceRegister(originalTracer); + tracer.close(); + } + + @ParameterizedTest + @CsvSource({ + "true, ALLOW, 100", + "false, ALLOW, 100", + "true, BLOCK, 100", + "false, BLOCK, 100", + "true, ERROR, 100", + "false, ERROR, 100", + "true, ALLOW, 1", + "false, ALLOW, 1", + "true, BLOCK, 1", + "false, BLOCK, 1", + "true, ERROR, 1", + "false, ERROR, 1" + }) + void isolatesHttpWorkAndRestoresCaller( + boolean callerPropagation, Outcome outcome, int scopeDepthLimit) throws Exception { + Properties properties = new Properties(); + properties.setProperty("trace.scope.depth.limit", Integer.toString(scopeDepthLimit)); + tracer = + CoreTracer.builder() + .withProperties(properties) + .writer(writer) + .strictTraceWrites(true) + .build(); + AgentTracer.forceRegister(tracer); + HttpUrl url = HttpUrl.get("http://localhost/evaluate"); + OkHttpClient client = mock(OkHttpClient.class); + Call call = mock(Call.class); + when(client.newCall(any(Request.class))).thenReturn(call); + when(call.execute()) + .thenAnswer( + invocation -> { + DDSpan span = (DDSpan) tracer.activeSpan(); + assertEquals( + scopeDepthLimit == 1 ? "parent" : "ai_guard", span.getOperationName().toString()); + assertFalse(tracer.isAsyncPropagationEnabled()); + if (outcome == Outcome.ERROR) { + throw new IOException("Transport failed"); + } + String action = outcome == Outcome.BLOCK ? "DENY" : "ALLOW"; + return new Response.Builder() + .request(new Request.Builder().url(url).build()) + .protocol(Protocol.HTTP_1_1) + .code(200) + .message("OK") + .body( + ResponseBody.create( + MediaType.get("application/json"), + "{\"data\":{\"attributes\":{\"action\":\"" + + action + + "\",\"is_blocking_enabled\":true}}}")) + .build(); + }); + AIGuardInternal evaluator = new AIGuardInternal(url, emptyMap(), client); + List messages = singletonList(AIGuard.Message.message("user", "Hello")); + AgentSpan parent = tracer.startSpan("test", "parent"); + try (ContextScope ignored = tracer.activateSpan(parent)) { + tracer.setAsyncPropagationEnabled(callerPropagation); + if (outcome == Outcome.ERROR) { + assertThrows( + AIGuard.AIGuardClientError.class, + () -> evaluator.evaluate(messages, AIGuard.Options.DEFAULT)); + } else if (outcome == Outcome.BLOCK) { + assertThrows( + AIGuard.AIGuardAbortError.class, + () -> evaluator.evaluate(messages, AIGuard.Options.DEFAULT)); + } else { + assertEquals( + AIGuard.Action.ALLOW, + evaluator.evaluate(messages, AIGuard.Options.DEFAULT).getAction()); + } + assertSame(parent, tracer.activeSpan()); + assertEquals(callerPropagation, tracer.isAsyncPropagationEnabled()); + verify(call).execute(); + } finally { + parent.finish(); + } + writer.waitForTraces(1); + assertEquals(1, writer.size()); + assertEquals(2, writer.get(0).size()); + DDSpan evaluation = + writer.get(0).stream() + .filter(span -> "ai_guard".contentEquals(span.getOperationName())) + .findFirst() + .get(); + assertEquals(parent.getSpanId(), evaluation.getParentId()); + assertTrue(evaluation.isFinished()); + } + + enum Outcome { + ALLOW, + BLOCK, + ERROR + } +} From 5296510eb8ca80e487536b2c43a78177093756d2 Mon Sep 17 00:00:00 2001 From: Andrea Marziali Date: Wed, 23 Sep 2026 17:17:27 +0200 Subject: [PATCH 29/35] Use table test --- .../AIGuardContextPropagationTest.java | 25 +++++++------------ 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/dd-java-agent/agent-aiguard/src/test/java/com/datadog/aiguard/AIGuardContextPropagationTest.java b/dd-java-agent/agent-aiguard/src/test/java/com/datadog/aiguard/AIGuardContextPropagationTest.java index aaaec185202..20fa2076010 100644 --- a/dd-java-agent/agent-aiguard/src/test/java/com/datadog/aiguard/AIGuardContextPropagationTest.java +++ b/dd-java-agent/agent-aiguard/src/test/java/com/datadog/aiguard/AIGuardContextPropagationTest.java @@ -32,8 +32,7 @@ import okhttp3.ResponseBody; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.CsvSource; +import org.tabletest.junit.TableTest; class AIGuardContextPropagationTest { private AgentTracer.TracerAPI originalTracer; @@ -52,20 +51,14 @@ void tearDown() { tracer.close(); } - @ParameterizedTest - @CsvSource({ - "true, ALLOW, 100", - "false, ALLOW, 100", - "true, BLOCK, 100", - "false, BLOCK, 100", - "true, ERROR, 100", - "false, ERROR, 100", - "true, ALLOW, 1", - "false, ALLOW, 1", - "true, BLOCK, 1", - "false, BLOCK, 1", - "true, ERROR, 1", - "false, ERROR, 1" + @TableTest({ + "scenario | callerPropagation | outcome | scopeDepthLimit", + "allow with evaluation scope | {true, false} | ALLOW | 100 ", + "block with evaluation scope | {true, false} | BLOCK | 100 ", + "error with evaluation scope | {true, false} | ERROR | 100 ", + "allow at scope depth limit | {true, false} | ALLOW | 1 ", + "block at scope depth limit | {true, false} | BLOCK | 1 ", + "error at scope depth limit | {true, false} | ERROR | 1 " }) void isolatesHttpWorkAndRestoresCaller( boolean callerPropagation, Outcome outcome, int scopeDepthLimit) throws Exception { From eaf9e2fd190529102d8594d152ad778585bc82c4 Mon Sep 17 00:00:00 2001 From: Andrea Marziali Date: Wed, 23 Sep 2026 17:04:58 +0200 Subject: [PATCH 30/35] Fix continuation cleanup when Apache async HTTP submission fails --- .../ApacheHttpAsyncClientInstrumentation.java | 20 +++++-- ...ttpAsyncClientSubmissionFailureTest.groovy | 57 +++++++++++++++++++ 2 files changed, 71 insertions(+), 6 deletions(-) create mode 100644 dd-java-agent/instrumentation/apache-httpclient/apache-httpasyncclient-4.0/src/test/groovy/ApacheHttpAsyncClientSubmissionFailureTest.groovy diff --git a/dd-java-agent/instrumentation/apache-httpclient/apache-httpasyncclient-4.0/src/main/java/datadog/trace/instrumentation/apachehttpasyncclient/ApacheHttpAsyncClientInstrumentation.java b/dd-java-agent/instrumentation/apache-httpclient/apache-httpasyncclient-4.0/src/main/java/datadog/trace/instrumentation/apachehttpasyncclient/ApacheHttpAsyncClientInstrumentation.java index 63f167ab659..6c728610980 100644 --- a/dd-java-agent/instrumentation/apache-httpclient/apache-httpasyncclient-4.0/src/main/java/datadog/trace/instrumentation/apachehttpasyncclient/ApacheHttpAsyncClientInstrumentation.java +++ b/dd-java-agent/instrumentation/apache-httpclient/apache-httpasyncclient-4.0/src/main/java/datadog/trace/instrumentation/apachehttpasyncclient/ApacheHttpAsyncClientInstrumentation.java @@ -96,13 +96,14 @@ public static class ClientAdvice { public static AgentSpan methodEnter( @Advice.Argument(value = 0, readOnly = false) HttpAsyncRequestProducer requestProducer, @Advice.Argument(2) HttpContext context, - @Advice.Argument(value = 3, readOnly = false) FutureCallback futureCallback) { + @Advice.Argument(value = 3, readOnly = false) FutureCallback futureCallback, + @Advice.Local("parentContinuation") ContextContinuation parentContinuation) { if (!(requestProducer instanceof DelegatingRequestProducer)) { requestProducer = new DelegatingRequestProducer(requestProducer); } - final ContextContinuation parentContinuation = currentContext().capture(); + parentContinuation = currentContext().capture(); final AgentSpan clientSpan = startSpan(APACHE_HTTPASYNCCLIENT.toString(), HTTP_REQUEST); DECORATE.afterStart(clientSpan); ((DelegatingRequestProducer) requestProducer).setSpan(clientSpan); @@ -116,12 +117,19 @@ public static AgentSpan methodEnter( @Advice.OnMethodExit(onThrowable = Throwable.class, suppress = Throwable.class) public static void methodExit( @Advice.Enter final AgentSpan span, - @Advice.Return final Object result, + @Advice.Local("parentContinuation") final ContextContinuation parentContinuation, @Advice.Thrown final Throwable throwable) { if (throwable != null) { - DECORATE.onError(span, throwable); - DECORATE.beforeFinish(span); - span.finish(); + try { + DECORATE.onError(span, throwable); + DECORATE.beforeFinish(span); + span.finish(); + } finally { + // Submission failed before a callback could take ownership. + if (parentContinuation != null) { + parentContinuation.release(); + } + } } } } diff --git a/dd-java-agent/instrumentation/apache-httpclient/apache-httpasyncclient-4.0/src/test/groovy/ApacheHttpAsyncClientSubmissionFailureTest.groovy b/dd-java-agent/instrumentation/apache-httpclient/apache-httpasyncclient-4.0/src/test/groovy/ApacheHttpAsyncClientSubmissionFailureTest.groovy new file mode 100644 index 00000000000..42d55ed3cb9 --- /dev/null +++ b/dd-java-agent/instrumentation/apache-httpclient/apache-httpasyncclient-4.0/src/test/groovy/ApacheHttpAsyncClientSubmissionFailureTest.groovy @@ -0,0 +1,57 @@ +import datadog.trace.agent.test.InstrumentationSpecification +import org.apache.http.client.methods.HttpGet +import org.apache.http.impl.nio.client.HttpAsyncClients +import org.apache.http.nio.client.methods.HttpAsyncMethods +import org.apache.http.nio.protocol.BasicAsyncResponseConsumer + +import java.util.concurrent.ExecutionException +import java.util.concurrent.TimeUnit + +import static datadog.trace.agent.test.utils.TraceUtils.runUnderTrace + +class ApacheHttpAsyncClientSubmissionFailureTest extends InstrumentationSpecification { + + def 'rejected submission releases the parent continuation: closed=#closed'() { + setup: + def client = HttpAsyncClients.createDefault() + if (closed) { + client.start() + client.close() + } + def producer = HttpAsyncMethods.create(new HttpGet('http://localhost/')) + def consumer = new BasicAsyncResponseConsumer() + Throwable failure = null + + when: + runUnderTrace('parent') { + try { + client.execute(producer, consumer, null).get(5, TimeUnit.SECONDS) + } catch (Exception e) { + // 4.0 throws directly; newer versions report rejection through the future. + failure = e instanceof ExecutionException ? e.cause : e + } + } + + then: + failure instanceof IllegalStateException + TEST_WRITER.waitForTraces(1) + TEST_WRITER.size() == 1 + def trace = TEST_WRITER.get(0) + trace.size() == 2 + def parent = trace.find { it.operationName.toString() == 'parent' } + def child = trace.find { it != parent } + parent != null + child.parentId == parent.spanId + child.error + child.getTag('error.type') == failure.class.name + child.getTag('component').toString() == 'apache-httpasyncclient' + + cleanup: + producer.close() + consumer.close() + client.close() + + where: + closed << [false, true] + } +} From 1d74bd388d771c979c932a7c60de613f37e1890a Mon Sep 17 00:00:00 2001 From: Andrea Marziali Date: Wed, 23 Sep 2026 17:30:03 +0200 Subject: [PATCH 31/35] Prevent Akka HTTP timeout cleanup from retaining request context --- .../bytebuddy/matcher/ignored_class_name.trie | 1 + .../groovy/AkkaHttpTimeoutCleanupTest.groovy | 77 +++++++++++++++++++ ...syncPropagatingDisableInstrumentation.java | 11 ++- 3 files changed, 88 insertions(+), 1 deletion(-) create mode 100644 dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/baseTest/groovy/AkkaHttpTimeoutCleanupTest.groovy diff --git a/dd-java-agent/agent-tooling/src/main/resources/datadog/trace/agent/tooling/bytebuddy/matcher/ignored_class_name.trie b/dd-java-agent/agent-tooling/src/main/resources/datadog/trace/agent/tooling/bytebuddy/matcher/ignored_class_name.trie index e86702c934c..e719f4684f1 100644 --- a/dd-java-agent/agent-tooling/src/main/resources/datadog/trace/agent/tooling/bytebuddy/matcher/ignored_class_name.trie +++ b/dd-java-agent/agent-tooling/src/main/resources/datadog/trace/agent/tooling/bytebuddy/matcher/ignored_class_name.trie @@ -133,6 +133,7 @@ 0 akka.http.impl.engine.client.PoolMasterActor 0 akka.http.impl.engine.client.pool.NewHostConnectionPool$* 0 akka.http.impl.engine.http2.Http2Ext +0 akka.http.impl.engine.server.HttpServerBluePrint$TimeoutAccessImpl 0 akka.http.impl.engine.server.HttpServerBluePrint$TimeoutAccessImpl$* 0 akka.http.impl.util.StreamUtils$* # saves ~0.1s skipping ~233 classes diff --git a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/baseTest/groovy/AkkaHttpTimeoutCleanupTest.groovy b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/baseTest/groovy/AkkaHttpTimeoutCleanupTest.groovy new file mode 100644 index 00000000000..99c1ce703cc --- /dev/null +++ b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/baseTest/groovy/AkkaHttpTimeoutCleanupTest.groovy @@ -0,0 +1,77 @@ +import akka.actor.ActorSystem +import akka.actor.Cancellable +import akka.http.javadsl.model.HttpRequest +import akka.stream.ActorMaterializer +import datadog.trace.agent.test.InstrumentationSpecification +import scala.concurrent.Await +import scala.concurrent.Promise$ +import scala.concurrent.duration.Duration +import scala.runtime.BoxedUnit +import spock.util.concurrent.PollingConditions + +import java.util.concurrent.TimeUnit + +import static datadog.trace.agent.test.utils.TraceUtils.runUnderTrace +import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.isAsyncPropagationEnabled +import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.setAsyncPropagationEnabled + +class AkkaHttpTimeoutCleanupTest extends InstrumentationSpecification { + + def 'timeout cleanup does not retain a request while its entity is unfinished: propagation=#enabled'() { + setup: + def system = ActorSystem.create('timeout-cleanup') + def materializer = ActorMaterializer.create(system) + def requestEnd = Promise$.MODULE$.apply() + def type = Class.forName('akka.http.impl.engine.server.HttpServerBluePrint$TimeoutAccessImpl') + def constructor = type.declaredConstructors[0] + constructor.accessible = true + def arguments = [ + HttpRequest.create(), + Duration.create(1, TimeUnit.DAYS), + requestEnd.future(), + null, + materializer + ] + if (constructor.parameterCount == 6) { + arguments.add(system.log()) + } + def access = constructor.newInstance(arguments as Object[]) + def clear = type.getDeclaredMethod('clear') + clear.accessible = true + + when: + runUnderTrace('request') { + setAsyncPropagationEnabled(enabled) + clear.invoke(access) + assert isAsyncPropagationEnabled() == enabled + } + + then: + // The request trace must finish without waiting for the entity or its timeout cleanup. + TEST_WRITER.waitForTraces(1) + TEST_WRITER.get(0).size() == 1 + !requestEnd.isCompleted() + + when: + requestEnd.success(BoxedUnit.UNIT) + def setup = Await.result(access.get(), Duration.create(5, TimeUnit.SECONDS)) + def scheduledTask = setup.class.getDeclaredMethod('scheduledTask') + scheduledTask.accessible = true + def task = scheduledTask.invoke(setup) as Cancellable + + then: + new PollingConditions(timeout: 5).eventually { + assert task.isCancelled() + } + + cleanup: + requestEnd?.trySuccess(BoxedUnit.UNIT) + materializer?.shutdown() + if (system != null) { + Await.result(system.terminate(), Duration.create(10, TimeUnit.SECONDS)) + } + + where: + enabled << [true, false] + } +} diff --git a/dd-java-agent/instrumentation/java/java-concurrent/java-concurrent-1.8/src/main/java/datadog/trace/instrumentation/java/concurrent/AsyncPropagatingDisableInstrumentation.java b/dd-java-agent/instrumentation/java/java-concurrent/java-concurrent-1.8/src/main/java/datadog/trace/instrumentation/java/concurrent/AsyncPropagatingDisableInstrumentation.java index 82578773d74..89dd1c93e43 100644 --- a/dd-java-agent/instrumentation/java/java-concurrent/java-concurrent-1.8/src/main/java/datadog/trace/instrumentation/java/concurrent/AsyncPropagatingDisableInstrumentation.java +++ b/dd-java-agent/instrumentation/java/java-concurrent/java-concurrent-1.8/src/main/java/datadog/trace/instrumentation/java/concurrent/AsyncPropagatingDisableInstrumentation.java @@ -118,7 +118,8 @@ public String[] knownMatchingTypes() { "io.netty.handler.timeout.IdleStateHandler", "io.grpc.netty.shaded.io.netty.handler.timeout.IdleStateHandler", "com.linecorp.armeria.client.HttpClientFactory", - "com.linecorp.armeria.client.HttpChannelPool" + "com.linecorp.armeria.client.HttpChannelPool", + "akka.http.impl.engine.server.HttpServerBluePrint$TimeoutAccessImpl" }; } @@ -141,6 +142,14 @@ public ElementMatcher hierarchyMatcher() { @Override public void methodAdvice(MethodTransformer transformer) { String advice = getClass().getName() + "$DisableAsyncAdvice"; + // Timeout cancellation is housekeeping; its request-entity future may never complete. + transformer.applyAdvice( + named("clear") + .and(takesNoArguments()) + .and( + isDeclaredBy( + named("akka.http.impl.engine.server.HttpServerBluePrint$TimeoutAccessImpl"))), + advice); transformer.applyAdvice(named("schedulePeriodically").and(isDeclaredBy(RX_WORKERS)), advice); transformer.applyAdvice( named("call").and(isDeclaredBy(named("rx.internal.operators.OperatorTimeoutBase"))), From be3821806d32d31e98b31862c13c17aa571ec4ec Mon Sep 17 00:00:00 2001 From: Andrea Marziali Date: Wed, 23 Sep 2026 17:55:43 +0200 Subject: [PATCH 32/35] Release Akka message continuations when discarded to dead letters --- .../groovy/AkkaDeadLetterTest.groovy | 89 +++++++++++++++++++ .../AkkaDeadLetterQueueInstrumentation.java | 72 +++++++++++++++ 2 files changed, 161 insertions(+) create mode 100644 dd-java-agent/instrumentation/akka/akka-actor-2.5/src/akka23Test/groovy/AkkaDeadLetterTest.groovy create mode 100644 dd-java-agent/instrumentation/akka/akka-actor-2.5/src/main/java/datadog/trace/instrumentation/akka/concurrent/AkkaDeadLetterQueueInstrumentation.java diff --git a/dd-java-agent/instrumentation/akka/akka-actor-2.5/src/akka23Test/groovy/AkkaDeadLetterTest.groovy b/dd-java-agent/instrumentation/akka/akka-actor-2.5/src/akka23Test/groovy/AkkaDeadLetterTest.groovy new file mode 100644 index 00000000000..666f3ee1e39 --- /dev/null +++ b/dd-java-agent/instrumentation/akka/akka-actor-2.5/src/akka23Test/groovy/AkkaDeadLetterTest.groovy @@ -0,0 +1,89 @@ +import akka.actor.ActorRef +import akka.actor.ActorSystem +import akka.actor.DeadLetter +import akka.actor.Props +import akka.actor.UntypedActor +import akka.testkit.TestKit +import akka.testkit.TestProbe +import datadog.trace.agent.test.InstrumentationSpecification +import scala.concurrent.duration.Duration + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.atomic.AtomicInteger + +import static datadog.trace.agent.test.utils.TraceUtils.basicSpan +import static datadog.trace.agent.test.utils.TraceUtils.runUnderTrace +import static java.util.concurrent.TimeUnit.SECONDS + +class AkkaDeadLetterTest extends InstrumentationSpecification { + def "release context for #count queued messages discarded on actor termination (dead letter: #deadLetter)"() { + setup: + def system = ActorSystem.create("dead-letter-test") + def entered = new CountDownLatch(1) + def stop = new CountDownLatch(1) + def processed = new AtomicInteger() + def actor = system.actorOf(Props.create(StoppingActor, entered, stop, processed)) + def watcher = new TestProbe(system) + watcher.watch(actor) + actor.tell("stop", ActorRef.noSender()) + assert entered.await(10, SECONDS) + + when: + runUnderTrace("parent") { + count.times { + def message = deadLetter ? new DeadLetter("discarded", system.deadLetters(), actor) : "discarded" + actor.tell(message, ActorRef.noSender()) + } + } + + then: + TEST_WRITER.empty + + when: + stop.countDown() + watcher.expectTerminated(actor, Duration.create(10, SECONDS)) + + then: + assertTraces(1) { + trace(1) { + basicSpan(it, "parent") + } + } + processed.get() == 0 + + cleanup: + stop?.countDown() + TestKit.shutdownActorSystem(system, Duration.create(10, SECONDS), true) + + where: + count | deadLetter + 1 | false + 10 | false + 1 | true + } + + static class StoppingActor extends UntypedActor { + private final CountDownLatch entered + private final CountDownLatch stop + private final AtomicInteger processed + + StoppingActor(CountDownLatch entered, CountDownLatch stop, AtomicInteger processed) { + this.entered = entered + this.stop = stop + this.processed = processed + } + + @Override + void onReceive(Object message) { + if (message == "stop") { + entered.countDown() + if (!stop.await(10, SECONDS)) { + throw new IllegalStateException("Timed out waiting to stop the actor") + } + context.stop(self) + } else { + processed.incrementAndGet() + } + } + } +} diff --git a/dd-java-agent/instrumentation/akka/akka-actor-2.5/src/main/java/datadog/trace/instrumentation/akka/concurrent/AkkaDeadLetterQueueInstrumentation.java b/dd-java-agent/instrumentation/akka/akka-actor-2.5/src/main/java/datadog/trace/instrumentation/akka/concurrent/AkkaDeadLetterQueueInstrumentation.java new file mode 100644 index 00000000000..9290f2ca889 --- /dev/null +++ b/dd-java-agent/instrumentation/akka/akka-actor-2.5/src/main/java/datadog/trace/instrumentation/akka/concurrent/AkkaDeadLetterQueueInstrumentation.java @@ -0,0 +1,72 @@ +package datadog.trace.instrumentation.akka.concurrent; + +import static datadog.trace.agent.tooling.bytebuddy.matcher.HierarchyMatchers.implementsInterface; +import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.nameStartsWith; +import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named; +import static java.util.Collections.singletonMap; +import static net.bytebuddy.matcher.ElementMatchers.isMethod; +import static net.bytebuddy.matcher.ElementMatchers.isPublic; +import static net.bytebuddy.matcher.ElementMatchers.returns; +import static net.bytebuddy.matcher.ElementMatchers.takesArgument; +import static net.bytebuddy.matcher.ElementMatchers.takesArguments; + +import akka.dispatch.Envelope; +import com.google.auto.service.AutoService; +import datadog.trace.agent.tooling.Instrumenter; +import datadog.trace.agent.tooling.InstrumenterModule; +import datadog.trace.bootstrap.InstrumentationContext; +import datadog.trace.bootstrap.instrumentation.java.concurrent.State; +import java.util.Map; +import net.bytebuddy.asm.Advice; +import net.bytebuddy.description.type.TypeDescription; +import net.bytebuddy.matcher.ElementMatcher; + +@AutoService(InstrumenterModule.class) +public class AkkaDeadLetterQueueInstrumentation extends InstrumenterModule.ContextTracking + implements Instrumenter.ForTypeHierarchy, Instrumenter.HasMethodAdvice { + + public AkkaDeadLetterQueueInstrumentation() { + super("akka_actor_receive", "akka_actor", "akka_concurrent", "java_concurrent"); + } + + @Override + public String hierarchyMarkerType() { + return "akka.dispatch.MessageQueue"; + } + + @Override + public ElementMatcher hierarchyMatcher() { + // Match the dead-letter queue without depending on Scala's anonymous-class numbering. + return nameStartsWith("akka.dispatch.Mailboxes$") + .and(implementsInterface(named("akka.dispatch.MessageQueue"))); + } + + @Override + public Map contextStore() { + return singletonMap("akka.dispatch.Envelope", State.class.getName()); + } + + @Override + public void methodAdvice(MethodTransformer transformer) { + transformer.applyAdvice( + isMethod() + .and(isPublic()) + .and(named("enqueue")) + .and(takesArguments(2)) + .and(takesArgument(0, named("akka.actor.ActorRef"))) + .and(takesArgument(1, named("akka.dispatch.Envelope"))) + .and(returns(void.class)), + getClass().getName() + "$EnqueueAdvice"); + } + + public static class EnqueueAdvice { + @Advice.OnMethodExit(onThrowable = Throwable.class, suppress = Throwable.class) + public static void afterEnqueue(@Advice.Argument(1) Envelope envelope) { + // Dead-letter publication discards the original envelope without invoking its actor. + State state = InstrumentationContext.get(Envelope.class, State.class).get(envelope); + if (state != null) { + state.closeContinuation(); + } + } + } +} From 9480acfb215f928a55f350e756d9ce605273279e Mon Sep 17 00:00:00 2001 From: Andrea Marziali Date: Thu, 24 Sep 2026 09:50:12 +0200 Subject: [PATCH 33/35] Add fixture cleanup --- .../smoketest/concurrent/AbstractConcurrentTest.groovy | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/dd-smoke-tests/concurrent/java-21/src/test/groovy/datadog/smoketest/concurrent/AbstractConcurrentTest.groovy b/dd-smoke-tests/concurrent/java-21/src/test/groovy/datadog/smoketest/concurrent/AbstractConcurrentTest.groovy index 00f87a94d6e..20c36a1786e 100644 --- a/dd-smoke-tests/concurrent/java-21/src/test/groovy/datadog/smoketest/concurrent/AbstractConcurrentTest.groovy +++ b/dd-smoke-tests/concurrent/java-21/src/test/groovy/datadog/smoketest/concurrent/AbstractConcurrentTest.groovy @@ -67,7 +67,15 @@ abstract class AbstractConcurrentTest extends AbstractSmokeTest { protected void receivedCorrectTrace() { waitForTrace(defaultPoll, checkTrace()) assert traceCount.get() == 1 - assert testedProcess.waitFor(TIMEOUT_SECS, SECONDS) + } + + def cleanupSpec() { + if (testedProcess == null) { + return + } + // Features may be skipped; finish the application before the superclass checks diagnostics. + assert testedProcess.waitFor(TIMEOUT_SECS, SECONDS): + "Concurrent application did not exit within ${TIMEOUT_SECS} seconds; see ${logFilePath}" assert testedProcess.exitValue() == 0 } } From a3b25f2987fd94affa4dad917e0b1487345941d9 Mon Sep 17 00:00:00 2001 From: Andrea Marziali Date: Thu, 24 Sep 2026 09:15:38 +0200 Subject: [PATCH 34/35] Prevent JDK virtual I/O threads from retaining request context --- .../bytebuddy/matcher/ignored_class_name.trie | 1 + .../java-lang/java-lang-21.0/build.gradle | 6 ++ .../lang/jdk21/PollerInstrumentation.java | 54 +++++++++++++++ .../jdk21/VirtualThreadInstrumentation.java | 16 ++++- .../HttpClientSelectorContextForkedTest.java | 47 +++++++++++++ .../lang/jdk21/PollerContextForkedTest.java | 69 +++++++++++++++++++ .../jdk21/VirtualThreadLifeCycleTest.java | 27 ++++++++ 7 files changed, 218 insertions(+), 2 deletions(-) create mode 100644 dd-java-agent/instrumentation/java/java-lang/java-lang-21.0/src/main/java/datadog/trace/instrumentation/java/lang/jdk21/PollerInstrumentation.java create mode 100644 dd-java-agent/instrumentation/java/java-lang/java-lang-21.0/src/test/java/testdog/trace/instrumentation/java/lang/jdk21/HttpClientSelectorContextForkedTest.java create mode 100644 dd-java-agent/instrumentation/java/java-lang/java-lang-21.0/src/test/java/testdog/trace/instrumentation/java/lang/jdk21/PollerContextForkedTest.java diff --git a/dd-java-agent/agent-tooling/src/main/resources/datadog/trace/agent/tooling/bytebuddy/matcher/ignored_class_name.trie b/dd-java-agent/agent-tooling/src/main/resources/datadog/trace/agent/tooling/bytebuddy/matcher/ignored_class_name.trie index e719f4684f1..d3fb308806a 100644 --- a/dd-java-agent/agent-tooling/src/main/resources/datadog/trace/agent/tooling/bytebuddy/matcher/ignored_class_name.trie +++ b/dd-java-agent/agent-tooling/src/main/resources/datadog/trace/agent/tooling/bytebuddy/matcher/ignored_class_name.trie @@ -107,6 +107,7 @@ 0 sun.rmi.server.* 0 sun.rmi.transport.* 0 sun.nio.ch.FileChannelImpl +0 sun.nio.ch.Poller$Pollers # USM needs it 0 sun.security.ssl.* 0 javax.net.ssl.SSLSocket diff --git a/dd-java-agent/instrumentation/java/java-lang/java-lang-21.0/build.gradle b/dd-java-agent/instrumentation/java/java-lang/java-lang-21.0/build.gradle index 2ca7778c773..473ec69ccb2 100644 --- a/dd-java-agent/instrumentation/java/java-lang/java-lang-21.0/build.gradle +++ b/dd-java-agent/instrumentation/java/java-lang/java-lang-21.0/build.gradle @@ -29,3 +29,9 @@ tasks.named("compileTestJava", JavaCompile) { dependencies { testImplementation project(':dd-java-agent:instrumentation:datadog:tracing:trace-annotation') } + +tasks.named('forkedTest', Test) { + jvmArgs '--add-opens=java.base/sun.nio.ch=ALL-UNNAMED', + '--add-opens=java.base/java.util.concurrent=ALL-UNNAMED', + '--add-opens=java.net.http/jdk.internal.net.http=ALL-UNNAMED' +} diff --git a/dd-java-agent/instrumentation/java/java-lang/java-lang-21.0/src/main/java/datadog/trace/instrumentation/java/lang/jdk21/PollerInstrumentation.java b/dd-java-agent/instrumentation/java/java-lang/java-lang-21.0/src/main/java/datadog/trace/instrumentation/java/lang/jdk21/PollerInstrumentation.java new file mode 100644 index 00000000000..079c920bc80 --- /dev/null +++ b/dd-java-agent/instrumentation/java/java-lang/java-lang-21.0/src/main/java/datadog/trace/instrumentation/java/lang/jdk21/PollerInstrumentation.java @@ -0,0 +1,54 @@ +package datadog.trace.instrumentation.java.lang.jdk21; + +import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named; +import static datadog.trace.bootstrap.instrumentation.api.Java8BytecodeBridge.rootContext; +import static net.bytebuddy.matcher.ElementMatchers.returns; +import static net.bytebuddy.matcher.ElementMatchers.takesArguments; + +import com.google.auto.service.AutoService; +import datadog.context.ContextScope; +import datadog.environment.JavaVirtualMachine; +import datadog.trace.agent.tooling.Instrumenter; +import datadog.trace.agent.tooling.InstrumenterModule; +import net.bytebuddy.asm.Advice; + +/** Keeps process-lifetime JDK I/O pollers independent of the first requesting thread. */ +@AutoService(InstrumenterModule.class) +public final class PollerInstrumentation extends InstrumenterModule.ContextTracking + implements Instrumenter.ForBootstrap, Instrumenter.ForSingleType, Instrumenter.HasMethodAdvice { + public PollerInstrumentation() { + super("java-lang", "java-lang-21", "virtual-thread"); + } + + @Override + public boolean isEnabled() { + return JavaVirtualMachine.isJavaVersionAtLeast(22) && super.isEnabled(); + } + + @Override + public String instrumentedType() { + return "sun.nio.ch.Poller$Pollers"; + } + + @Override + public void methodAdvice(MethodTransformer transformer) { + transformer.applyAdvice( + named("start").and(takesArguments(0)).and(returns(void.class)), + getClass().getName() + "$StartAdvice"); + } + + public static final class StartAdvice { + @Advice.OnMethodEnter(suppress = Throwable.class) + public static ContextScope enter() { + // Disabling async propagation alone still leaves the raw context on virtual threads. + return rootContext().attach(); + } + + @Advice.OnMethodExit(onThrowable = Throwable.class, suppress = Throwable.class) + public static void exit(@Advice.Enter ContextScope scope) { + if (scope != null) { + scope.close(); + } + } + } +} diff --git a/dd-java-agent/instrumentation/java/java-lang/java-lang-21.0/src/main/java/datadog/trace/instrumentation/java/lang/jdk21/VirtualThreadInstrumentation.java b/dd-java-agent/instrumentation/java/java-lang/java-lang-21.0/src/main/java/datadog/trace/instrumentation/java/lang/jdk21/VirtualThreadInstrumentation.java index 6fe60df8d46..df9d6c77168 100644 --- a/dd-java-agent/instrumentation/java/java-lang/java-lang-21.0/src/main/java/datadog/trace/instrumentation/java/lang/jdk21/VirtualThreadInstrumentation.java +++ b/dd-java-agent/instrumentation/java/java-lang/java-lang-21.0/src/main/java/datadog/trace/instrumentation/java/lang/jdk21/VirtualThreadInstrumentation.java @@ -25,6 +25,7 @@ import datadog.trace.bootstrap.instrumentation.java.lang.VirtualThreadState; import java.util.Collection; import java.util.Map; +import java.util.concurrent.Executor; import net.bytebuddy.asm.Advice; import net.bytebuddy.asm.Advice.OnMethodEnter; import net.bytebuddy.asm.Advice.OnMethodExit; @@ -104,7 +105,10 @@ public Map contextStore() { @Override public void methodAdvice(MethodTransformer transformer) { - transformer.applyAdvice(isConstructor(), getClass().getName() + "$Construct"); + transformer.applyAdvice( + isConstructor() + .and(takesArguments(Executor.class, String.class, int.class, Runnable.class)), + getClass().getName() + "$Construct"); transformer.applyAdvice(isMethod().and(named("mount")), getClass().getName() + "$Mount"); transformer.applyAdvice(isMethod().and(named("unmount")), getClass().getName() + "$Unmount"); transformer.applyAdvice( @@ -117,11 +121,19 @@ public void methodAdvice(MethodTransformer transformer) { public static final class Construct { @OnMethodExit(suppress = Throwable.class) - public static void afterInit(@Advice.This Object virtualThread) { + public static void afterInit( + @Advice.This Object virtualThread, @Advice.Argument(3) Runnable task) { Context context = currentContext(); if (context == rootContext()) { return; // No active context to propagate, avoid creating state } + // Since JDK 26 the HTTP client's selector can be a virtual thread. Its lifetime belongs to + // the client, so neither a continuation nor the raw creating request context belongs on it. + if (task.getClass() + .getName() + .equals("jdk.internal.net.http.HttpClientImpl$SelectorManager")) { + return; + } VirtualThreadState state = new VirtualThreadState(context, context.capture()); ContextStore store = InstrumentationContext.get(VIRTUAL_THREAD_CLASS_NAME, VIRTUAL_THREAD_STATE_CLASS_NAME); diff --git a/dd-java-agent/instrumentation/java/java-lang/java-lang-21.0/src/test/java/testdog/trace/instrumentation/java/lang/jdk21/HttpClientSelectorContextForkedTest.java b/dd-java-agent/instrumentation/java/java-lang/java-lang-21.0/src/test/java/testdog/trace/instrumentation/java/lang/jdk21/HttpClientSelectorContextForkedTest.java new file mode 100644 index 00000000000..0b1728ed2b7 --- /dev/null +++ b/dd-java-agent/instrumentation/java/java-lang/java-lang-21.0/src/test/java/testdog/trace/instrumentation/java/lang/jdk21/HttpClientSelectorContextForkedTest.java @@ -0,0 +1,47 @@ +package testdog.trace.instrumentation.java.lang.jdk21; + +import static datadog.trace.agent.test.assertions.SpanMatcher.span; +import static datadog.trace.agent.test.assertions.TraceMatcher.trace; +import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.activateSpan; +import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.activeSpan; +import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.startSpan; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assumptions.assumeTrue; +import static testdog.trace.instrumentation.java.lang.jdk21.PollerContextForkedTest.field; +import static testdog.trace.instrumentation.java.lang.jdk21.PollerContextForkedTest.virtualThreadState; + +import datadog.context.ContextScope; +import datadog.trace.agent.test.AbstractInstrumentationTest; +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import java.net.http.HttpClient; +import org.junit.jupiter.api.Test; + +class HttpClientSelectorContextForkedTest extends AbstractInstrumentationTest { + @Test + void selectorDoesNotRetainCreatingRequest() throws Exception { + assumeTrue(Runtime.version().feature() >= 26); + HttpClient client = null; + AgentSpan parent = startSpan("test", "client-owner"); + try { + try (ContextScope ignored = activateSpan(parent)) { + client = HttpClient.newHttpClient(); + Object impl = field(client.getClass(), "impl").get(client); + Thread selector = (Thread) field(impl.getClass(), "selmgrThread").get(impl); + assertTrue(selector.isVirtual()); + assertTrue(selector.isAlive()); + assertNull(virtualThreadState(selector), "live selector must not retain request context"); + assertSame(parent, activeSpan()); + } finally { + parent.finish(); + } + // Publish while the client is still alive: closing it must not be needed to release parent. + assertTraces(trace(span().root().operationName("client-owner"))); + } finally { + if (client != null) { + client.close(); + } + } + } +} diff --git a/dd-java-agent/instrumentation/java/java-lang/java-lang-21.0/src/test/java/testdog/trace/instrumentation/java/lang/jdk21/PollerContextForkedTest.java b/dd-java-agent/instrumentation/java/java-lang/java-lang-21.0/src/test/java/testdog/trace/instrumentation/java/lang/jdk21/PollerContextForkedTest.java new file mode 100644 index 00000000000..a2fb68eed29 --- /dev/null +++ b/dd-java-agent/instrumentation/java/java-lang/java-lang-21.0/src/test/java/testdog/trace/instrumentation/java/lang/jdk21/PollerContextForkedTest.java @@ -0,0 +1,69 @@ +package testdog.trace.instrumentation.java.lang.jdk21; + +import static datadog.trace.agent.test.assertions.SpanMatcher.span; +import static datadog.trace.agent.test.assertions.TraceMatcher.trace; +import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.activateSpan; +import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.activeSpan; +import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.startSpan; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import datadog.context.ContextScope; +import datadog.trace.agent.test.AbstractInstrumentationTest; +import datadog.trace.agent.tooling.bytebuddy.matcher.GlobalIgnores; +import datadog.trace.bootstrap.FieldBackedContextStores; +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import java.lang.reflect.Field; +import java.util.Set; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledForJreRange; +import org.junit.jupiter.api.condition.JRE; + +class PollerContextForkedTest extends AbstractInstrumentationTest { + @Test + @EnabledForJreRange(min = JRE.JAVA_22) + void firstPollerInitializationDoesNotRetainRequest() throws Exception { + // Set these before Poller.; changing them afterward cannot reconfigure its pollers. + // The fork isolates the properties and poller initialization from other tests. + System.setProperty("jdk.pollerMode", "VTHREAD_POLLERS"); + System.setProperty("jdk.readPollers", "1"); + System.setProperty("jdk.writePollers", "1"); + assertFalse(GlobalIgnores.isIgnored("sun.nio.ch.Poller$Pollers", false)); + assertTrue(GlobalIgnores.isIgnored("sun.nio.ch.Poller", false)); + AgentSpan parent = startSpan("test", "poller-owner"); + try (ContextScope ignored = activateSpan(parent)) { + Class poller = Class.forName("sun.nio.ch.Poller"); + Object pollers = field(poller, "POLLERS").get(null); + Object executor = field(pollers.getClass(), "executor").get(pollers); + @SuppressWarnings("unchecked") + Set threads = (Set) field(executor.getClass(), "threads").get(executor); + assertEquals(2, threads.size(), "one read poller and one write poller must be running"); + for (Thread thread : threads) { + assertTrue(thread.isVirtual()); + assertNull( + virtualThreadState(thread), "poller must retain neither context nor continuation"); + } + assertSame(parent, activeSpan(), "poller startup must restore the caller context"); + } finally { + parent.finish(); + } + assertTraces(trace(span().root().operationName("poller-owner"))); + } + + static Object virtualThreadState(Thread thread) { + int storeId = + FieldBackedContextStores.getContextStoreId( + "java.lang.VirtualThread", + "datadog.trace.bootstrap.instrumentation.java.lang.VirtualThreadState"); + return FieldBackedContextStores.getContextStore(storeId).get(thread); + } + + static Field field(Class type, String name) throws NoSuchFieldException { + Field field = type.getDeclaredField(name); + field.setAccessible(true); + return field; + } +} diff --git a/dd-java-agent/instrumentation/java/java-lang/java-lang-21.0/src/test/java/testdog/trace/instrumentation/java/lang/jdk21/VirtualThreadLifeCycleTest.java b/dd-java-agent/instrumentation/java/java-lang/java-lang-21.0/src/test/java/testdog/trace/instrumentation/java/lang/jdk21/VirtualThreadLifeCycleTest.java index 54690e14f73..e883503d7de 100644 --- a/dd-java-agent/instrumentation/java/java-lang/java-lang-21.0/src/test/java/testdog/trace/instrumentation/java/lang/jdk21/VirtualThreadLifeCycleTest.java +++ b/dd-java-agent/instrumentation/java/java-lang/java-lang-21.0/src/test/java/testdog/trace/instrumentation/java/lang/jdk21/VirtualThreadLifeCycleTest.java @@ -3,13 +3,20 @@ import static datadog.trace.agent.test.assertions.SpanMatcher.span; import static datadog.trace.agent.test.assertions.TraceMatcher.SORT_BY_START_TIME; import static datadog.trace.agent.test.assertions.TraceMatcher.trace; +import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.activateSpan; +import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.activeSpan; +import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.startSpan; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; +import datadog.context.ContextScope; import datadog.trace.agent.test.AbstractInstrumentationTest; import datadog.trace.api.CorrelationIdentifier; import datadog.trace.api.GlobalTracer; import datadog.trace.api.Trace; +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; import java.time.Duration; import java.util.ArrayList; import java.util.List; @@ -21,6 +28,26 @@ public class VirtualThreadLifeCycleTest extends AbstractInstrumentationTest { private static final Duration TIMEOUT = Duration.ofSeconds(10); + @Test + void propagationDoesNotDependOnInheritableThreadLocals() throws InterruptedException { + AgentSpan parent = startSpan("test", "parent"); + try (ContextScope ignored = activateSpan(parent)) { + for (boolean inheritThreadLocals : new boolean[] {true, false}) { + AtomicReference inherited = new AtomicReference<>(); + Thread child = + Thread.ofVirtual() + .inheritInheritableThreadLocals(inheritThreadLocals) + .start(() -> inherited.set(activeSpan())); + assertTrue(child.join(TIMEOUT)); + assertSame(parent, inherited.get()); + assertSame(parent, activeSpan()); + } + } finally { + parent.finish(); + } + assertTraces(trace(span().root().operationName("parent"))); + } + @DisplayName("test context restored after virtual thread remounts") @Test void testContextRestoredAfterVirtualThreadRemount() { From 7cc7f3f3b1d93761fc3499866f64dbd5bfe242e3 Mon Sep 17 00:00:00 2001 From: Andrea Marziali Date: Thu, 24 Sep 2026 09:38:02 +0200 Subject: [PATCH 35/35] Use a classvalue --- .../instrumentation/java/lang/VirtualThreadState.java | 10 ++++++++++ .../java/lang/jdk21/VirtualThreadInstrumentation.java | 6 +----- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/java/lang/VirtualThreadState.java b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/java/lang/VirtualThreadState.java index 658988bce94..f2c875a2719 100644 --- a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/java/lang/VirtualThreadState.java +++ b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/java/lang/VirtualThreadState.java @@ -2,6 +2,7 @@ import datadog.context.Context; import datadog.context.ContextContinuation; +import datadog.trace.api.GenericClassValue; /** * This class holds the saved context and scope continuation for a virtual thread. @@ -10,6 +11,15 @@ * mount/unmount. */ public final class VirtualThreadState { + private static final ClassValue PROPAGATE_CONTEXT = + GenericClassValue.of( + type -> !type.getName().equals("jdk.internal.net.http.HttpClientImpl$SelectorManager")); + + /** The HTTP selector belongs to its client, not to the request creating the client. */ + public static boolean shouldPropagateContext(Runnable task) { + return PROPAGATE_CONTEXT.get(task.getClass()); + } + /** The virtual thread's saved context (scope stack snapshot). */ private Context context; diff --git a/dd-java-agent/instrumentation/java/java-lang/java-lang-21.0/src/main/java/datadog/trace/instrumentation/java/lang/jdk21/VirtualThreadInstrumentation.java b/dd-java-agent/instrumentation/java/java-lang/java-lang-21.0/src/main/java/datadog/trace/instrumentation/java/lang/jdk21/VirtualThreadInstrumentation.java index df9d6c77168..acc968ff481 100644 --- a/dd-java-agent/instrumentation/java/java-lang/java-lang-21.0/src/main/java/datadog/trace/instrumentation/java/lang/jdk21/VirtualThreadInstrumentation.java +++ b/dd-java-agent/instrumentation/java/java-lang/java-lang-21.0/src/main/java/datadog/trace/instrumentation/java/lang/jdk21/VirtualThreadInstrumentation.java @@ -127,11 +127,7 @@ public static void afterInit( if (context == rootContext()) { return; // No active context to propagate, avoid creating state } - // Since JDK 26 the HTTP client's selector can be a virtual thread. Its lifetime belongs to - // the client, so neither a continuation nor the raw creating request context belongs on it. - if (task.getClass() - .getName() - .equals("jdk.internal.net.http.HttpClientImpl$SelectorManager")) { + if (!VirtualThreadState.shouldPropagateContext(task)) { return; } VirtualThreadState state = new VirtualThreadState(context, context.capture());