Only warn about trace buffer overflow when a kept trace is lost

dd-trace-core/src/main/java/datadog/trace/common/writer/RemoteWriter.java

@@ -94,7 +94,27 @@ public void write(final List<DDSpan> trace) {
             if (log.isDebugEnabled()) {
               log.debug("Dropped due to a buffer overflow: {}", trace);
             } else {
-              rlLog.warn("Dropped due to a buffer overflow: [{} spans]", trace.size());
+              rlLog.warn(
+                  "Dropped a kept trace due to a buffer overflow: [{} spans]."
+                      + " Traces are being produced faster than they can be sent to the agent.",
+                  trace.size());
+            }
+            handleDroppedTrace(trace);
+            break;
+          case DROPPED_BUFFER_OVERFLOW_SAMPLED_OUT:
+            // Only sampled-out traces were lost, so this is not worth alarming the user over.
+            log.debug("Dropped a sampled-out trace due to a buffer overflow: {}", trace);
+            handleDroppedTrace(trace);
+            break;
+          case DROPPED_BUFFER_OVERFLOW_SINGLE_SPAN:
+            if (log.isDebugEnabled()) {
+              log.debug(
+                  "Dropped a single span sampling candidate due to a buffer overflow: {}", trace);
+            } else {
+              rlLog.warn(
+                  "Dropped a single span sampling candidate due to a buffer overflow: [{} spans]."
+                      + " Traces are being produced faster than they can be sent to the agent.",
+                  trace.size());
             }
             handleDroppedTrace(trace);
             break;

dd-trace-core/src/main/java/datadog/trace/common/writer/ddagent/Prioritization.java

@@ -85,18 +85,22 @@ private EnsureTraceStrategy(
     @Override
     public <T extends CoreSpan<T>> PublishResult publish(
         T root, int priority, final List<T> trace) {
+      if (root.isForceKeep()) {
+        blockingOffer(primary, trace);
+        return PublishResult.ENQUEUED_FOR_SERIALIZATION;
+      }
       switch (priority) {
         case SAMPLER_DROP:
         case USER_DROP:
           if (spanSampling != null) {
             // send dropped traces for single span sampling
             return spanSampling.offer(trace)
                 ? PublishResult.ENQUEUED_FOR_SINGLE_SPAN_SAMPLING
-                : PublishResult.DROPPED_BUFFER_OVERFLOW;
+                : PublishResult.DROPPED_BUFFER_OVERFLOW_SINGLE_SPAN;
           }
           return secondary.offer(trace)
               ? PublishResult.ENQUEUED_FOR_SERIALIZATION
-              : PublishResult.DROPPED_BUFFER_OVERFLOW;
+              : PublishResult.DROPPED_BUFFER_OVERFLOW_SAMPLED_OUT;
         default:
           blockingOffer(primary, trace);
           return PublishResult.ENQUEUED_FOR_SERIALIZATION;
@@ -135,14 +139,14 @@ public <T extends CoreSpan<T>> PublishResult publish(T root, int priority, List<
             // send dropped traces for single span sampling
             return spanSampling.offer(trace)
                 ? PublishResult.ENQUEUED_FOR_SINGLE_SPAN_SAMPLING
-                : PublishResult.DROPPED_BUFFER_OVERFLOW;
+                : PublishResult.DROPPED_BUFFER_OVERFLOW_SINGLE_SPAN;
           }
           if (droppingPolicy.active()) {
             return PublishResult.DROPPED_BY_POLICY;
           }
           return secondary.offer(trace)
               ? PublishResult.ENQUEUED_FOR_SERIALIZATION
-              : PublishResult.DROPPED_BUFFER_OVERFLOW;
+              : PublishResult.DROPPED_BUFFER_OVERFLOW_SAMPLED_OUT;
         default:
           return primary.offer(trace)
               ? PublishResult.ENQUEUED_FOR_SERIALIZATION

dd-trace-core/src/main/java/datadog/trace/common/writer/ddagent/PrioritizationStrategy.java

@@ -10,7 +10,25 @@ enum PublishResult {
     ENQUEUED_FOR_SERIALIZATION,
     ENQUEUED_FOR_SINGLE_SPAN_SAMPLING,
     DROPPED_BY_POLICY,
-    DROPPED_BUFFER_OVERFLOW
+    /**
+     * A trace that sampling decided to keep could not be enqueued because the queue was full. The
+     * trace is lost and will be missing from the UI.
+     */
+    DROPPED_BUFFER_OVERFLOW,
+    /**
+     * A trace that sampling already decided to drop could not be enqueued because the queue was
+     * full. No kept trace is lost. Such traces are only enqueued at all when the agent computes
+     * trace stats itself, so the sole consequence is a small loss of accuracy in agent-computed
+     * stats; with client-side stats enabled they are discarded before reaching a queue.
+     */
+    DROPPED_BUFFER_OVERFLOW_SAMPLED_OUT,
+    /**
+     * A trace that sampling decided to drop could not be offered to the single span sampling queue
+     * because that queue was full. Unlike {@link #DROPPED_BUFFER_OVERFLOW_SAMPLED_OUT}, this is not
+     * a stats-only loss: the trace's spans were candidates to be individually kept by single span
+     * sampling, so losing them can mean losing spans that would otherwise have reached the UI.
+     */
+    DROPPED_BUFFER_OVERFLOW_SINGLE_SPAN
   }
 
   <T extends CoreSpan<T>> PublishResult publish(T root, int priority, List<T> trace);

dd-trace-core/src/test/java/datadog/trace/common/writer/DDAgentWriterTest.java

@@ -147,9 +147,10 @@ void testWriterWritePublishForSingleSpanSampling() {
   }
 
   @TableTest({
-    "scenario          | publishResult          ",
-    "buffer overflow   | DROPPED_BUFFER_OVERFLOW",
-    "dropped by policy | DROPPED_BY_POLICY      "
+    "scenario                      | publishResult                      ",
+    "buffer overflow               | DROPPED_BUFFER_OVERFLOW            ",
+    "buffer overflow (sampled out) | DROPPED_BUFFER_OVERFLOW_SAMPLED_OUT",
+    "dropped by policy             | DROPPED_BY_POLICY                  "
   })
   void testWriterWritePublishFails(PublishResult publishResult) {
     List<DDSpan> trace =
@@ -191,9 +192,10 @@ void testWriterWriteClosed() {
   }
 
   @TableTest({
-    "scenario          | publishResult          ",
-    "dropped by policy | DROPPED_BY_POLICY      ",
-    "buffer overflow   | DROPPED_BUFFER_OVERFLOW"
+    "scenario                      | publishResult                      ",
+    "dropped by policy             | DROPPED_BY_POLICY                  ",
+    "buffer overflow               | DROPPED_BUFFER_OVERFLOW            ",
+    "buffer overflow (sampled out) | DROPPED_BUFFER_OVERFLOW_SAMPLED_OUT"
   })
   void testDroppedTraceIsCounted(PublishResult publishResult) {
     // setup - use local mocks to avoid interference with instance-level mocks

dd-trace-core/src/test/java/datadog/trace/common/writer/DDIntakeWriterTest.java

@@ -149,9 +149,10 @@ void testWriterWritePublishForSingleSpanSampling() {
   }
 
   @TableTest({
-    "scenario          | publishResult          ",
-    "buffer overflow   | DROPPED_BUFFER_OVERFLOW",
-    "dropped by policy | DROPPED_BY_POLICY      "
+    "scenario                      | publishResult                      ",
+    "buffer overflow               | DROPPED_BUFFER_OVERFLOW            ",
+    "buffer overflow (sampled out) | DROPPED_BUFFER_OVERFLOW_SAMPLED_OUT",
+    "dropped by policy             | DROPPED_BY_POLICY                  "
   })
   void testWriterWritePublishFails(PublishResult publishResult) {
     List<DDSpan> trace =
@@ -196,9 +197,10 @@ void testWriterWriteClosed() {
   }
 
   @TableTest({
-    "scenario          | publishResult          ",
-    "dropped by policy | DROPPED_BY_POLICY      ",
-    "buffer overflow   | DROPPED_BUFFER_OVERFLOW"
+    "scenario                      | publishResult                      ",
+    "dropped by policy             | DROPPED_BY_POLICY                  ",
+    "buffer overflow               | DROPPED_BUFFER_OVERFLOW            ",
+    "buffer overflow (sampled out) | DROPPED_BUFFER_OVERFLOW_SAMPLED_OUT"
   })
   void testDroppedTraceIsCounted(PublishResult publishResult) {
     // setup - use local mocks