events, int maxStackDepth) {
diff --git a/src/main/java/com/intellij/rt/debugger/agent/CollectionBreakpointInstrumentor.java b/src/main/java/com/intellij/rt/debugger/agent/CollectionBreakpointInstrumentor.java
index ea220c1..c0c3656 100644
--- a/src/main/java/com/intellij/rt/debugger/agent/CollectionBreakpointInstrumentor.java
+++ b/src/main/java/com/intellij/rt/debugger/agent/CollectionBreakpointInstrumentor.java
@@ -18,7 +18,7 @@
import static com.intellij.rt.debugger.agent.CaptureAgent.getInternalClsName;
-@SuppressWarnings({"UseOfSystemOutOrSystemErr", "CallToPrintStackTrace", "rawtypes"})
+@SuppressWarnings("rawtypes")
public class CollectionBreakpointInstrumentor {
private static final String OBJECT_TYPE = "Ljava/lang/Object;";
private static final String STRING_TYPE = "Ljava/lang/String;";
@@ -139,13 +139,9 @@ private static void processFailedToInstrumentError(String className, Exception e
private static void writeDebugInfo(String className, byte[] bytes) {
try {
System.out.println("instrumented: " + className);
- FileOutputStream stream = new FileOutputStream("instrumented_" + className.replaceAll("/", "_") + ".class");
- try {
- stream.write(bytes);
- }
- finally {
- stream.close();
- }
+ try (FileOutputStream stream = new FileOutputStream("instrumented_" + className.replace("/", "_") + ".class")) {
+ stream.write(bytes);
+ }
}
catch (IOException e) {
e.printStackTrace();
diff --git a/src/main/java/com/intellij/rt/debugger/agent/DebuggerAgent.java b/src/main/java/com/intellij/rt/debugger/agent/DebuggerAgent.java
index fa1a7f8..faf993d 100644
--- a/src/main/java/com/intellij/rt/debugger/agent/DebuggerAgent.java
+++ b/src/main/java/com/intellij/rt/debugger/agent/DebuggerAgent.java
@@ -14,7 +14,6 @@
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ThreadFactory;
-@SuppressWarnings("UseOfSystemOutOrSystemErr")
public class DebuggerAgent {
private static final String KEEP_SUFFIX = "[keep]";
diff --git a/src/main/java/com/intellij/rt/debugger/agent/LogCaptureStorage.java b/src/main/java/com/intellij/rt/debugger/agent/LogCaptureStorage.java
index 581b436..b06bbae 100644
--- a/src/main/java/com/intellij/rt/debugger/agent/LogCaptureStorage.java
+++ b/src/main/java/com/intellij/rt/debugger/agent/LogCaptureStorage.java
@@ -10,6 +10,7 @@
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
+import java.util.zip.Deflater;
import java.util.zip.GZIPOutputStream;
public class LogCaptureStorage {
@@ -27,36 +28,57 @@ protected Boolean initialValue() {
static final String BATCHING_ENABLED_PROPERTY = "logCaptureBatchingEnabled";
static final String BATCHING_FLUSH_PERIOD_PROPERTY = "logCaptureBatchingFlushPeriod";
static final String BATCHING_BUFFER_SIZE_PROPERTY = "logCaptureBatchingBufferSize";
- private static final long DEFAULT_BUFFER_SIZE = calculateDefaultBufferSize();
- private static final int ESTIMATED_THROWABLE_BYTES = 3000;
+ static final String FORCE_BATCHING_BUFFER_SIZE_PROPERTY = "logCaptureForceBatchingBufferSize";
+ private static final int ESTIMATED_THROWABLE_BYTES = 3000; // See the comment at the top of ThrowableCapacityOverhead
private static boolean BATCHING_ENABLED;
private static long BUFFER_SIZE;
private static boolean STDOUT_CAPTURE_ENABLED;
- // It's used by the debugger.
+ // The debugger reads this value via evaluation to learn how many events have been created by the agent.
+ // The agent owns all writes.
static final AtomicLong EVENT_COUNTER = new AtomicLong();
+ /**
+ * The debugger writes the greatest event id that was received and decoded successfully via the {@link #packBatchedData()} call.
+ * The confirmation is done outside the method itself because it can be called multiple times (see javadoc).
+ * The confirmation is not guaranteed, it is used only as an optimization against repeated sending of the same events.
+ *
+ * The agent checks the flushed state based on this field and {@link #LAST_FLUSHED_EVENT_ID}.
+ */
+ static volatile long CONFIRMED_FLUSHED_EVENT_ID = -1;
+
// It contains raw events that are waiting to be packed.
// New ones could be added concurrently.
- // Raw or packed data can be flushed concurrently, leading to sending the same events multiple times.
- // It's ok and is handled by the debugger using IDs.
static final ConcurrentLinkedQueue EVENTS = new ConcurrentLinkedQueue<>();
static final AtomicLong EVENTS_PAYLOAD_BYTES = new AtomicLong();
+
+ // It contains packed batches that are waiting to be sent.
+ // New ones could be added concurrently.
static final ConcurrentLinkedQueue PACKED_BATCHES = new ConcurrentLinkedQueue<>();
static final AtomicLong PACKED_BATCHES_BYTES = new AtomicLong();
+ // Raw or packed data can be flushed concurrently, leading to sending the same events multiple times.
+ // It's ok and is handled by the debugger using IDs.
static final AtomicLong LAST_FLUSHED_EVENT_ID = new AtomicLong(-1);
static final AtomicLong LAST_PACKED_EVENT_ID = new AtomicLong(-1);
static final AtomicLong LAST_LOGGING_BREAKPOINT_EVENT_ID = new AtomicLong(-1);
- private interface MemoryFootprintEstimate {
- int memoryFootprintEstimate();
+ private abstract static class MemoryFootprintEstimate {
+ private final AtomicBoolean removed = new AtomicBoolean();
+
+ public abstract int memoryFootprintEstimate();
- boolean markRemoved();
+ /** Returns true if the event was removed for the first time. */
+ public boolean markRemoved() {
+ return removed.compareAndSet(false, true);
+ }
}
- static class Event implements MemoryFootprintEstimate {
+ /**
+ * A single event.
+ */
+ static class Event extends MemoryFootprintEstimate {
public static final byte STD_OUTPUT_TYPE = 0;
public static final byte LOGGING_BREAKPOINT_TYPE = 1;
@@ -65,7 +87,6 @@ static class Event implements MemoryFootprintEstimate {
public final byte[] payload;
public final Throwable throwable;
public final CaptureStorage.CapturedStack stack;
- private final AtomicBoolean removed = new AtomicBoolean();
public Event(long id, byte type, byte[] payload, Throwable throwable, CaptureStorage.CapturedStack stack) {
this.id = id;
@@ -81,15 +102,26 @@ public int memoryFootprintEstimate() {
}
@Override
- public boolean markRemoved() {
- return removed.compareAndSet(false, true);
+ public int hashCode() {
+ // This override is not necessary for correctness, but it is a bit faster than Object.hashCode.
+ return (int) (id ^ (id >>> 32));
+ }
+
+ @Override
+ public final boolean equals(Object o) {
+ if (!(o instanceof Event)) return false;
+
+ Event event = (Event) o;
+ return id == event.id;
}
}
- static class PackedBatch implements MemoryFootprintEstimate {
+ /**
+ * Multiple events, compressed and packed. Batch is prepared to be sent to the debugger.
+ */
+ static class PackedBatch extends MemoryFootprintEstimate {
public final byte[] data;
public final long lastEventId;
- private final AtomicBoolean removed = new AtomicBoolean();
public PackedBatch(byte[] data, long lastEventId) {
this.data = data;
@@ -100,11 +132,6 @@ public PackedBatch(byte[] data, long lastEventId) {
public int memoryFootprintEstimate() {
return data.length;
}
-
- @Override
- public boolean markRemoved() {
- return removed.compareAndSet(false, true);
- }
}
private static final FileDescriptor FD_OUT = FileDescriptor.out;
@@ -120,8 +147,7 @@ public static boolean init(Properties properties, boolean logCaptureEnabled) {
ENABLED = true;
STDOUT_CAPTURE_ENABLED = logCaptureEnabled;
BATCHING_ENABLED = Boolean.parseBoolean(properties.getProperty(BATCHING_ENABLED_PROPERTY, "true"));
- // Split in 2 halves for raw events and compressed batches.
- BUFFER_SIZE = getBufferSize(properties) / 2;
+ BUFFER_SIZE = getBufferSize(properties);
if (BATCHING_ENABLED && !batchingSchedulerStarted) {
batchingSchedulerStarted = true;
@@ -150,16 +176,17 @@ public void run() {
return true;
}
- static long calculateDefaultBufferSize() {
- long defaultMax = 5L * 1024L * 1024L; // 5 MB
- long runtimeMax = Runtime.getRuntime().maxMemory() / 100; // 1% of max heap
- return Math.min(defaultMax, runtimeMax);
- }
-
static long getBufferSize(Properties properties) {
- return Long.parseLong(properties.getProperty(
- BATCHING_BUFFER_SIZE_PROPERTY,
- String.valueOf(DEFAULT_BUFFER_SIZE)));
+ String forcedBufferSize = properties.getProperty(FORCE_BATCHING_BUFFER_SIZE_PROPERTY);
+ if (forcedBufferSize != null) {
+ return Long.parseLong(forcedBufferSize);
+ }
+ String stringMaxSize = properties.getProperty(BATCHING_BUFFER_SIZE_PROPERTY);
+ long maxBufferSize = stringMaxSize == null
+ ? 5 * 1024L * 1024L // 5MB
+ : Long.parseLong(stringMaxSize);
+ long runtimeMax = Runtime.getRuntime().maxMemory() / 100; // 1% of max heap
+ return Math.min(maxBufferSize, runtimeMax);
}
private static long createNextEventId(int eventType) {
@@ -234,6 +261,7 @@ private static void handleException(Throwable e) {
* before it appears on the debugger side. The clearing happens in the periodic flush cycle.
*/
static String packBatchedData() throws IOException {
+ clearConfirmedBatches();
packRawEvents();
List packedBatchesSnapshot = new ArrayList<>(PACKED_BATCHES);
if (packedBatchesSnapshot.isEmpty()) return null;
@@ -271,24 +299,52 @@ private static void enqueuePackedBatch(List events) throws IOException {
}
private static void flushBatchedData(boolean forceOutput) throws IOException {
- if (forceOutput || currentEventsEstimatedBytes() > BUFFER_SIZE) {
+ clearConfirmedBatches();
+ if (forceOutput) {
packRawEvents();
+ } else {
+ long eventsBytes = currentEventsEstimatedBytes();
+ long packedBytes = PACKED_BATCHES_BYTES.get();
+ if (eventsBytes + packedBytes <= BUFFER_SIZE) {
+ return;
+ }
+ packRawEvents();
+ packedBytes = PACKED_BATCHES_BYTES.get();
+ // We will zip too often if there is little space for raw events.
+ if (packedBytes <= BUFFER_SIZE * 9 / 10) {
+ return;
+ }
}
- if (!forceOutput && PACKED_BATCHES_BYTES.get() <= BUFFER_SIZE) return;
List packedBatchesSnapshot = new ArrayList<>(PACKED_BATCHES);
if (packedBatchesSnapshot.isEmpty()) return;
outputWritten(packBatches(packedBatchesSnapshot));
- long removedBytes = removeItems(PACKED_BATCHES, packedBatchesSnapshot);
+ markBatchesFlushed(packedBatchesSnapshot);
+ }
+
+ private static void markBatchesFlushed(Collection batches) {
+ long removedBytes = removeItems(PACKED_BATCHES, batches);
PACKED_BATCHES_BYTES.addAndGet(-removedBytes);
- setIfGreater(LAST_FLUSHED_EVENT_ID, findMaxPackedEventId(packedBatchesSnapshot));
+ setIfGreater(LAST_FLUSHED_EVENT_ID, findMaxPackedEventId(batches));
+ }
+
+ private static void clearConfirmedBatches() {
+ long confirmedId = CONFIRMED_FLUSHED_EVENT_ID;
+ if (confirmedId <= LAST_FLUSHED_EVENT_ID.get()) return;
+ Set alreadyConfirmed = new HashSet<>();
+ for (PackedBatch batch : PACKED_BATCHES) {
+ if (batch.lastEventId <= confirmedId) {
+ alreadyConfirmed.add(batch);
+ }
+ }
+ markBatchesFlushed(alreadyConfirmed);
}
private static byte[] packBytes(List events) throws IOException {
assert !events.isEmpty();
ByteArrayOutputStream bas = new ByteArrayOutputStream(); // no need to close it
- try (GZIPOutputStream gos = new GZIPOutputStream(bas);
+ try (GZIPOutputStream gos = new FastGzipOutputStream(bas);
DataOutputStream dos = new DataOutputStream(gos)) {
CapturedStackDeduplicator.StackDictionary stackDictionary =
CapturedStackDeduplicator.createStackDictionary(events, MAX_STACK_DEPTH);
@@ -316,6 +372,13 @@ private static byte[] packBytes(List events) throws IOException {
return bas.toByteArray();
}
+ private static class FastGzipOutputStream extends GZIPOutputStream {
+ FastGzipOutputStream(ByteArrayOutputStream out) throws IOException {
+ super(out);
+ def.setLevel(Deflater.BEST_SPEED);
+ }
+ }
+
private static byte[] packStack(List stackTrace) throws IOException {
ByteArrayOutputStream bas = new ByteArrayOutputStream(); // no need to close it
try (DataOutputStream dos = new DataOutputStream(bas)) {
@@ -387,7 +450,7 @@ private static long findMaxPackedEventId(Collection packedBatches)
}
private static long removeItems(ConcurrentLinkedQueue queue, Collection items) {
- Set itemsToRemove = new HashSet<>(items);
+ Set itemsToRemove = items instanceof Set ? (Set) items : new HashSet<>(items);
long removedBytes = 0;
for (Iterator queueIterator = queue.iterator(); queueIterator.hasNext(); ) {
diff --git a/src/main/java/com/intellij/rt/debugger/agent/ThrowableInterner.java b/src/main/java/com/intellij/rt/debugger/agent/ThrowableInterner.java
index 5ac9f33..3c4de4e 100644
--- a/src/main/java/com/intellij/rt/debugger/agent/ThrowableInterner.java
+++ b/src/main/java/com/intellij/rt/debugger/agent/ThrowableInterner.java
@@ -61,7 +61,7 @@ static Throwable intern(Throwable throwable, Object backtrace) {
// The raw VM backtrace is captured by new Throwable(), unlike StackTraceElement[] which is
// created lazily and relatively expensively by getStackTrace().
- BacktraceKey key = new BacktraceKey(throwable, backtrace);
+ BacktraceKey key = new BacktraceKey(backtrace);
Throwable interned = INTERNED_THROWABLES.putIfAbsent(key, throwable);
return interned == null ? throwable : interned;
}
@@ -79,20 +79,12 @@ static void disable(String message) {
}
private static final class BacktraceKey {
- private final Class myThrowableClass;
private final Object myBacktrace;
private final int myHashCode;
- private BacktraceKey(Throwable throwable, Object backtrace) {
- myThrowableClass = throwable.getClass();
+ private BacktraceKey(Object backtrace) {
myBacktrace = backtrace;
- myHashCode = computeHashCode(myThrowableClass, backtrace);
- }
-
- private static int computeHashCode(Class throwableClass, Object backtrace) {
- int result = throwableClass.hashCode();
- result = 31 * result + backtraceHashCode(backtrace);
- return result;
+ myHashCode = backtraceHashCode(backtrace);
}
@Override
@@ -102,7 +94,6 @@ public boolean equals(Object obj) {
BacktraceKey key = (BacktraceKey) obj;
return myHashCode == key.myHashCode &&
- myThrowableClass == key.myThrowableClass &&
backtracesEqual(myBacktrace, key.myBacktrace);
}
diff --git a/src/main/java/com/intellij/rt/debugger/agent/ThrowableTransformer.java b/src/main/java/com/intellij/rt/debugger/agent/ThrowableTransformer.java
index c651e64..41fbbaa 100644
--- a/src/main/java/com/intellij/rt/debugger/agent/ThrowableTransformer.java
+++ b/src/main/java/com/intellij/rt/debugger/agent/ThrowableTransformer.java
@@ -9,6 +9,15 @@
import java.lang.instrument.ClassFileTransformer;
import java.security.ProtectionDomain;
+/**
+ * Achieves two extra behaviors:
+ *
+ * - each created throwable is registered in {@link CaptureStorage#captureThrowable(Throwable)}
+ * to be able to access the async stack trace in {@code printStackTrace} methods via {@link CaptureStorage#getAsyncStackTrace(Throwable)};
+ * - the raw VM backtrace is extracted and passed to {@link ThrowableInterner}
+ * through {@link CaptureStorage#captureThrowableBacktrace(Object)}.
+ *
+ */
class ThrowableTransformer implements ClassFileTransformer {
static final String THROWABLE_NAME = CaptureAgent.getInternalClsName(Throwable.class);
@@ -26,12 +35,18 @@ public byte[] transform(ClassLoader loader,
return transformer.accept(new ClassVisitor(Opcodes.API_VERSION, transformer.writer) {
private String myBacktraceFieldName;
private String myBacktraceFieldDescriptor;
+ private boolean myMultipleBacktraceFields;
@Override
public FieldVisitor visitField(int access, String name, String descriptor, String signature, Object value) {
if (isBacktraceField(name, descriptor)) {
- myBacktraceFieldName = name;
- myBacktraceFieldDescriptor = descriptor;
+ if (myBacktraceFieldName == null) {
+ myBacktraceFieldName = name;
+ myBacktraceFieldDescriptor = descriptor;
+ } else {
+ myMultipleBacktraceFields = true;
+ ThrowableInterner.disable("Capture agent: cannot capture Throwable backtrace, ambiguous backtrace fields found");
+ }
}
return super.visitField(access, name, descriptor, signature, value);
}
@@ -41,19 +56,21 @@ public MethodVisitor visitMethod(final int access, String name, String descripto
MethodVisitor superMethodVisitor = super.visitMethod(access, name, descriptor, signature, exceptions);
switch (name) {
case "":
- // Insert CaptureStorage calls in the end of constructors.
+ // Insert extra calls at the end of constructors.
return new MethodVisitor(api, superMethodVisitor) {
@Override
public void visitInsn(int opcode) {
if (opcode == Opcodes.RETURN) {
- if (myBacktraceFieldName != null) {
+ if (myBacktraceFieldName != null && !myMultipleBacktraceFields) {
+ // Extract backtrace.
mv.visitVarInsn(Opcodes.ALOAD, 0);
mv.visitFieldInsn(Opcodes.GETFIELD, THROWABLE_NAME, myBacktraceFieldName, myBacktraceFieldDescriptor);
CaptureAgent.invokeStorageMethod(mv, "captureThrowableBacktrace");
}
else {
- ThrowableInterner.disable("Capture agent: cannot capture Throwable backtrace, no supported backtrace field was found");
+ ThrowableInterner.disable("Capture agent: cannot capture Throwable backtrace, no supported backtrace field was found before the constructor");
}
+ // Perform async stack trace capture.
mv.visitVarInsn(Opcodes.ALOAD, 0);
CaptureAgent.invokeStorageMethod(mv, "captureThrowable");
}
@@ -85,7 +102,6 @@ public void visitMethodInsn(int opcode, String owner, String name, String descri
}
catch (Exception e) {
System.out.println("Capture agent: failed to instrument " + className);
- //noinspection CallToPrintStackTrace
e.printStackTrace();
}
}
@@ -93,7 +109,10 @@ public void visitMethodInsn(int opcode, String owner, String name, String descri
}
private static boolean isBacktraceField(String name, String descriptor) {
- if (!"backtrace".equals(name) && !"walkback".equals(name)) return false;
- return descriptor.startsWith("L") || descriptor.startsWith("[");
+ // HotSpot: Object backtrace
+ // OpenJ9: Object walkback
+ // Moreover, we accept any reference type just to be ready for any variations.
+ return ("backtrace".equals(name) || "walkback".equals(name)) &&
+ (descriptor.startsWith("L") || descriptor.startsWith("["));
}
}
diff --git a/src/test/java/com/intellij/rt/debugger/agent/LogCaptureEncodingTest.java b/src/test/java/com/intellij/rt/debugger/agent/LogCaptureEncodingTest.java
index 699dac5..4cd5983 100644
--- a/src/test/java/com/intellij/rt/debugger/agent/LogCaptureEncodingTest.java
+++ b/src/test/java/com/intellij/rt/debugger/agent/LogCaptureEncodingTest.java
@@ -221,17 +221,6 @@ public void exceededRawEventBufferThresholdPacksEvents() throws Exception {
assertEquals("packed batches use the same buffer threshold and are sent immediately", 1, LogCaptureStorage.outputWrittenDumpForTests.size());
}
- @Test
- public void rawEventsUseHalfOfBufferSizeAsThreshold() throws Exception {
- properties.put(LogCaptureStorage.BATCHING_BUFFER_SIZE_PROPERTY, String.valueOf(2 * 3000));
- LogCaptureStorage.init(properties, true);
-
- capture(FileDescriptor.out, "a\n");
-
- assertEquals("raw event bytes are packed after exceeding half the buffer size", 0, LogCaptureStorage.EVENTS.size());
- assertEquals(0, LogCaptureStorage.EVENTS_PAYLOAD_BYTES.get());
- }
-
@Test
public void packBatchedDataPacksPendingRawEventsBeforeReturning() throws Exception {
properties.put(LogCaptureStorage.BATCHING_BUFFER_SIZE_PROPERTY, LARGE_BUFFER_SIZE);
diff --git a/src/test/java/com/intellij/rt/debugger/agent/ThrowableCapacityOverhead.java b/src/test/java/com/intellij/rt/debugger/agent/ThrowableCapacityOverhead.java
index 0a92767..f8b6198 100644
--- a/src/test/java/com/intellij/rt/debugger/agent/ThrowableCapacityOverhead.java
+++ b/src/test/java/com/intellij/rt/debugger/agent/ThrowableCapacityOverhead.java
@@ -143,7 +143,7 @@ private static void measureAfterBatchZipping(int stackDepth, int repeats) throws
Properties properties = new Properties();
properties.put(LogCaptureStorage.BATCHING_ENABLED_PROPERTY, "true");
properties.put(LogCaptureStorage.BATCHING_FLUSH_PERIOD_PROPERTY, "999999999");
- properties.put(LogCaptureStorage.BATCHING_BUFFER_SIZE_PROPERTY, String.valueOf(Long.MAX_VALUE));
+ properties.put(LogCaptureStorage.FORCE_BATCHING_BUFFER_SIZE_PROPERTY, String.valueOf(Long.MAX_VALUE));
LogCaptureStorage.init(properties, true);
recursiveCapture(stackDepth, repeats, "stdout message\n".getBytes(StandardCharsets.UTF_8));