diff --git a/buildSrc/src/main/kotlin/datadog/gradle/plugin/dump/DumpHangedTestPlugin.kt b/buildSrc/src/main/kotlin/datadog/gradle/plugin/dump/DumpHangedTestPlugin.kt index 95a2ee21411..00d1e7ef0e9 100644 --- a/buildSrc/src/main/kotlin/datadog/gradle/plugin/dump/DumpHangedTestPlugin.kt +++ b/buildSrc/src/main/kotlin/datadog/gradle/plugin/dump/DumpHangedTestPlugin.kt @@ -35,7 +35,7 @@ class DumpHangedTestPlugin : Plugin { /** Plugin properties */ abstract class DumpHangedTestProperties @Inject constructor(objects: ObjectFactory) { // Time offset (in seconds) before a test reaches its timeout at which dumps should be started. - // Defaults to 60 seconds. + // Defaults to 180 seconds. val dumpOffset: Property = objects.property(Long::class.java) } @@ -82,7 +82,7 @@ class DumpHangedTestPlugin : Plugin { return } - val dumpOffset = props.dumpOffset.getOrElse(60) + val dumpOffset = props.dumpOffset.getOrElse(180) val delay = t.timeout.map { it.minusSeconds(dumpOffset) }.orNull if (delay == null || delay.seconds < 0) { @@ -123,25 +123,28 @@ class DumpHangedTestPlugin : Plugin { throw IOException("Could not create dump directory $dumpsDir") } - var executorCount = 0 + val processes = mutableListOf() ProcessHandle.current().children().use { children -> children.filter { it.info().commandLine().getOrElse { "" }.contains("Gradle Test Executor") } .forEach { process -> - executorCount++ - collectDump(t, dumpsDir, process) + processes.add(process) process.children().use { descendants -> - descendants.forEach { child -> collectDump(t, dumpsDir, child) } + descendants.forEach { child -> processes.add(child) } } } } - if (executorCount == 0) { + if (processes.isEmpty()) { t.logger.warn("No Gradle test executors found for ${t.path}; attempting all-JVM thread dumps") } + // Preserve all thread stacks before a slow heap dump can consume the remaining time. + processes.forEach { process -> collectThreadDump(t, dumpsDir, process) } + // Just in case collect all thread dumps by using special PID `0`. val allThreadsFile = file(dumpsDir, "all-thread-dumps") runCmd(t.logger, t.path, Redirect.to(allThreadsFile), "jcmd", "0", "Thread.print", "-l") + processes.forEach { process -> collectHeapDump(t, dumpsDir, process) } t.logger.quiet("Finished dump collection for ${t.path}; output directory: $dumpsDir") } catch (e: InterruptedException) { Thread.currentThread().interrupt() @@ -196,7 +199,7 @@ class DumpHangedTestPlugin : Plugin { } } - private fun collectDump( + private fun collectThreadDump( t: Task, baseDir: File, process: ProcessHandle @@ -208,13 +211,18 @@ class DumpHangedTestPlugin : Plugin { // It will be writen into `/tmp/javacore.YYYYMMDD.HHMMSS.PID.SEQ.txt runCmd(t.logger, t.path, Redirect.INHERIT, "kill", "-3", pid) } else { - // Collect heap dump by pid. - val heapDumpPath = file(baseDir, "$pid-heap-dump", "hprof").absolutePath - runCmd(t.logger, t.path, Redirect.INHERIT, "jcmd", pid, "GC.heap_dump", heapDumpPath) - // Collect thread dump by pid. val threadDumpFile = file(baseDir, "$pid-thread-dump", "log") runCmd(t.logger, t.path, Redirect.to(threadDumpFile), "jcmd", pid, "Thread.print", "-l") } } + + private fun collectHeapDump(t: Task, baseDir: File, process: ProcessHandle) { + if (process.info().command().getOrElse { "" }.contains("/ibm8")) { + return + } + val pid = process.pid().toString() + val heapDumpPath = file(baseDir, "$pid-heap-dump", "hprof").absolutePath + runCmd(t.logger, t.path, Redirect.INHERIT, "jcmd", pid, "GC.heap_dump", heapDumpPath) + } } diff --git a/buildSrc/src/test/kotlin/datadog/gradle/plugin/dump/DumpHangedTestIntegrationTest.kt b/buildSrc/src/test/kotlin/datadog/gradle/plugin/dump/DumpHangedTestIntegrationTest.kt index 20451ddeaeb..6e33a27b007 100644 --- a/buildSrc/src/test/kotlin/datadog/gradle/plugin/dump/DumpHangedTestIntegrationTest.kt +++ b/buildSrc/src/test/kotlin/datadog/gradle/plugin/dump/DumpHangedTestIntegrationTest.kt @@ -42,6 +42,14 @@ class DumpHangedTestIntegrationTest : GradleFixture() { assertNotNull(dumpFiles.find { it.startsWith("all-thread-dumps") }) } + @Test + fun `should start dumps three minutes before timeout by default`() { + val output = runGradleTest(testSleepMillis = 10_000, timeoutSeconds = 185, dumpOffset = null) + + assertTrue(output.contains("Taking dumps after 5 seconds delay for :test")) + assertFalse(output.any { it.contains("has exceeded its configured timeout") }) + } + @Test fun `should report directory failures with stack trace`() { writeFile("build/dumps", "This file prevents creating the dump directory") @@ -55,7 +63,7 @@ class DumpHangedTestIntegrationTest : GradleFixture() { @Test @EnabledOnOs(OS.LINUX, OS.MAC) - fun `should attempt thread dumps after heap dump failure`() { + fun `should collect all thread dumps before attempting heap dumps`() { val jcmd = writeFile( "bin/jcmd", """ @@ -83,9 +91,17 @@ class DumpHangedTestIntegrationTest : GradleFixture() { assertTrue(dumps.any { it.name.contains("-thread-dump-") && it.readText().startsWith("Synthetic thread dump") }) assertTrue(dumps.any { it.name.startsWith("all-thread-dumps-") && it.readText().contains("PID 0") }) assertTrue(output.any { it.startsWith("Finished dump collection for :test;") }) + val firstHeapDump = output.indexOfFirst { it.startsWith("Starting dump command") && it.contains("GC.heap_dump") } + val lastThreadDump = output.indexOfLast { it.startsWith("Completed dump command") && it.contains("Thread.print") } + assertTrue(lastThreadDump >= 0 && firstHeapDump > lastThreadDump) } - private fun runGradleTest(testSleepMillis: Long, env: Map = emptyMap()): List { + private fun runGradleTest( + testSleepMillis: Long, + env: Map = emptyMap(), + timeoutSeconds: Long = 20, + dumpOffset: Long? = 5 + ): List { writeSettings("""rootProject.name = "test-project"""") writeRootProject( @@ -111,13 +127,11 @@ class DumpHangedTestIntegrationTest : GradleFixture() { } dumpHangedTest { - // Set the dump offset for 5 seconds to trigger taking dumps after 15 seconds. - dumpOffset.set(5) + ${dumpOffset?.let { "dumpOffset.set($it)" } ?: ""} } tasks.withType().configureEach { - // Set test timeout after 20 seconds. - timeout.set(Duration.ofSeconds(20)) + timeout.set(Duration.ofSeconds($timeoutSeconds)) useJUnitPlatform() }