From fcb6357586dc16c347a132f2d35e99aa2cb5c4d8 Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Thu, 24 Sep 2026 17:06:50 -0400 Subject: [PATCH] Fix IBM JVM hang dump collection --- .../plugin/dump/DumpHangedTestPlugin.kt | 75 ++++++---- .../dump/DumpHangedTestIntegrationTest.kt | 133 +++++++++++++++++- docs/how_to_dump_hanged_test.md | 3 +- 3 files changed, 181 insertions(+), 30 deletions(-) diff --git a/buildSrc/src/main/kotlin/datadog/gradle/plugin/dump/DumpHangedTestPlugin.kt b/buildSrc/src/main/kotlin/datadog/gradle/plugin/dump/DumpHangedTestPlugin.kt index 00d1e7ef0e9..e63086d5a89 100644 --- a/buildSrc/src/main/kotlin/datadog/gradle/plugin/dump/DumpHangedTestPlugin.kt +++ b/buildSrc/src/main/kotlin/datadog/gradle/plugin/dump/DumpHangedTestPlugin.kt @@ -10,6 +10,7 @@ import org.gradle.api.provider.Provider import org.gradle.api.services.BuildService import org.gradle.api.services.BuildServiceParameters import org.gradle.api.tasks.testing.Test +import org.gradle.jvm.toolchain.JvmVendorSpec import org.gradle.kotlin.dsl.extra import org.gradle.kotlin.dsl.withType import java.io.File @@ -67,14 +68,27 @@ class DumpHangedTestPlugin : Plugin { project.allprojects { pluginManager.withPlugin("java") { tasks.withType().configureEach { - doFirst { schedule(this, scheduler, props) } + doFirst { + // Single source of truth for "is this an IBM JVM", shared with `collectThreadDump`. + val ibmJvm = isIbmJvmVendor(javaLauncher.get().metadata.vendor) + if (ibmJvm) { + // Where IBM/OpenJ9 writes the javacore on SIGQUIT; read at JVM startup, inherited by children. + environment("IBM_JAVACOREDIR", dumpDirectory(this).absolutePath) + } + schedule(this, ibmJvm, scheduler, props) + } doLast { cleanup(this) } } } } } - private fun schedule(t: Task, scheduler: Provider, props: DumpHangedTestProperties) { + private fun schedule( + t: Task, + ibmJvm: Boolean, + scheduler: Provider, + props: DumpHangedTestProperties + ) { val taskName = t.path if (t.extra.has(DUMP_FUTURE_KEY)) { @@ -93,31 +107,15 @@ class DumpHangedTestPlugin : Plugin { val future = scheduler.get().schedule({ t.logger.quiet("Taking dumps after ${delay.seconds} seconds delay for $taskName") - takeDump(t) + takeDump(t, ibmJvm) }, delay) t.extra.set(DUMP_FUTURE_KEY, future) } - private fun takeDump(t: Task) { + private fun takeDump(t: Task, ibmJvm: Boolean) { try { - // Use Gradle's build dir and adjust for CI artifacts collection if needed. - val dumpsDir: File = t.project.layout.buildDirectory - .dir("dumps") - .map { dir -> - if (t.project.providers.environmentVariable("CI").isPresent) { - // Move reports into the folder collected by the collect_reports.sh script. - File( - dir.asFile.absolutePath.replace( - "dd-trace-java/dd-java-agent", - "dd-trace-java/workspace/dd-java-agent" - ) - ) - } else { - dir.asFile - } - } - .get() + val dumpsDir = dumpDirectory(t) if (!dumpsDir.isDirectory && !dumpsDir.mkdirs()) { throw IOException("Could not create dump directory $dumpsDir") @@ -139,12 +137,12 @@ class DumpHangedTestPlugin : Plugin { } // Preserve all thread stacks before a slow heap dump can consume the remaining time. - processes.forEach { process -> collectThreadDump(t, dumpsDir, process) } + processes.forEach { process -> collectThreadDump(t, ibmJvm, dumpsDir, process) } // Just in case collect all thread dumps by using special PID `0`. val allThreadsFile = file(dumpsDir, "all-thread-dumps") runCmd(t.logger, t.path, Redirect.to(allThreadsFile), "jcmd", "0", "Thread.print", "-l") - processes.forEach { process -> collectHeapDump(t, dumpsDir, process) } + processes.forEach { process -> collectHeapDump(t, ibmJvm, dumpsDir, process) } t.logger.quiet("Finished dump collection for ${t.path}; output directory: $dumpsDir") } catch (e: InterruptedException) { Thread.currentThread().interrupt() @@ -157,6 +155,23 @@ class DumpHangedTestPlugin : Plugin { private fun file(baseDir: File, name: String, ext: String = "log") = File(baseDir, "$name-${System.currentTimeMillis()}.$ext") + private fun dumpDirectory(t: Task): File = t.project.layout.buildDirectory + .dir("dumps") + .map { dir -> + if (t.project.providers.environmentVariable("CI").isPresent) { + // Move reports into the folder collected by the collect_reports.sh script. + File( + dir.asFile.absolutePath.replace( + "dd-trace-java/dd-java-agent", + "dd-trace-java/workspace/dd-java-agent" + ) + ) + } else { + dir.asFile + } + } + .get() + private fun cleanup(t: Task) { val future = t.extra .takeIf { it.has(DUMP_FUTURE_KEY) } @@ -201,14 +216,14 @@ class DumpHangedTestPlugin : Plugin { private fun collectThreadDump( t: Task, + ibmJvm: Boolean, baseDir: File, process: ProcessHandle ) { val pid = process.pid().toString() - if (process.info().command().getOrElse { "" }.contains("/ibm8")) { - // On IBM JDK thread dump can be collected by signaling process with `kill -3`. - // It will be writen into `/tmp/javacore.YYYYMMDD.HHMMSS.PID.SEQ.txt + if (ibmJvm) { + // On IBM JDK `kill -3` writes a javacore into the IBM_JAVACOREDIR set when the task started. runCmd(t.logger, t.path, Redirect.INHERIT, "kill", "-3", pid) } else { // Collect thread dump by pid. @@ -217,8 +232,9 @@ class DumpHangedTestPlugin : Plugin { } } - private fun collectHeapDump(t: Task, baseDir: File, process: ProcessHandle) { - if (process.info().command().getOrElse { "" }.contains("/ibm8")) { + private fun collectHeapDump(t: Task, ibmJvm: Boolean, baseDir: File, process: ProcessHandle) { + if (ibmJvm) { + // `jcmd GC.heap_dump` is a HotSpot-only diagnostic command. return } val pid = process.pid().toString() @@ -226,3 +242,6 @@ class DumpHangedTestPlugin : Plugin { runCmd(t.logger, t.path, Redirect.INHERIT, "jcmd", pid, "GC.heap_dump", heapDumpPath) } } + +/** Whether [vendor] is an IBM JVM (Semeru included), as a toolchain vendor or raw `java.vendor`. */ +internal fun isIbmJvmVendor(vendor: String): Boolean = JvmVendorSpec.IBM.matches(vendor) diff --git a/buildSrc/src/test/kotlin/datadog/gradle/plugin/dump/DumpHangedTestIntegrationTest.kt b/buildSrc/src/test/kotlin/datadog/gradle/plugin/dump/DumpHangedTestIntegrationTest.kt index 6e33a27b007..25a7c4ff4f4 100644 --- a/buildSrc/src/test/kotlin/datadog/gradle/plugin/dump/DumpHangedTestIntegrationTest.kt +++ b/buildSrc/src/test/kotlin/datadog/gradle/plugin/dump/DumpHangedTestIntegrationTest.kt @@ -1,15 +1,23 @@ package datadog.gradle.plugin.dump import datadog.gradle.plugin.GradleFixture +import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertFalse import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Assumptions.assumeTrue import org.junit.jupiter.api.Test import org.junit.jupiter.api.assertNotNull import org.junit.jupiter.api.condition.EnabledOnOs import org.junit.jupiter.api.condition.OS import java.io.File +import java.util.Properties class DumpHangedTestIntegrationTest : GradleFixture() { + companion object { + /** A `jcmd Thread.print`, excluding the all-JVM `jcmd 0` sweep. */ + private val PER_PROCESS_THREAD_PRINT = Regex("""jcmd [1-9]\d* Thread\.print""") + } + @Test fun `should not take dumps`() { val output = runGradleTest(testSleepMillis = 1000) @@ -22,6 +30,48 @@ class DumpHangedTestIntegrationTest : GradleFixture() { assertFalse(buildFile("dumps").exists()) // Assert no dumps created. } + @Test + fun `should recognize IBM JVM vendor`() { + // `JavaInstallationMetadata.getVendor()` reports the display name, `java.vendor` the raw one. + assertTrue(isIbmJvmVendor("IBM")) + assertTrue(isIbmJvmVendor("IBM Corporation")) + assertTrue(isIbmJvmVendor("ibm corporation")) + assertFalse(isIbmJvmVendor("Eclipse Adoptium")) + assertFalse(isIbmJvmVendor("Oracle Corporation")) + } + + @Test + fun `should not configure IBM javacore directory on non-IBM JVM`() { + assumeTrue(!isIbmJvmVendor(System.getProperty("java.vendor")), "needs a non-IBM JVM") + + val output = runGradleTest(testSleepMillis = 1000, assertNoIbmJavaCoreDir = true) + + // The generated assertion only counts if the build ran it and it passed. + assertTrue(output.any { it.startsWith("BUILD SUCCESSFUL") }, output.joinToString("\n")) + assertEquals(listOf("ibmJavaCoreDirIsNotSet", "test"), testCaseNames().sorted()) + } + + @Test + @EnabledOnOs(OS.LINUX, OS.MAC) + fun `should collect javacore dumps on IBM JVM`() { + val ibmJdk = findIbmJdk() + assumeTrue(ibmJdk != null, "needs a locally installed IBM JVM") + + val output = runGradleTest(testSleepMillis = 25_000, ibmJdk = ibmJdk) + + // `kill -3` replaces the per-process `jcmd Thread.print`, and no heap dump is attempted. + assertTrue(output.any { it.startsWith("Starting dump command for :test:") && it.contains("kill -3") }) + assertFalse(output.any { PER_PROCESS_THREAD_PRINT.containsMatchIn(it) }) + assertFalse(output.any { it.contains("GC.heap_dump") }) + + // Landing here proves IBM_JAVACOREDIR reached the forked JVM; the default is its working dir. + val dumps = buildFile("dumps").listFiles().orEmpty() + assertTrue( + dumps.any { it.name.startsWith("javacore.") && it.name.endsWith(".txt") }, + "no javacore in ${dumps.map { it.name }}" + ) + } + @Test fun `should take dumps`() { val output = runGradleTest(testSleepMillis = 25_0000) @@ -96,18 +146,56 @@ class DumpHangedTestIntegrationTest : GradleFixture() { assertTrue(lastThreadDump >= 0 && firstHeapDump > lastThreadDump) } + /** Runs a Gradle build of a single-test project under the dump plugin. */ private fun runGradleTest( testSleepMillis: Long, env: Map = emptyMap(), timeoutSeconds: Long = 20, - dumpOffset: Long? = 5 + dumpOffset: Long? = 5, + ibmJdk: IbmJdk? = null, + assertNoIbmJavaCoreDir: Boolean = false ): List { + val javaCoreDirImport = if (assertNoIbmJavaCoreDir) { + "import static org.junit.jupiter.api.Assertions.assertNull;" + } else { + "" + } + val javaCoreDirTest = if (assertNoIbmJavaCoreDir) { + """ + @Test + public void ibmJavaCoreDirIsNotSet() { + assertNull(System.getenv("IBM_JAVACOREDIR")); + } + """ + } else { + "" + } + + if (ibmJdk != null) { + writeGradleProperties( + "org.gradle.java.installations.paths=${ibmJdk.home.absolutePath}", + append = true + ) + } + val ibmToolchain = ibmJdk?.let { + """ + java { + toolchain { + languageVersion.set(JavaLanguageVersion.of(${it.majorVersion})) + vendor.set(JvmVendorSpec.IBM) + } + } + """ + }.orEmpty() + writeSettings("""rootProject.name = "test-project"""") writeRootProject( """ import java.time.Duration import org.gradle.api.tasks.testing.Test + import org.gradle.jvm.toolchain.JavaLanguageVersion + import org.gradle.jvm.toolchain.JvmVendorSpec plugins { id("java") @@ -116,6 +204,8 @@ class DumpHangedTestIntegrationTest : GradleFixture() { group = "datadog.dump.test" + $ibmToolchain + repositories { mavenCentral() } @@ -141,9 +231,12 @@ class DumpHangedTestIntegrationTest : GradleFixture() { writeJavaSource( "SimpleTest", """ + $javaCoreDirImport import org.junit.jupiter.api.Test; public class SimpleTest { + $javaCoreDirTest + @Test public void test() throws InterruptedException { Thread.sleep($testSleepMillis); @@ -155,4 +248,42 @@ class DumpHangedTestIntegrationTest : GradleFixture() { return run("test", env = env, forwardOutput = true).output.lines() } + + /** Names of the test cases the generated project reported, from its JUnit XML report. */ + private fun testCaseNames(): List { + val report = buildFile("test-results/test/TEST-SimpleTest.xml") + assertTrue(report.isFile, "missing test report $report") + val testCases = parseXml(report).getElementsByTagName("testcase") + return (0 until testCases.length) + .map { testCases.item(it).attributes.getNamedItem("name").nodeValue.removeSuffix("()") } + } + + /** A locally installed IBM JVM, as resolved from its `release` file. */ + private data class IbmJdk(val home: File, val majorVersion: Int) + + /** Newest installed IBM JVM, from the CI `JAVA__HOME` vars or the platform JVM dirs. */ + private fun findIbmJdk(): IbmJdk? { + val candidates = mutableListOf() + System.getenv() + .filterKeys { it.matches(Regex("JAVA_(IBM|SEMERU)\\d+_HOME")) } + .values.mapTo(candidates, ::File) + for (root in listOf("/Library/Java/JavaVirtualMachines", "/usr/lib/jvm")) { + for (jvm in File(root).listFiles().orEmpty()) { + candidates += File(jvm, "Contents/Home") // macOS bundle layout + candidates += jvm + } + } + return candidates.mapNotNull { it.toIbmJdk() }.maxByOrNull { it.majorVersion } + } + + private fun File.toIbmJdk(): IbmJdk? { + val release = File(this, "release").takeIf { it.isFile } ?: return null + val properties = Properties().apply { release.inputStream().use { load(it) } } + // Values in `release` are quoted, e.g. IMPLEMENTOR="IBM Corporation". + fun property(name: String) = properties.getProperty(name).orEmpty().trim('"') + + if (!isIbmJvmVendor(property("IMPLEMENTOR"))) return null + val major = property("JAVA_VERSION").removePrefix("1.").substringBefore('.').toIntOrNull() + return major?.let { IbmJdk(this, it) } + } } diff --git a/docs/how_to_dump_hanged_test.md b/docs/how_to_dump_hanged_test.md index cad4071d67f..ba10078b549 100644 --- a/docs/how_to_dump_hanged_test.md +++ b/docs/how_to_dump_hanged_test.md @@ -100,7 +100,8 @@ Quick verification checklist: ### IBM JDK (javacore thread dumps only): - In this case, dumps are produced via `kill -3` and written as `javacore` text files (thread dumps). -- Collect root-level javacore artifacts with the path pattern `reports/javacore.YYYYMMDD.HHMMSS.PID.SEQ.txt`. +- Open the report folder of the failed module/test task, same as for HotSpot. +- You should see files such as `javacore.YYYYMMDD.HHMMSS.PID.SEQ.txt`. ![Javacores](how_to_dump_hanged_test/javacores.png) ## Step 5: Run the investigation