From 96333c84eeb0a512628d228b1b77f3409fc55572 Mon Sep 17 00:00:00 2001 From: Brice Dutheil Date: Wed, 23 Sep 2026 23:31:11 +0200 Subject: [PATCH 01/15] feat: track test container images as Gradle inputs --- build-logic/settings.gradle.kts | 1 + build-logic/testcontainers/README.md | 59 ++++ build-logic/testcontainers/build.gradle.kts | 41 +++ .../testcontainers/ContainerImageArguments.kt | 22 ++ .../testcontainers/ImageResolver.kt | 101 ++++++ .../testcontainers/TestcontainersPlugin.kt | 97 ++++++ .../TestcontainersPluginTest.kt | 309 ++++++++++++++++++ .../datastax-cassandra-3.0/build.gradle | 3 + .../test/groovy/CassandraClientTest.groovy | 5 +- .../datastax-cassandra-3.8/build.gradle | 3 + .../test/groovy/CassandraClientTest.groovy | 5 +- .../datastax-cassandra-4.0/build.gradle | 3 + .../test/groovy/CassandraClientTest.groovy | 5 +- .../google-pubsub-1.116/build.gradle | 3 + .../src/test/groovy/PubSubTest.groovy | 4 +- .../instrumentation/jdbc/build.gradle | 11 + .../RemoteJDBCInstrumentationTest.groovy | 15 +- .../DataDogRegistryImageNameSubstitutor.java | 35 -- .../test/resources/testcontainers.properties | 1 - .../vertx-mysql-client-3.9/build.gradle | 3 + .../src/test/java/TestDatabases.java | 5 +- .../vertx-mysql-client-4.0/build.gradle | 3 + .../src/test/java/TestDatabases.java | 5 +- .../vertx-pg-client-4.0/build.gradle | 3 + .../src/test/java/TestDatabases.java | 5 +- dd-smoke-tests/websphere-jmx/build.gradle | 3 + .../smoketest/WebSphereJmxSmokeTest.groovy | 2 +- 27 files changed, 704 insertions(+), 48 deletions(-) create mode 100644 build-logic/testcontainers/README.md create mode 100644 build-logic/testcontainers/build.gradle.kts create mode 100644 build-logic/testcontainers/src/main/kotlin/datadog/buildlogic/testcontainers/ContainerImageArguments.kt create mode 100644 build-logic/testcontainers/src/main/kotlin/datadog/buildlogic/testcontainers/ImageResolver.kt create mode 100644 build-logic/testcontainers/src/main/kotlin/datadog/buildlogic/testcontainers/TestcontainersPlugin.kt create mode 100644 build-logic/testcontainers/src/test/kotlin/datadog/buildlogic/testcontainers/TestcontainersPluginTest.kt delete mode 100644 dd-java-agent/instrumentation/jdbc/src/test/java/test/DataDogRegistryImageNameSubstitutor.java delete mode 100644 dd-java-agent/instrumentation/jdbc/src/test/resources/testcontainers.properties diff --git a/build-logic/settings.gradle.kts b/build-logic/settings.gradle.kts index 6d1ceec50bb..006d14e9c9a 100644 --- a/build-logic/settings.gradle.kts +++ b/build-logic/settings.gradle.kts @@ -55,3 +55,4 @@ rootProject.name = "build-logic" include(":conventions") include(":smoke-test") +include(":testcontainers") diff --git a/build-logic/testcontainers/README.md b/build-logic/testcontainers/README.md new file mode 100644 index 00000000000..e879c975bcb --- /dev/null +++ b/build-logic/testcontainers/README.md @@ -0,0 +1,59 @@ +# Container images as test inputs + +Apply this plugin only in modules that use containers: + +```groovy +plugins { + id 'dd-trace-java.testcontainers' +} + +dependencies { + testContainerImage(image('cassandra:4', 'test.cassandra.image')) +} +``` + +Read the property when constructing the container. Keep Testcontainers' compatibility +declaration when the image can come from a mirror: + +```java +DockerImageName.parse(System.getProperty("test.cassandra.image")) + .asCompatibleSubstituteFor("cassandra") +``` + +Each source set gets a `ContainerImage` declaration method. Images follow +`implementation` configuration inheritance, so a `latestDepTest` suite extending +`testImplementation` inherits its images. The matching `Test` task, `forkedTest`, +and `ForkedTest` companions receive the properties. A separate suite can +declare its own images with, for example, `integrationTestContainerImage(...)`. +Run IDE tests through Gradle, or supply the named image properties explicitly. + +The plugin resolves each effective tag once per build, when Gradle snapshots test +inputs. An annotated JVM argument provider includes property names and immutable +`registry/repository@sha256:...` values in the test fingerprint, then passes those +same values to the JVM. Unchanged images allow `UP-TO-DATE`/`FROM-CACHE`; changed +images select a different cache entry. Configuration-cache reuse still refreshes +tags. Unrelated tasks and skipped tests perform no registry requests. Resolution +failure stops the test instead of trusting stale results. + +Jib reads registry manifests without pulling layers or requiring a Docker daemon. +Private registries use Docker's `config.json` and credential helpers, honoring +`DOCKER_CONFIG`. Remote registries require TLS; loopback registries also permit +local development certificates and HTTP. + +Docker Hub substitution honors `TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX`, then the +user's `.testcontainers.properties` and source-set `testcontainers.properties`. +Explicit registry names bypass this prefix, matching Testcontainers. Other mappings +belong in the declaration; JDBC's SQL Server declaration selects its CI mirror there. +Custom image substitutors and `*.container.image` overrides in these configuration +sources are rejected because they could replace the resolved digest at runtime. +Image substitution supplied by dependency JAR resources is not supported. + +Only declared images are tracked. This migration covers Cassandra, Pub/Sub, +JDBC, Vert.x MySQL/PostgreSQL and WebSphere fixtures. Other fixtures and implicit +Testcontainers helpers such as Alpine/Ryuk require separate adoption. + +Run the hermetic registry and Gradle cache tests with: + +```shell +build-brief ./gradlew -p build-logic :testcontainers:test :testcontainers:validatePlugins +``` diff --git a/build-logic/testcontainers/build.gradle.kts b/build-logic/testcontainers/build.gradle.kts new file mode 100644 index 00000000000..dbf6d83dcea --- /dev/null +++ b/build-logic/testcontainers/build.gradle.kts @@ -0,0 +1,41 @@ +plugins { + `java-gradle-plugin` + `kotlin-dsl` + `jvm-test-suite` +} + +java { + sourceCompatibility = JavaVersion.VERSION_1_8 + targetCompatibility = JavaVersion.VERSION_1_8 +} + +kotlin { + compilerOptions { + jvmTarget.set(org.jetbrains.kotlin.gradle.dsl.JvmTarget.JVM_1_8) + } +} + +dependencies { + implementation("com.google.cloud.tools:jib-core:0.28.2") +} + +gradlePlugin { + plugins { + create("testcontainers") { + id = "dd-trace-java.testcontainers" + implementationClass = "datadog.buildlogic.testcontainers.TestcontainersPlugin" + } + } +} + +testing { + suites { + named("test") { + useJUnitJupiter(libs.versions.junit5) + dependencies { + implementation(libs.assertj.core) + implementation(gradleTestKit()) + } + } + } +} diff --git a/build-logic/testcontainers/src/main/kotlin/datadog/buildlogic/testcontainers/ContainerImageArguments.kt b/build-logic/testcontainers/src/main/kotlin/datadog/buildlogic/testcontainers/ContainerImageArguments.kt new file mode 100644 index 00000000000..0da3ad13134 --- /dev/null +++ b/build-logic/testcontainers/src/main/kotlin/datadog/buildlogic/testcontainers/ContainerImageArguments.kt @@ -0,0 +1,22 @@ +package datadog.buildlogic.testcontainers + +import org.gradle.api.provider.Provider +import org.gradle.api.tasks.Input +import org.gradle.api.tasks.Internal +import org.gradle.process.CommandLineArgumentProvider +import java.io.File + +class ContainerImageArguments( + @get:Internal val declarations: Map, + @get:Internal val imageEnvironment: Map, + @get:Internal val configurationFiles: List, + @get:Internal val resolver: Provider, +) : CommandLineArgumentProvider { + // Read during Test input snapshotting, after skip predicates and before cache lookup. + // Only the service reference is serialized by the configuration cache; its memo is per build. + @get:Input + val images: Map + get() = resolver.get().resolve(declarations, imageEnvironment, configurationFiles) + + override fun asArguments(): Iterable = images.map { (name, image) -> "-D$name=$image" } +} diff --git a/build-logic/testcontainers/src/main/kotlin/datadog/buildlogic/testcontainers/ImageResolver.kt b/build-logic/testcontainers/src/main/kotlin/datadog/buildlogic/testcontainers/ImageResolver.kt new file mode 100644 index 00000000000..b91de5be102 --- /dev/null +++ b/build-logic/testcontainers/src/main/kotlin/datadog/buildlogic/testcontainers/ImageResolver.kt @@ -0,0 +1,101 @@ +package datadog.buildlogic.testcontainers + +import com.google.cloud.tools.jib.api.ImageReference +import com.google.cloud.tools.jib.api.RegistryUnauthorizedException +import com.google.cloud.tools.jib.event.EventHandlers +import com.google.cloud.tools.jib.frontend.CredentialRetrieverFactory +import com.google.cloud.tools.jib.http.FailoverHttpClient +import com.google.cloud.tools.jib.registry.RegistryClient +import org.gradle.api.GradleException +import org.gradle.api.services.BuildService +import org.gradle.api.services.BuildServiceParameters +import java.io.File +import java.nio.file.Paths +import java.util.Properties +import java.util.concurrent.ConcurrentHashMap + +/** Resolves manifests without a Docker daemon or image-layer downloads. Memoized for this build only. */ +abstract class ImageResolver : BuildService { + private val resolved = ConcurrentHashMap() + + fun resolve( + declarations: Map, + environment: Map, + configurationFiles: List, + ): Map { + val configuration = Properties() + configurationFiles.asReversed().filter { it.isFile }.forEach { file -> + file.inputStream().use { configuration.load(it) } + } + val customSubstitution = + configuration + .stringPropertyNames() + .filter { + it == "image.substitutor" || it.endsWith(".container.image") + }.any { configuration.getProperty(it).isNotBlank() } || + environment.any { (key, value) -> + (key == "TESTCONTAINERS_IMAGE_SUBSTITUTOR" || key.endsWith("_CONTAINER_IMAGE")) && value.isNotBlank() + } + check(!customSubstitution) { + "Custom Testcontainers image substitutions can replace a fingerprinted digest; move image overrides into testContainerImage declarations" + } + val prefix = + environment["TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX"]?.takeIf { it.isNotEmpty() } + ?: configuration.getProperty("hub.image.name.prefix", "") + return declarations.mapValues { (_, reference) -> resolve(reference, prefix) } + } + + private fun resolve( + reference: String, + hubPrefix: String, + ): String { + val firstComponent = reference.substringBefore('/') + val explicitRegistry = + reference.contains('/') && + (firstComponent.contains('.') || firstComponent.contains(':') || firstComponent == "localhost") + val effective = if (explicitRegistry) reference else hubPrefix + reference + return resolved.computeIfAbsent(effective) { resolveManifest(it) } + } + + private fun resolveManifest(reference: String): String { + try { + val image = ImageReference.parse(reference) + val repository = "${image.registry}/${image.repository}" + if (image.digest.isPresent) { + return "$repository@${image.digest.get()}" + } + // Local registries are useful for development and hermetic tests. Remote registries require TLS. + val loopback = image.registry.substringBefore(':') in setOf("localhost", "127.0.0.1") + val http = FailoverHttpClient(loopback, loopback) {} + try { + val factory = RegistryClient.factory(EventHandlers.NONE, image.registry, image.repository, http) + var client = factory.newRegistryClient() + val manifest = + try { + client.pullManifest(image.tag.orElse("latest")) + } catch (unauthorized: RegistryUnauthorizedException) { + val dockerConfig = + System.getenv("DOCKER_CONFIG") + ?: Paths.get(System.getProperty("user.home"), ".docker").toString() + val credential = + CredentialRetrieverFactory + .forImage(image) {} + .dockerConfig(Paths.get(dockerConfig, "config.json")) + .retrieve() + .orElse(null) + client = factory.setCredential(credential).newRegistryClient() + if (!client.doPullBearerAuth()) { + if (credential == null) throw unauthorized + client.configureBasicAuth() + } + client.pullManifest(image.tag.orElse("latest")) + } + return "$repository@${manifest.digest}" + } finally { + http.shutDown() + } + } catch (failure: Exception) { + throw GradleException("Cannot resolve test container image '$reference'; refusing to reuse test results without its digest", failure) + } + } +} diff --git a/build-logic/testcontainers/src/main/kotlin/datadog/buildlogic/testcontainers/TestcontainersPlugin.kt b/build-logic/testcontainers/src/main/kotlin/datadog/buildlogic/testcontainers/TestcontainersPlugin.kt new file mode 100644 index 00000000000..f72545ad9c9 --- /dev/null +++ b/build-logic/testcontainers/src/main/kotlin/datadog/buildlogic/testcontainers/TestcontainersPlugin.kt @@ -0,0 +1,97 @@ +package datadog.buildlogic.testcontainers + +import groovy.lang.Closure +import org.gradle.api.Plugin +import org.gradle.api.Project +import org.gradle.api.plugins.ExtensionAware +import org.gradle.api.tasks.SourceSetContainer +import org.gradle.api.tasks.testing.Test +import org.gradle.kotlin.dsl.getByType +import org.gradle.kotlin.dsl.withType +import java.io.File + +/** Declares container images alongside the dependencies of the test suite that consumes them. */ +class TestcontainersPlugin : Plugin { + override fun apply(project: Project) { + val declarations = mutableMapOf>() + val dependencyDsl = (project.dependencies as ExtensionAware).extensions.extraProperties + dependencyDsl.set( + "image", + object : Closure(null) { + fun doCall( + reference: String, + systemProperty: String, + ): ContainerImage { + require(reference.isNotBlank()) { "Container image reference must not be empty" } + require(systemProperty.matches(Regex("[A-Za-z0-9_.-]+"))) { + "Invalid container image system property: $systemProperty" + } + return ContainerImage(reference, systemProperty) + } + }, + ) + + val resolver = + project.gradle.sharedServices.registerIfAbsent( + "testContainerImageResolver", + ImageResolver::class.java, + ) {} + + project.pluginManager.withPlugin("java") { + val sourceSets = project.extensions.getByType() + sourceSets.all { + val images = declarations.getOrPut(name) { linkedMapOf() } + dependencyDsl.set( + "${name}ContainerImage", + object : Closure(null) { + fun doCall(image: ContainerImage) { + require(images.putIfAbsent(image.systemProperty, image.reference) == null) { + "Container image system property '${image.systemProperty}' is declared twice in $name" + } + } + }, + ) + } + // Suite inheritance is configured by build scripts after the plugin is applied. + project.afterEvaluate { + project.tasks.withType().configureEach { + val suiteName = if (name == "forkedTest") "test" else name.removeSuffix("ForkedTest") + val suite = + sourceSets.findByName(name) ?: sourceSets.findByName(suiteName) + ?: return@configureEach + val hierarchy = project.configurations.getByName(suite.implementationConfigurationName).hierarchy + val images = linkedMapOf() + val inheritedSourceSets = + sourceSets.filter { sourceSet -> + hierarchy.any { it.name == sourceSet.implementationConfigurationName } + } + inheritedSourceSets.forEach { sourceSet -> + declarations[sourceSet.name]?.forEach { (property, reference) -> + require(images.putIfAbsent(property, reference).let { it == null || it == reference }) { + "Conflicting container images for '$property' in $path" + } + } + } + if (images.isNotEmpty()) { + usesService(resolver) + val configurationFiles = + listOf(File(System.getProperty("user.home"), ".testcontainers.properties")) + + inheritedSourceSets.flatMap { it.resources.srcDirs }.map { File(it, "testcontainers.properties") } + val imageEnvironment = + (project.providers.environmentVariablesPrefixedBy("TESTCONTAINERS_").get() + environment) + .filterKeys { + it == "TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX" || it == "TESTCONTAINERS_IMAGE_SUBSTITUTOR" || + (it.startsWith("TESTCONTAINERS_") && it.endsWith("_CONTAINER_IMAGE")) + }.mapValues { it.value.toString() } + jvmArgumentProviders.add(ContainerImageArguments(images, imageEnvironment, configurationFiles, resolver)) + } + } + } + } + } +} + +data class ContainerImage( + val reference: String, + val systemProperty: String, +) diff --git a/build-logic/testcontainers/src/test/kotlin/datadog/buildlogic/testcontainers/TestcontainersPluginTest.kt b/build-logic/testcontainers/src/test/kotlin/datadog/buildlogic/testcontainers/TestcontainersPluginTest.kt new file mode 100644 index 00000000000..46dfde1e7fb --- /dev/null +++ b/build-logic/testcontainers/src/test/kotlin/datadog/buildlogic/testcontainers/TestcontainersPluginTest.kt @@ -0,0 +1,309 @@ +package datadog.buildlogic.testcontainers + +import com.sun.net.httpserver.HttpsConfigurator +import com.sun.net.httpserver.HttpsServer +import org.assertj.core.api.Assertions.assertThat +import org.gradle.testkit.runner.GradleRunner +import org.gradle.testkit.runner.TaskOutcome +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import java.net.InetSocketAddress +import java.nio.file.Files +import java.nio.file.Path +import java.security.KeyStore +import java.security.MessageDigest +import java.util.concurrent.atomic.AtomicInteger +import java.util.concurrent.atomic.AtomicReference +import javax.net.ssl.KeyManagerFactory +import javax.net.ssl.SSLContext + +class TestcontainersPluginTest { + @TempDir + lateinit var directory: Path + + @Test + fun `moving images are refreshed before cache lookup including configuration cache reuse`() { + val manifests = AtomicReference(manifest("1")) + val requests = AtomicInteger() + val server = registry() + server.createContext("/v2/") { exchange -> + requests.incrementAndGet() + val body = manifests.get().toByteArray() + exchange.responseHeaders.add("Content-Type", "application/vnd.oci.image.manifest.v1+json") + exchange.responseHeaders.add("Docker-Content-Digest", digest(manifests.get())) + exchange.sendResponseHeaders(200, body.size.toLong()) + exchange.responseBody.use { it.write(body) } + } + server.start() + try { + fixture("127.0.0.1:${server.address.port}/library/cassandra:4") + assertThat(run("help").task(":help")?.outcome).isEqualTo(TaskOutcome.SUCCESS) + assertThat(requests.get()).isZero() + assertThat(run("test", "-PskipTests").task(":test")?.outcome).isEqualTo(TaskOutcome.SKIPPED) + assertThat(requests.get()).isZero() + + assertThat(run("test").task(":test")?.outcome).isEqualTo(TaskOutcome.SUCCESS) + assertThat(report()).contains("library/cassandra@${digest(manifests.get())}") + val firstRequests = requests.get() + val warm = run("test") + assertThat(warm.output).contains("Reusing configuration cache") + assertThat(warm.task(":test")?.outcome).isEqualTo(TaskOutcome.UP_TO_DATE) + assertThat(requests.get()).isGreaterThan(firstRequests) + + manifests.set(manifest("2")) + val changed = run("test") + assertThat(changed.output).contains("Reusing configuration cache") + assertThat(changed.task(":test")?.outcome).isEqualTo(TaskOutcome.SUCCESS) + assertThat(report()).contains("library/cassandra@${digest(manifests.get())}") + + directory.resolve("build").toFile().deleteRecursively() + val restored = run("test") + assertThat(restored.output).contains("Reusing configuration cache") + assertThat(restored.task(":test")?.outcome).isEqualTo(TaskOutcome.FROM_CACHE) + assertThat(report()).contains("library/cassandra@${digest(manifests.get())}") + + assertThat(run("latestDepTest", "latestDepTestForkedTest").task(":latestDepTestForkedTest")?.outcome) + .isIn(TaskOutcome.SUCCESS, TaskOutcome.FROM_CACHE) + assertThat(directory.resolve("build/test-results/latestDepTest/TEST-ImageTest.xml").toFile().readText()) + .contains("library/cassandra@${digest(manifests.get())}") + val requestsBeforeUnrelatedSuite = requests.get() + assertThat(run("isolatedTest").task(":isolatedTest")?.outcome).isEqualTo(TaskOutcome.SUCCESS) + assertThat(run("emptyTest").task(":emptyTest")?.outcome).isEqualTo(TaskOutcome.NO_SOURCE) + assertThat(requests.get()).isEqualTo(requestsBeforeUnrelatedSuite) + + server.stop(0) + assertThat(runner("test").buildAndFail().output) + .contains("Cannot resolve test container image") + } finally { + server.stop(0) + } + } + + @Test + fun `hub prefix is resolved and pinned references require no registry`() { + val digest = "sha256:${"a".repeat(64)}" + fixture("cassandra@$digest") + val result = + runner("test") + .withEnvironment( + System.getenv() + + ("TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX" to "mirror.example/team/"), + ).build() + assertThat(result.task(":test")?.outcome).isEqualTo(TaskOutcome.SUCCESS) + assertThat(report()).contains("mirror.example/team/cassandra@$digest") + val changed = + runner("test") + .withEnvironment( + System.getenv() + + ("TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX" to "another.example/team/"), + ).build() + assertThat(changed.task(":test")?.outcome).isEqualTo(TaskOutcome.SUCCESS) + assertThat(report()).contains("another.example/team/cassandra@$digest") + + Files.createDirectories(directory.resolve("src/test/resources")) + directory + .resolve("src/test/resources/testcontainers.properties") + .toFile() + .writeText("image.substitutor=untracked.CustomSubstitutor\n") + assertThat(runner("test").buildAndFail().output) + .contains("move image overrides into testContainerImage declarations") + } + + @Test + fun `bearer authentication resolves the registry manifest`() { + val server = registry() + val tokenRequests = AtomicInteger() + val authorizedRequests = AtomicInteger() + server.createContext("/token") { exchange -> + tokenRequests.incrementAndGet() + val body = """{"token":"fixture-token"}""".toByteArray() + exchange.responseHeaders.add("Content-Type", "application/json") + exchange.sendResponseHeaders(200, body.size.toLong()) + exchange.responseBody.use { it.write(body) } + } + server.createContext("/v2/") { exchange -> + if (exchange.requestHeaders.getFirst("Authorization") != "Bearer fixture-token") { + exchange.responseHeaders.add( + "WWW-Authenticate", + "Bearer realm=\"https://127.0.0.1:${server.address.port}/token\",service=\"fixture\",scope=\"repository:library/cassandra:pull\"", + ) + exchange.sendResponseHeaders(401, -1) + exchange.close() + } else { + authorizedRequests.incrementAndGet() + val body = manifest("3").toByteArray() + exchange.responseHeaders.add("Content-Type", "application/vnd.oci.image.manifest.v1+json") + exchange.sendResponseHeaders(200, body.size.toLong()) + exchange.responseBody.use { it.write(body) } + } + } + server.start() + try { + fixture("127.0.0.1:${server.address.port}/library/cassandra:4") + assertThat(run("test").task(":test")?.outcome).isEqualTo(TaskOutcome.SUCCESS) + assertThat(tokenRequests.get()).isPositive() + assertThat(authorizedRequests.get()).isPositive() + assertThat(report()).contains(digest(manifest("3"))) + } finally { + server.stop(0) + } + } + + private fun fixture(image: String) { + directory.resolve("settings.gradle").toFile().writeText( + """ + rootProject.name = 'container-image-fixture' + buildCache { local { directory = file('cache') } } + """.trimIndent(), + ) + val junitClasspath = + listOf( + "org.junit.jupiter.api.Test", + "org.junit.jupiter.engine.JupiterTestEngine", + "org.junit.platform.engine.TestEngine", + "org.junit.platform.launcher.Launcher", + "org.junit.platform.commons.JUnitException", + "org.opentest4j.AssertionFailedError", + ).joinToString(", ") { + "'${Path.of( + Class + .forName(it) + .protectionDomain.codeSource.location + .toURI(), + )}'" + } + directory.resolve("build.gradle").toFile().writeText( + """ + plugins { + id 'java' + id 'dd-trace-java.testcontainers' + } + sourceSets { + latestDepTest { java.srcDirs = sourceSets.test.java.srcDirs } + isolatedTest + emptyTest + } + configurations.latestDepTestImplementation.extendsFrom(configurations.testImplementation) + configurations.emptyTestImplementation.extendsFrom(configurations.testImplementation) + dependencies { + testImplementation files($junitClasspath) + isolatedTestImplementation files($junitClasspath) + testContainerImage(image('$image', 'test.cassandra.image')) + } + tasks.register('latestDepTest', Test) { + testClassesDirs = sourceSets.latestDepTest.output.classesDirs + classpath = sourceSets.latestDepTest.runtimeClasspath + } + tasks.register('latestDepTestForkedTest', Test) { + testClassesDirs = sourceSets.latestDepTest.output.classesDirs + classpath = sourceSets.latestDepTest.runtimeClasspath + } + tasks.register('isolatedTest', Test) { + testClassesDirs = sourceSets.isolatedTest.output.classesDirs + classpath = sourceSets.isolatedTest.runtimeClasspath + } + tasks.register('emptyTest', Test) { + testClassesDirs = sourceSets.emptyTest.output.classesDirs + classpath = sourceSets.emptyTest.runtimeClasspath + } + def skip = providers.gradleProperty('skipTests') + tasks.withType(Test).configureEach { + useJUnitPlatform() + onlyIf { !skip.isPresent() } + } + """.trimIndent(), + ) + Files.createDirectories(directory.resolve("src/test/java")) + directory.resolve("src/test/java/ImageTest.java").toFile().writeText( + """ + import org.junit.jupiter.api.Test; + import static org.junit.jupiter.api.Assertions.assertTrue; + public class ImageTest { + @Test public void imageIsPinned() { + String image = System.getProperty("test.cassandra.image"); + assertTrue(image.matches(".+@sha256:[a-f0-9]{64}"), image); + System.out.println(image); + } + } + """.trimIndent(), + ) + Files.createDirectories(directory.resolve("src/isolatedTest/java")) + directory.resolve("src/isolatedTest/java/PlainTest.java").toFile().writeText( + """ + import org.junit.jupiter.api.Test; + import static org.junit.jupiter.api.Assertions.assertNull; + public class PlainTest { + @Test public void hasNoContainerDependency() { + assertNull(System.getProperty("test.cassandra.image")); + } + } + """.trimIndent(), + ) + } + + private fun runner(vararg arguments: String) = + GradleRunner + .create() + .withProjectDir(directory.toFile()) + .withPluginClasspath() + .withArguments( + *arguments, + "--build-cache", + "--configuration-cache", + "--stacktrace", + "--max-workers=2", + "-Dorg.gradle.jvmargs=-Xmx512m", + ) + + private fun run(vararg arguments: String) = runner(*arguments).build() + + private fun report() = directory.resolve("build/test-results/test/TEST-ImageTest.xml").toFile().readText() + + private fun manifest(character: String) = + """{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json","config":{"mediaType":"application/vnd.oci.image.config.v1+json","digest":"sha256:${character.repeat( + 64, + )}","size":2},"layers":[]}""" + + private fun registry(): HttpsServer { + val keystore = directory.resolve("registry.p12") + val keytool = Path.of(System.getProperty("java.home"), "bin", "keytool").toString() + val process = + ProcessBuilder( + keytool, + "-genkeypair", + "-alias", + "registry", + "-keyalg", + "RSA", + "-keystore", + keystore.toString(), + "-storepass", + "fixture", + "-keypass", + "fixture", + "-dname", + "CN=localhost", + "-validity", + "1", + "-noprompt", + ).redirectErrorStream(true).start() + val output = process.inputStream.bufferedReader().readText() + check(process.waitFor() == 0) { output } + val keys = KeyStore.getInstance("PKCS12") + Files.newInputStream(keystore).use { keys.load(it, "fixture".toCharArray()) } + val manager = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()) + manager.init(keys, "fixture".toCharArray()) + val context = SSLContext.getInstance("TLS") + context.init(manager.keyManagers, null, null) + return HttpsServer.create(InetSocketAddress("127.0.0.1", 0), 0).apply { + httpsConfigurator = HttpsConfigurator(context) + } + } + + private fun digest(body: String) = + "sha256:" + + MessageDigest + .getInstance("SHA-256") + .digest(body.toByteArray()) + .joinToString("") { "%02x".format(it) } +} diff --git a/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-3.0/build.gradle b/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-3.0/build.gradle index f74982c2038..9718e557981 100644 --- a/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-3.0/build.gradle +++ b/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-3.0/build.gradle @@ -1,5 +1,6 @@ plugins { id 'dd-trace-java.module.instrumentation' + id 'dd-trace-java.testcontainers' } muzzle { @@ -50,6 +51,8 @@ testJvmConstraints { addTestSuiteForDir('latestDepTest', 'test') dependencies { + testContainerImage(image('cassandra:3', 'test.cassandra.image')) + constraints { testImplementation("com.google.guava:guava") { version { strictly "19.0" } diff --git a/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-3.0/src/test/groovy/CassandraClientTest.groovy b/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-3.0/src/test/groovy/CassandraClientTest.groovy index d4e0ac21065..85d414bb5c3 100644 --- a/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-3.0/src/test/groovy/CassandraClientTest.groovy +++ b/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-3.0/src/test/groovy/CassandraClientTest.groovy @@ -13,6 +13,7 @@ import datadog.trace.bootstrap.instrumentation.api.InstrumentationTags import datadog.trace.bootstrap.instrumentation.api.Tags import datadog.trace.core.DDSpan import org.testcontainers.containers.CassandraContainer +import org.testcontainers.utility.DockerImageName import spock.lang.Shared import java.time.Duration @@ -42,7 +43,9 @@ abstract class CassandraClientTest extends VersionedNamingTestBase { CassandraContainer container def setupSpec() { - container = new CassandraContainer("cassandra:3").withStartupTimeout(Duration.ofSeconds(120)) + def image = DockerImageName.parse(System.getProperty("test.cassandra.image")) + .asCompatibleSubstituteFor("cassandra") + container = new CassandraContainer(image).withStartupTimeout(Duration.ofSeconds(120)) container.start() cluster = container.getCluster() port = container.getMappedPort(9042) diff --git a/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-3.8/build.gradle b/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-3.8/build.gradle index 535c56510e1..5e74728b352 100644 --- a/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-3.8/build.gradle +++ b/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-3.8/build.gradle @@ -1,5 +1,6 @@ plugins { id 'dd-trace-java.module.instrumentation' + id 'dd-trace-java.testcontainers' } muzzle { @@ -26,6 +27,8 @@ testJvmConstraints { addTestSuiteForDir('latestDepTest', 'test') dependencies { + testContainerImage(image('cassandra:3', 'test.cassandra.image')) + compileOnly group: 'com.datastax.cassandra', name: 'cassandra-driver-core', version: '3.8.0' compileOnly group: 'com.google.guava', name: 'guava', version: '18.0' diff --git a/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-3.8/src/test/groovy/CassandraClientTest.groovy b/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-3.8/src/test/groovy/CassandraClientTest.groovy index d4e0ac21065..85d414bb5c3 100644 --- a/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-3.8/src/test/groovy/CassandraClientTest.groovy +++ b/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-3.8/src/test/groovy/CassandraClientTest.groovy @@ -13,6 +13,7 @@ import datadog.trace.bootstrap.instrumentation.api.InstrumentationTags import datadog.trace.bootstrap.instrumentation.api.Tags import datadog.trace.core.DDSpan import org.testcontainers.containers.CassandraContainer +import org.testcontainers.utility.DockerImageName import spock.lang.Shared import java.time.Duration @@ -42,7 +43,9 @@ abstract class CassandraClientTest extends VersionedNamingTestBase { CassandraContainer container def setupSpec() { - container = new CassandraContainer("cassandra:3").withStartupTimeout(Duration.ofSeconds(120)) + def image = DockerImageName.parse(System.getProperty("test.cassandra.image")) + .asCompatibleSubstituteFor("cassandra") + container = new CassandraContainer(image).withStartupTimeout(Duration.ofSeconds(120)) container.start() cluster = container.getCluster() port = container.getMappedPort(9042) diff --git a/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-4.0/build.gradle b/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-4.0/build.gradle index 595c7874922..5a87a224cd0 100644 --- a/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-4.0/build.gradle +++ b/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-4.0/build.gradle @@ -1,5 +1,6 @@ plugins { id 'dd-trace-java.module.instrumentation' + id 'dd-trace-java.testcontainers' } muzzle { @@ -20,6 +21,8 @@ testJvmConstraints { addTestSuiteForDir('latestDepTest', 'test') dependencies { + testContainerImage(image('cassandra:4', 'test.cassandra.image')) + compileOnly group: 'com.datastax.oss', name: 'java-driver-core', version: '4.0.0' // ProgrammaticConfig, required to set the timeout, wasn't added until 4.0.1 diff --git a/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-4.0/src/test/groovy/CassandraClientTest.groovy b/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-4.0/src/test/groovy/CassandraClientTest.groovy index ba3aadb8773..fd3d735bfac 100644 --- a/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-4.0/src/test/groovy/CassandraClientTest.groovy +++ b/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-4.0/src/test/groovy/CassandraClientTest.groovy @@ -13,6 +13,7 @@ import datadog.trace.bootstrap.instrumentation.api.InstrumentationTags import datadog.trace.bootstrap.instrumentation.api.Tags import datadog.trace.core.DDSpan import org.testcontainers.containers.CassandraContainer +import org.testcontainers.utility.DockerImageName import spock.lang.Shared import spock.util.concurrent.BlockingVariable @@ -44,7 +45,9 @@ abstract class CassandraClientTest extends VersionedNamingTestBase { CassandraContainer container def setupSpec() { - container = new CassandraContainer("cassandra:4").withStartupTimeout(Duration.ofSeconds(120)) + def image = DockerImageName.parse(System.getProperty("test.cassandra.image")) + .asCompatibleSubstituteFor("cassandra") + container = new CassandraContainer(image).withStartupTimeout(Duration.ofSeconds(120)) container.start() port = container.getMappedPort(9042) address = new InetSocketAddress(container.getHost(), port) diff --git a/dd-java-agent/instrumentation/google-pubsub-1.116/build.gradle b/dd-java-agent/instrumentation/google-pubsub-1.116/build.gradle index 03f919585a7..a2b03630339 100644 --- a/dd-java-agent/instrumentation/google-pubsub-1.116/build.gradle +++ b/dd-java-agent/instrumentation/google-pubsub-1.116/build.gradle @@ -1,5 +1,6 @@ plugins { id 'dd-trace-java.module.instrumentation' + id 'dd-trace-java.testcontainers' } muzzle { @@ -16,6 +17,8 @@ addTestSuiteExtendingForDir('latestDepForkedTest', 'latestDepTest', 'test') dependencies { + testContainerImage(image('gcr.io/google.com/cloudsdktool/google-cloud-cli:emulators', 'test.pubsub.image')) + compileOnly group: 'com.google.cloud', name: 'google-cloud-pubsub', version: '1.116.0' testImplementation group: 'org.testcontainers', name: 'gcloud', version: libs.versions.testcontainers.get() testImplementation group: 'com.google.cloud', name: 'google-cloud-pubsub', version: '1.116.0' diff --git a/dd-java-agent/instrumentation/google-pubsub-1.116/src/test/groovy/PubSubTest.groovy b/dd-java-agent/instrumentation/google-pubsub-1.116/src/test/groovy/PubSubTest.groovy index 3317485f031..7c40ed076ae 100644 --- a/dd-java-agent/instrumentation/google-pubsub-1.116/src/test/groovy/PubSubTest.groovy +++ b/dd-java-agent/instrumentation/google-pubsub-1.116/src/test/groovy/PubSubTest.groovy @@ -90,7 +90,9 @@ abstract class PubSubTest extends VersionedNamingTestBase { } def setupSpec() { - emulator = new PubSubEmulatorContainer(DockerImageName.parse("gcr.io/google.com/cloudsdktool/google-cloud-cli:emulators")) + def image = DockerImageName.parse(System.getProperty("test.pubsub.image")) + .asCompatibleSubstituteFor("gcr.io/google.com/cloudsdktool/google-cloud-cli") + emulator = new PubSubEmulatorContainer(image) emulator.start() channel = ManagedChannelBuilder.forTarget(emulator.getEmulatorEndpoint()).usePlaintext().build() transportChannelProvider = FixedTransportChannelProvider.create(GrpcTransportChannel.create(channel)) diff --git a/dd-java-agent/instrumentation/jdbc/build.gradle b/dd-java-agent/instrumentation/jdbc/build.gradle index 6ee3987ca2d..722c6e60386 100644 --- a/dd-java-agent/instrumentation/jdbc/build.gradle +++ b/dd-java-agent/instrumentation/jdbc/build.gradle @@ -1,6 +1,7 @@ plugins { id 'java-test-fixtures' id 'dd-trace-java.module.instrumentation' + id 'dd-trace-java.testcontainers' id 'dd-trace-java.call-site-instrumentation' id 'dd-trace-java.jmh-conventions' } @@ -18,7 +19,17 @@ addTestSuiteForDir('oldPostgresTest', 'test') addTestSuiteForDir('latestDepTest', 'test') addTestSuiteExtendingForDir('latestDepJava11Test', 'latestDepTest', 'test') +// This non-Docker-Hub image has a separate mirror in CI. +def sqlServerImage = providers.environmentVariable('CI').isPresent() + ? 'registry.ddbuild.io/images/mirror/sqlserver:latest' + : 'mcr.microsoft.com/mssql/server:latest' + dependencies { + testContainerImage(image('postgres:11.2', 'test.postgres.image')) + testContainerImage(image('mysql:8.0', 'test.mysql.image')) + testContainerImage(image(sqlServerImage, 'test.sqlserver.image')) + testContainerImage(image('gvenzl/oracle-free:23.5-slim-faststart', 'test.oracle.image')) + compileOnly group: 'com.zaxxer', name: 'HikariCP', version: '2.4.0' testImplementation project(':dd-java-agent:agent-iast:iast-test-fixtures') diff --git a/dd-java-agent/instrumentation/jdbc/src/test/groovy/RemoteJDBCInstrumentationTest.groovy b/dd-java-agent/instrumentation/jdbc/src/test/groovy/RemoteJDBCInstrumentationTest.groovy index 069971e97ef..d8608095149 100644 --- a/dd-java-agent/instrumentation/jdbc/src/test/groovy/RemoteJDBCInstrumentationTest.groovy +++ b/dd-java-agent/instrumentation/jdbc/src/test/groovy/RemoteJDBCInstrumentationTest.groovy @@ -154,7 +154,9 @@ abstract class RemoteJDBCInstrumentationTest extends VersionedNamingTestBase { return } - PostgreSQLContainer server = new PostgreSQLContainer("postgres:11.2") + def image = DockerImageName.parse(System.getProperty("test.postgres.image")) + .asCompatibleSubstituteFor("postgres") + PostgreSQLContainer server = new PostgreSQLContainer(image) .withDatabaseName(dbName.get(POSTGRESQL)) .withUsername(jdbcUserNames.get(POSTGRESQL)) .withPassword(jdbcPasswords.get(POSTGRESQL)) @@ -175,7 +177,9 @@ abstract class RemoteJDBCInstrumentationTest extends VersionedNamingTestBase { return } - MySQLContainer server = new MySQLContainer("mysql:8.0") + def image = DockerImageName.parse(System.getProperty("test.mysql.image")) + .asCompatibleSubstituteFor("mysql") + MySQLContainer server = new MySQLContainer(image) .withDatabaseName(dbName.get(MYSQL)) .withUsername(jdbcUserNames.get(MYSQL)) .withPassword(jdbcPasswords.get(MYSQL)) @@ -198,7 +202,9 @@ abstract class RemoteJDBCInstrumentationTest extends VersionedNamingTestBase { return } - MSSQLServerContainer server = new MSSQLServerContainer(MSSQLServerContainer.IMAGE) + def image = DockerImageName.parse(System.getProperty("test.sqlserver.image")) + .asCompatibleSubstituteFor(MSSQLServerContainer.IMAGE) + MSSQLServerContainer server = new MSSQLServerContainer(image) .acceptLicense() .withPassword(jdbcPasswords.get(SQLSERVER)) // SQL Server can occasionally abort while booting on virtualized CI hosts. @@ -221,7 +227,8 @@ abstract class RemoteJDBCInstrumentationTest extends VersionedNamingTestBase { } // Earlier Oracle version images (oracle-xe) don't work on arm64 - DockerImageName oracleImage = DockerImageName.parse("gvenzl/oracle-free:23.5-slim-faststart").asCompatibleSubstituteFor("gvenzl/oracle-xe") + DockerImageName oracleImage = DockerImageName.parse(System.getProperty("test.oracle.image")) + .asCompatibleSubstituteFor("gvenzl/oracle-xe") OracleContainer server = new OracleContainer(oracleImage) .withStartupTimeout(Duration.ofMinutes(5)) .withUsername(jdbcUserNames.get(ORACLE)) diff --git a/dd-java-agent/instrumentation/jdbc/src/test/java/test/DataDogRegistryImageNameSubstitutor.java b/dd-java-agent/instrumentation/jdbc/src/test/java/test/DataDogRegistryImageNameSubstitutor.java deleted file mode 100644 index 8f7693f49d1..00000000000 --- a/dd-java-agent/instrumentation/jdbc/src/test/java/test/DataDogRegistryImageNameSubstitutor.java +++ /dev/null @@ -1,35 +0,0 @@ -package test; - -import org.testcontainers.utility.DockerImageName; -import org.testcontainers.utility.ImageNameSubstitutor; - -/** - * A custom {@link ImageNameSubstitutor} implementation that rewrites Docker image names to use - * Datadog's internal registry {@code registry.ddbuild.io} when running in a CI environment. - * - *

Images from DockerHub already mirrored by {@code registry.ddbuild.io} via environment variable - * {@code TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX} - * - *

For images from other repositories custom image name substitutor should be implemented. - * Internal registry is faster and not affected by rate limiting. - */ -public class DataDogRegistryImageNameSubstitutor extends ImageNameSubstitutor { - @Override - public DockerImageName apply(DockerImageName original) { - String name = original.asCanonicalNameString(); - - if (System.getenv("CI") != null) { - // For now, we need to mirror Microsoft SQL Server images only. - name = - name.replace( - "mcr.microsoft.com/mssql/server:", "registry.ddbuild.io/images/mirror/sqlserver:"); - } - - return DockerImageName.parse(name); - } - - @Override - protected String getDescription() { - return "Image name substitutor to load images from registry.ddbuild.io"; - } -} diff --git a/dd-java-agent/instrumentation/jdbc/src/test/resources/testcontainers.properties b/dd-java-agent/instrumentation/jdbc/src/test/resources/testcontainers.properties deleted file mode 100644 index 03fb660c6d1..00000000000 --- a/dd-java-agent/instrumentation/jdbc/src/test/resources/testcontainers.properties +++ /dev/null @@ -1 +0,0 @@ -image.substitutor=test.DataDogRegistryImageNameSubstitutor diff --git a/dd-java-agent/instrumentation/vertx/vertx-mysql-client/vertx-mysql-client-3.9/build.gradle b/dd-java-agent/instrumentation/vertx/vertx-mysql-client/vertx-mysql-client-3.9/build.gradle index 4485ea3aa32..1f8423384f0 100644 --- a/dd-java-agent/instrumentation/vertx/vertx-mysql-client/vertx-mysql-client-3.9/build.gradle +++ b/dd-java-agent/instrumentation/vertx/vertx-mysql-client/vertx-mysql-client-3.9/build.gradle @@ -1,5 +1,6 @@ plugins { id 'dd-trace-java.module.instrumentation' + id 'dd-trace-java.testcontainers' } muzzle { @@ -20,6 +21,8 @@ tasks.named("latestDepTest", Test) { } dependencies { + testContainerImage(image('mysql:8.0', 'test.mysql.image')) + compileOnly group: 'io.vertx', name: 'vertx-mysql-client', version: '3.9.0' testImplementation group: 'io.vertx', name: 'vertx-mysql-client', version: '3.9.0' diff --git a/dd-java-agent/instrumentation/vertx/vertx-mysql-client/vertx-mysql-client-3.9/src/test/java/TestDatabases.java b/dd-java-agent/instrumentation/vertx/vertx-mysql-client/vertx-mysql-client-3.9/src/test/java/TestDatabases.java index 47793bd21cc..5d7111ae69a 100644 --- a/dd-java-agent/instrumentation/vertx/vertx-mysql-client/vertx-mysql-client-3.9/src/test/java/TestDatabases.java +++ b/dd-java-agent/instrumentation/vertx/vertx-mysql-client/vertx-mysql-client-3.9/src/test/java/TestDatabases.java @@ -6,6 +6,7 @@ import java.util.Map; import java.util.concurrent.TimeUnit; import org.testcontainers.containers.MySQLContainer; +import org.testcontainers.utility.DockerImageName; public class TestDatabases implements Closeable { @@ -20,7 +21,9 @@ public static TestDatabases initialise(String dbName) { private TestDatabases(String dbName) { Map infos = new HashMap<>(); mysql = - new MySQLContainer("mysql:8.0") + new MySQLContainer( + DockerImageName.parse(System.getProperty("test.mysql.image")) + .asCompatibleSubstituteFor("mysql")) .withDatabaseName(dbName) .withUsername("sa") .withPassword("sa"); diff --git a/dd-java-agent/instrumentation/vertx/vertx-mysql-client/vertx-mysql-client-4.0/build.gradle b/dd-java-agent/instrumentation/vertx/vertx-mysql-client/vertx-mysql-client-4.0/build.gradle index ecdd088f75a..c314173c643 100644 --- a/dd-java-agent/instrumentation/vertx/vertx-mysql-client/vertx-mysql-client-4.0/build.gradle +++ b/dd-java-agent/instrumentation/vertx/vertx-mysql-client/vertx-mysql-client-4.0/build.gradle @@ -1,5 +1,6 @@ plugins { id 'dd-trace-java.module.instrumentation' + id 'dd-trace-java.testcontainers' } muzzle { @@ -19,6 +20,8 @@ tasks.named("latestDepTest", Test) { } dependencies { + testContainerImage(image('mysql:8.0', 'test.mysql.image')) + compileOnly group: 'io.vertx', name: 'vertx-mysql-client', version: '4.0.0' testImplementation group: 'io.vertx', name: 'vertx-mysql-client', version: '4.0.0' diff --git a/dd-java-agent/instrumentation/vertx/vertx-mysql-client/vertx-mysql-client-4.0/src/test/java/TestDatabases.java b/dd-java-agent/instrumentation/vertx/vertx-mysql-client/vertx-mysql-client-4.0/src/test/java/TestDatabases.java index d3b86359399..e6ed288255b 100644 --- a/dd-java-agent/instrumentation/vertx/vertx-mysql-client/vertx-mysql-client-4.0/src/test/java/TestDatabases.java +++ b/dd-java-agent/instrumentation/vertx/vertx-mysql-client/vertx-mysql-client-4.0/src/test/java/TestDatabases.java @@ -6,6 +6,7 @@ import java.util.Map; import java.util.concurrent.TimeUnit; import org.testcontainers.containers.MySQLContainer; +import org.testcontainers.utility.DockerImageName; public class TestDatabases implements Closeable { @@ -15,7 +16,9 @@ public class TestDatabases implements Closeable { private TestDatabases(String dbName) { Map infos = new HashMap<>(); mysql = - new MySQLContainer("mysql:8.0") + new MySQLContainer( + DockerImageName.parse(System.getProperty("test.mysql.image")) + .asCompatibleSubstituteFor("mysql")) .withDatabaseName(dbName) .withUsername("sa") .withPassword("sa"); diff --git a/dd-java-agent/instrumentation/vertx/vertx-pg-client/vertx-pg-client-4.0/build.gradle b/dd-java-agent/instrumentation/vertx/vertx-pg-client/vertx-pg-client-4.0/build.gradle index 99a8aee5236..0db168b53ad 100644 --- a/dd-java-agent/instrumentation/vertx/vertx-pg-client/vertx-pg-client-4.0/build.gradle +++ b/dd-java-agent/instrumentation/vertx/vertx-pg-client/vertx-pg-client-4.0/build.gradle @@ -1,5 +1,6 @@ plugins { id 'dd-trace-java.module.instrumentation' + id 'dd-trace-java.testcontainers' } muzzle { @@ -27,6 +28,8 @@ tasks.named("latestDepTest", Test) { } dependencies { + testContainerImage(image('postgres:16-alpine', 'test.postgres.image')) + compileOnly group: 'io.vertx', name: 'vertx-pg-client', version: '4.1.1' testImplementation group: 'io.vertx', name: 'vertx-pg-client', version: '4.1.1' diff --git a/dd-java-agent/instrumentation/vertx/vertx-pg-client/vertx-pg-client-4.0/src/test/java/TestDatabases.java b/dd-java-agent/instrumentation/vertx/vertx-pg-client/vertx-pg-client-4.0/src/test/java/TestDatabases.java index 1e31905cf79..12f06ccaac3 100644 --- a/dd-java-agent/instrumentation/vertx/vertx-pg-client/vertx-pg-client-4.0/src/test/java/TestDatabases.java +++ b/dd-java-agent/instrumentation/vertx/vertx-pg-client/vertx-pg-client-4.0/src/test/java/TestDatabases.java @@ -6,6 +6,7 @@ import java.util.Map; import java.util.concurrent.TimeUnit; import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.utility.DockerImageName; public class TestDatabases implements Closeable { @@ -15,7 +16,9 @@ public class TestDatabases implements Closeable { private TestDatabases(String dbName) { Map infos = new HashMap<>(); pgsql = - new PostgreSQLContainer("postgres:16-alpine") + new PostgreSQLContainer( + DockerImageName.parse(System.getProperty("test.postgres.image")) + .asCompatibleSubstituteFor("postgres")) .withDatabaseName(dbName) .withUsername("postgres") .withPassword("postgres"); diff --git a/dd-smoke-tests/websphere-jmx/build.gradle b/dd-smoke-tests/websphere-jmx/build.gradle index 33a96fb8465..b3ee93ac840 100644 --- a/dd-smoke-tests/websphere-jmx/build.gradle +++ b/dd-smoke-tests/websphere-jmx/build.gradle @@ -1,8 +1,11 @@ plugins { id 'dd-trace-java.module.smoke-test' + id 'dd-trace-java.testcontainers' } dependencies { + testContainerImage(image('icr.io/appcafe/websphere-traditional:latest', 'test.websphere.image')) + testImplementation project(':dd-smoke-tests') testImplementation libs.testcontainers } diff --git a/dd-smoke-tests/websphere-jmx/src/test/groovy/datadog/smoketest/WebSphereJmxSmokeTest.groovy b/dd-smoke-tests/websphere-jmx/src/test/groovy/datadog/smoketest/WebSphereJmxSmokeTest.groovy index 81dde64229d..cc329bc8056 100644 --- a/dd-smoke-tests/websphere-jmx/src/test/groovy/datadog/smoketest/WebSphereJmxSmokeTest.groovy +++ b/dd-smoke-tests/websphere-jmx/src/test/groovy/datadog/smoketest/WebSphereJmxSmokeTest.groovy @@ -71,7 +71,7 @@ class WebSphereJmxSmokeTest extends AbstractSmokeTest { } } - websphere = new GenericContainer("icr.io/appcafe/websphere-traditional:latest") + websphere = new GenericContainer(System.getProperty("test.websphere.image")) // inject wished jvm props for the server we are running .withCopyFileToContainer(MountableFile.forClasspathResource("jvm-config.props"), "/work/config/") // copy the agent jar From 2c65ad64f4de30e8481d239415e1edcaf307eb0f Mon Sep 17 00:00:00 2001 From: Brice Dutheil Date: Thu, 24 Sep 2026 09:54:36 +0200 Subject: [PATCH 02/15] fix: isolate test container image resolution dependencies --- build-logic/testcontainers/README.md | 3 + build-logic/testcontainers/build.gradle.kts | 32 ++++++++- .../TestcontainersPluginTest.kt | 67 ++++++++++++++----- 3 files changed, 86 insertions(+), 16 deletions(-) diff --git a/build-logic/testcontainers/README.md b/build-logic/testcontainers/README.md index e879c975bcb..0f729bdc431 100644 --- a/build-logic/testcontainers/README.md +++ b/build-logic/testcontainers/README.md @@ -36,6 +36,9 @@ tags. Unrelated tasks and skipped tests perform no registry requests. Resolution failure stops the test instead of trusting stale results. Jib reads registry manifests without pulling layers or requiring a Docker daemon. +Its dependencies are relocated inside the plugin JAR so older libraries exported +by `buildSrc` cannot override its HTTP client. Tests load that JAR through an +included build with an older HttpClient on the `buildSrc` classpath. Private registries use Docker's `config.json` and credential helpers, honoring `DOCKER_CONFIG`. Remote registries require TLS; loopback registries also permit local development certificates and HTTP. diff --git a/build-logic/testcontainers/build.gradle.kts b/build-logic/testcontainers/build.gradle.kts index dbf6d83dcea..9668da207b4 100644 --- a/build-logic/testcontainers/build.gradle.kts +++ b/build-logic/testcontainers/build.gradle.kts @@ -2,6 +2,7 @@ plugins { `java-gradle-plugin` `kotlin-dsl` `jvm-test-suite` + alias(libs.plugins.shadow) } java { @@ -15,8 +16,37 @@ kotlin { } } +val jib by configurations.creating +val conflictingBuildSrc by configurations.creating +configurations.compileOnly { extendsFrom(jib) } + dependencies { - implementation("com.google.cloud.tools:jib-core:0.28.2") + jib("com.google.cloud.tools:jib-core:0.28.2") + conflictingBuildSrc("org.apache.httpcomponents:httpclient:4.3.5") +} + +// buildSrc exports an older HttpClient through the parent classloader. Keep Jib's +// dependencies private, including when this plugin is consumed as an included build. +tasks.shadowJar { + configurations = listOf(jib) + enableAutoRelocation = true + relocationPrefix = "datadog.buildlogic.testcontainers.internal" + mergeServiceFiles() +} +configurations.apiElements { + outgoing.artifacts.clear() + outgoing.artifact(tasks.shadowJar) +} +configurations.runtimeElements { + outgoing.artifacts.clear() + outgoing.artifact(tasks.shadowJar) +} +tasks.pluginUnderTestMetadata { + pluginClasspath.setFrom(tasks.shadowJar) +} +tasks.test { + inputs.files(conflictingBuildSrc) + systemProperty("test.buildSrc.classpath", conflictingBuildSrc.asPath) } gradlePlugin { diff --git a/build-logic/testcontainers/src/test/kotlin/datadog/buildlogic/testcontainers/TestcontainersPluginTest.kt b/build-logic/testcontainers/src/test/kotlin/datadog/buildlogic/testcontainers/TestcontainersPluginTest.kt index 46dfde1e7fb..7ee486630b1 100644 --- a/build-logic/testcontainers/src/test/kotlin/datadog/buildlogic/testcontainers/TestcontainersPluginTest.kt +++ b/build-logic/testcontainers/src/test/kotlin/datadog/buildlogic/testcontainers/TestcontainersPluginTest.kt @@ -7,11 +7,13 @@ import org.gradle.testkit.runner.GradleRunner import org.gradle.testkit.runner.TaskOutcome import org.junit.jupiter.api.Test import org.junit.jupiter.api.io.TempDir +import java.io.File import java.net.InetSocketAddress import java.nio.file.Files import java.nio.file.Path import java.security.KeyStore import java.security.MessageDigest +import java.util.Properties import java.util.concurrent.atomic.AtomicInteger import java.util.concurrent.atomic.AtomicReference import javax.net.ssl.KeyManagerFactory @@ -110,7 +112,7 @@ class TestcontainersPluginTest { } @Test - fun `bearer authentication resolves the registry manifest`() { + fun `bearer authentication resolves manifests with an older HttpClient in buildSrc`() { val server = registry() val tokenRequests = AtomicInteger() val authorizedRequests = AtomicInteger() @@ -140,7 +142,40 @@ class TestcontainersPluginTest { server.start() try { fixture("127.0.0.1:${server.address.port}/library/cassandra:4") - assertThat(run("test").task(":test")?.outcome).isEqualTo(TaskOutcome.SUCCESS) + // Reproduce the parent classloader supplied by Aether in the real buildSrc. + val httpClasspath = + System.getProperty("test.buildSrc.classpath").split(File.pathSeparator).joinToString(", ") { "'$it'" } + Files.createDirectories(directory.resolve("buildSrc/src/main/java")) + directory.resolve("buildSrc/src/main/java/BuildLogic.java").toFile().writeText("public class BuildLogic {}\n") + directory.resolve("buildSrc/build.gradle").toFile().writeText( + """ + plugins { id 'java' } + dependencies { implementation files($httpClasspath) } + """.trimIndent(), + ) + // Load as an included build instead of using TestKit's injected plugin classpath. + val metadata = Properties() + javaClass.classLoader.getResourceAsStream("plugin-under-test-metadata.properties")!!.use { metadata.load(it) } + val pluginClasspath = metadata.getProperty("implementation-classpath").split(File.pathSeparator).joinToString(", ") { "'$it'" } + Files.createDirectories(directory.resolve("plugin")) + directory.resolve("plugin/settings.gradle").toFile().writeText("rootProject.name = 'fixture-plugin'\n") + directory.resolve("plugin/build.gradle").toFile().writeText( + """ + plugins { id 'java-gradle-plugin' } + dependencies { implementation files($pluginClasspath) } + gradlePlugin { + plugins { + testcontainers { + id = 'dd-trace-java.testcontainers' + implementationClass = 'datadog.buildlogic.testcontainers.TestcontainersPlugin' + } + } + } + """.trimIndent(), + ) + val settings = directory.resolve("settings.gradle").toFile() + settings.writeText("pluginManagement { includeBuild('plugin') }\n" + settings.readText()) + assertThat(runner("test", injectPluginClasspath = false).build().task(":test")?.outcome).isEqualTo(TaskOutcome.SUCCESS) assertThat(tokenRequests.get()).isPositive() assertThat(authorizedRequests.get()).isPositive() assertThat(report()).contains(digest(manifest("3"))) @@ -241,19 +276,21 @@ class TestcontainersPluginTest { ) } - private fun runner(vararg arguments: String) = - GradleRunner - .create() - .withProjectDir(directory.toFile()) - .withPluginClasspath() - .withArguments( - *arguments, - "--build-cache", - "--configuration-cache", - "--stacktrace", - "--max-workers=2", - "-Dorg.gradle.jvmargs=-Xmx512m", - ) + private fun runner( + vararg arguments: String, + injectPluginClasspath: Boolean = true, + ) = GradleRunner + .create() + .withProjectDir(directory.toFile()) + .apply { if (injectPluginClasspath) withPluginClasspath() } + .withArguments( + *arguments, + "--build-cache", + "--configuration-cache", + "--stacktrace", + "--max-workers=2", + "-Dorg.gradle.jvmargs=-Xmx512m", + ) private fun run(vararg arguments: String) = runner(*arguments).build() From 71bd0f9c3cfb07be9e362c24504bbba76aeed9a1 Mon Sep 17 00:00:00 2001 From: Brice Dutheil Date: Thu, 24 Sep 2026 09:54:49 +0200 Subject: [PATCH 03/15] ci: focus image fingerprinting tests on JDK 25 --- .gitlab-ci.yml | 1315 +---------------- .../datastax-cassandra-3.0/build.gradle | 5 - .../datastax-cassandra-3.8/build.gradle | 5 - .../datastax-cassandra-4.0/build.gradle | 6 - .../instrumentation/jdbc/build.gradle | 4 +- dd-smoke-tests/websphere-jmx/build.gradle | 3 +- 6 files changed, 55 insertions(+), 1283 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bf87d739e1d..44caad57cbf 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,145 +1,23 @@ -include: - - local: ".gitlab/one-pipeline.locked.yml" - - local: ".gitlab/benchmarks.yml" - - local: ".gitlab/exploration-tests.yml" - - local: ".gitlab/ci-visibility-tests.yml" - - project: 'DataDog/apm-reliability/apm-sdks-benchmarks' - file: '.gitlab/ci-java-spring-petclinic-parallel.yml' - ref: &apm_sdks_benchmarks_sha '5e416ea523f39bbdf9e44f39e6895a07d3715ff5' # pinned by .github/workflows/update-apm-sdks-benchmarks-reference.yaml - - project: 'DataDog/apm-reliability/apm-sdks-benchmarks' - file: '.gitlab/ci-java-load-parallel.yml' - ref: *apm_sdks_benchmarks_sha - - project: 'DataDog/apm-reliability/apm-sdks-benchmarks' - file: '.gitlab/ci-java-startup-parallel.yml' - ref: *apm_sdks_benchmarks_sha - - project: 'DataDog/apm-reliability/apm-sdks-benchmarks' - file: '.gitlab/ci-java-dacapo-parallel.yml' - ref: *apm_sdks_benchmarks_sha - - project: 'DataDog/apm-reliability/apm-sdks-benchmarks' - file: '.gitlab/ci-java-post-pr-comment.yml' - ref: *apm_sdks_benchmarks_sha - - local: ".gitlab/java-benchmark-configs.yml" - -stages: - - build - - publish - - java-spring-petclinic-parallel - - java-spring-petclinic-parallel-slo - - java-startup-parallel - - java-startup-parallel-slo - - java-load-parallel - - java-load-parallel-slo - - java-dacapo-parallel - - java-dacapo-parallel-slo - - java-post-pr-comment - - shared-pipeline-build - - shared-pipeline-test - - publish-release-artifacts - - shared-pipeline-publish - - benchmarks - - tests - - tests-arm64 - - test-summary - - exploration-tests - - ci-visibility-tests - - generate-signing-key +# Temporary pipeline for the container-image fingerprinting work. +stages: [tests] variables: - APM_SDKS_BENCHMARKS_SHA: *apm_sdks_benchmarks_sha - # Test the OpenTelemetry Operator-compatible Java image jobs from one-pipeline. - CI_OTEL_OPERATOR_IMAGES_ENABLED: "true" - CI_OTEL_OPERATOR_LANGUAGE: java - # Gitlab runner features; see https://docs.gitlab.com/runner/configuration/feature-flags.html - # Fold and time all script sections - FF_SCRIPT_SECTIONS: 1 - REGISTRY: 486234852809.dkr.ecr.us-east-1.amazonaws.com - BUILD_JOB_NAME: "build" DEPENDENCY_CACHE_POLICY: pull BUILD_CACHE_POLICY: pull - GRADLE_VERSION: "9.7.1" # must match gradle-wrapper.properties + GRADLE_VERSION: "9.7.1" MASS_READ_URL: "https://mass-read.us1.ddbuild.io" MAVEN_REPOSITORY_PROXY: "https://depot-read-api-java.us1.ddbuild.io/magicmirror/magicmirror/@current/" GRADLE_PLUGIN_PROXY: "https://depot-read-api-java.us1.ddbuild.io/magicmirror/magicmirror/@current/" - BUILDER_IMAGE_REPO: "registry.ddbuild.io/images/mirror/dd-trace-java-docker-build" # images are pinned in images/mirror.lock.yaml in the DataDog/images repo - BUILDER_IMAGE_VERSION_PREFIX: "ci-" # use either an empty string (e.g. "") for latest images or a version followed by a hyphen (e.g. "ci-" or "123_merge-") + BUILDER_IMAGE_REPO: "registry.ddbuild.io/images/mirror/dd-trace-java-docker-build" + BUILDER_IMAGE_VERSION_PREFIX: "ci-" TEST_COUNTS_S3_BUCKET: "dd-trace-java-ci-test-reports" - REPO_NOTIFICATION_CHANNEL: "#apm-java-escalations" - DEFAULT_TEST_JVMS: /^(8|11|17|21|25|tip)$/ # the latest "tip" version is 26 - PROFILE_TESTS: - description: "Enable profiling of tests" - value: "false" - NON_DEFAULT_JVMS: - description: "Enable tests on JVMs that are not the default" - value: "false" - RUN_FLAKY_TESTS: - description: "Enable flaky tests" - value: "false" - - # One pipeline injection package size ratchet - OCI_PACKAGE_MAX_SIZE_BYTES: 40_000_000 - LIB_INJECTION_IMAGE_MAX_SIZE_BYTES: 40_000_000 + PROFILE_TESTS: "false" + testJvm: "25" + CI_SPLIT: "1/1" -# trigger new commit cancel workflow: auto_cancel: on_new_commit: interruptible - rules: - # skip gitlab pipeline for github merge queue - we are currently using the datadog merge queue - - if: '$CI_COMMIT_BRANCH =~ /^gh-readonly-queue\//' - when: never - - if: '$CI_COMMIT_BRANCH == "master"' - variables: - SYSTEM_TESTS_RUN_ALL_VMS: "true" - auto_cancel: - on_new_commit: none - - if: '$CI_COMMIT_BRANCH =~ /^v[0-9]+\.[0-9]+\.[0-9]+$/' - auto_cancel: - on_new_commit: none - - when: always - -.test_matrix: &test_matrix - - testJvm: &test_jvms - - "8" - - "11" - - "17" - - "21" - - "25" - - "27" # JDK 27 TODO: remove after GA (tip will move to 27) - - "semeru11" - - "oracle8" - - "zulu8" - - "semeru8" - - "ibm8" - - "zulu11" - - "semeru17" - - "tip" - CI_SPLIT: ["1/1"] - -# Gitlab doesn't support "parallel" and "parallel:matrix" at the same time -# These blocks emulate "parallel" by including it in the matrix -.test_matrix_2: &test_matrix_2 - - testJvm: *test_jvms - CI_SPLIT: ["1/2", "2/2"] - -.test_matrix_4: &test_matrix_4 - - testJvm: *test_jvms - CI_SPLIT: ["1/4", "2/4", "3/4", "4/4"] - -.test_matrix_6: &test_matrix_6 - - testJvm: *test_jvms - CI_SPLIT: ["1/6", "2/6", "3/6", "4/6", "5/6", "6/6"] - -.test_matrix_8: &test_matrix_8 - - testJvm: *test_jvms - CI_SPLIT: ["1/8", "2/8", "3/8", "4/8", "5/8", "6/8", "7/8", "8/8"] - -.test_matrix_12: &test_matrix_12 - - testJvm: *test_jvms - CI_SPLIT: [ "1/12", "2/12", "3/12", "4/12", "5/12", "6/12", "7/12", "8/12", "9/12", "10/12", "11/12", "12/12" ] - -.master_only: &master_only - - if: $CI_COMMIT_BRANCH == "master" - when: on_success default: tags: [ "arch:amd64" ] @@ -148,7 +26,6 @@ default: .set_datadog_api_keys: &set_datadog_api_keys - export DATADOG_API_KEY_PROD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.DATADOG_API_KEY_PROD --with-decryption --query "Parameter.Value" --out text) -# CI_NODE_INDEX and CI_NODE_TOTAL are 1-indexed and not always set. These steps normalize the numbers for jobs .normalize_node_index: &normalize_node_index - if [ "$CI_NO_SPLIT" == "true" ] ; then CI_NODE_INDEX=1; CI_NODE_TOTAL=1; fi # A job uses parallel but doesn't intend to split by index - if [ -n "$CI_SPLIT" ]; then CI_NODE_INDEX="${CI_SPLIT%%/*}"; CI_NODE_TOTAL="${CI_SPLIT##*/}"; fi @@ -199,15 +76,6 @@ default: KUBERNETES_MEMORY_REQUEST: 32Gi KUBERNETES_MEMORY_LIMIT: 32Gi -.gitlab_base_ref_params: &gitlab_base_ref_params - - | - export GIT_BASE_REF=$(.gitlab/find-gh-base-ref.sh) - if [[ -n "$GIT_BASE_REF" ]]; then - export GRADLE_PARAMS="$GRADLE_PARAMS -PgitBaseRef=origin/$GIT_BASE_REF" - else - echo "Failed to find base ref for PR" >&2 - fi - .gradle_build: &gradle_build image: ${BUILDER_IMAGE_REPO}:${BUILDER_IMAGE_VERSION_PREFIX}base stage: build @@ -287,474 +155,6 @@ default: - *cgroup_info - *container_info -# Check and fail early if maven central credentials are incorrect. When a new token is generated -# on the central publisher portal, it invalidates the old one. This check prevents going further. -# See https://datadoghq.atlassian.net/wiki/x/Oog5OgE -maven-central-pre-release-check: - image: ${BUILDER_IMAGE_REPO}:${BUILDER_IMAGE_VERSION_PREFIX}base - stage: .pre - rules: - - if: '$CI_COMMIT_BRANCH == "master"' - when: on_success - allow_failure: false - - if: '$CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+$/' - when: on_success - allow_failure: false - script: - - | - MAVEN_CENTRAL_USERNAME=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.central_username --with-decryption --query "Parameter.Value" --out text) - MAVEN_CENTRAL_PASSWORD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.central_password --with-decryption --query "Parameter.Value" --out text) - # See https://central.sonatype.org/publish/publish-portal-ossrh-staging-api/ - # Use the staging API search endpoint to validate the tokens without relying on a specific deployment - AUTHORIZATION_HEADER="Authorization: Bearer $(printf '%s:%s' "$MAVEN_CENTRAL_USERNAME" "$MAVEN_CENTRAL_PASSWORD" | base64)" - if ! curl --silent --show-error --fail \ - "https://ossrh-staging-api.central.sonatype.com/manual/search/repositories?ip=any" \ - --header "$AUTHORIZATION_HEADER" \ - > /dev/null; then - echo "Failed to authenticate tokens against maven central staging API. Check credentials and see https://datadoghq.atlassian.net/wiki/x/Oog5OgE" - exit 1 - fi - -dd-octo-sts-pre-release-check: - image: registry.ddbuild.io/images/dd-octo-sts-ci-base:2025.06-1 - stage: .pre - tags: [ "arch:amd64" ] - id_tokens: - DDOCTOSTS_ID_TOKEN: - aud: dd-octo-sts - rules: - - if: '$POPULATE_CACHE' - when: never - - if: '$CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+$/' - when: on_success - allow_failure: false - before_script: - - dd-octo-sts version - - dd-octo-sts debug --scope DataDog/dd-trace-java --policy self.gitlab.release - - dd-octo-sts token --scope DataDog/dd-trace-java --policy self.gitlab.release > test-github-token.txt - script: - - gh auth login --with-token < test-github-token.txt - - gh auth status - after_script: - - dd-octo-sts revoke -t $(cat test-github-token.txt) - retry: - max: 2 - when: always - -build: - needs: - - job: maven-central-pre-release-check - optional: true - - job: dd-octo-sts-pre-release-check - optional: true - extends: .gradle_build - variables: - BUILD_CACHE_POLICY: push - CACHE_TYPE: "lib" - DEPENDENCY_CACHE_POLICY: pull - script: - - if [ $CI_PIPELINE_SOURCE == "schedule" ] ; then ./gradlew resolveAndLockAll --write-locks $GRADLE_ARGS; fi - - ./gradlew --version - - ./gradlew clean :dd-java-agent:shadowJar :dd-java-agent:check :dd-trace-api:jar :dd-trace-ot:shadowJar :products:feature-flagging:feature-flagging-api:jar -PskipTests -x spotlessCheck $GRADLE_ARGS - - echo UPSTREAM_TRACER_VERSION=$(java -jar workspace/dd-java-agent/build/libs/*.jar) >> upstream.env - - echo "BUILD_JOB_NAME=$CI_JOB_NAME" >> build.env - - echo "BUILD_JOB_ID=$CI_JOB_ID" >> build.env - artifacts: - when: always - paths: - - 'workspace/dd-java-agent/build/libs/*.jar' - - 'workspace/dd-trace-api/build/libs/*.jar' - - 'workspace/dd-trace-ot/build/libs/*.jar' - - 'workspace/products/feature-flagging/feature-flagging-api/build/libs/*.jar' - - 'upstream.env' - - '.gradle/daemon/*/*.out.log' - reports: - dotenv: build.env - -build_tests: - extends: .gradle_build - variables: - <<: *tier_xl_variables - BUILD_CACHE_POLICY: push - DEPENDENCY_CACHE_POLICY: pull - parallel: - matrix: - - GRADLE_TARGET: ":baseTest" - CACHE_TYPE: "base" - - GRADLE_TARGET: ":profilingTest" - CACHE_TYPE: "profiling" - - GRADLE_TARGET: ":instrumentationTest" - CACHE_TYPE: "inst" - - GRADLE_TARGET: ":instrumentationLatestDepTest" - CACHE_TYPE: "latestdep" - - GRADLE_TARGET: ":smokeTest" - CACHE_TYPE: "smoke" - MAVEN_OPTS: "-Xms256M -Xmx1024M" - script: - - *gitlab_base_ref_params - - ./gradlew --version - - ./gradlew clean $GRADLE_TARGET $GRADLE_PARAMS -PskipTests $GRADLE_ARGS - -populate_plugin_cache: - extends: .gradle_build - cache: - - &plugin_cache - key: dependency-plugins-v1 - paths: - - .gradle-plugin-cache/caches/modules-2 - policy: pull-push - when: always - unprotect: true - rules: - - if: '$POPULATE_CACHE' - when: on_success - script: - - mkdir -p .gradle-plugin-cache - - sudo chown -R 1001:1001 .gradle-plugin-cache - - export GRADLE_USER_HOME=$(pwd)/.gradle-plugin-cache - - | - gradle_status=0 - ./gradlew help -PskipTests $GRADLE_ARGS || gradle_status=$? - if ! find .gradle-plugin-cache/caches/modules-2/files-2.1 -type f -print -quit 2>/dev/null | grep -q .; then - echo "WARNING: No plugin artifacts were cached; skipping the cache upload." - rm -rf .gradle-plugin-cache/caches/modules-2 - fi - exit "$gradle_status" - # This cache warmer is an optimization; population jobs can continue with a cold plugin cache. - allow_failure: true - -populate_dep_cache: - extends: build_tests - # Wait for build-cache producers so their caches are uploaded before this job restores them. - needs: - - job: populate_plugin_cache - - job: build - artifacts: false - - job: build_tests - artifacts: false - variables: - BUILD_CACHE_POLICY: pull - DEPENDENCY_CACHE_POLICY: push - cache: - - <<: *plugin_cache - policy: pull - - *dependency_cache - - *build_cache - rules: - - if: '$POPULATE_CACHE' - when: on_success - before_script: - - !reference [.gradle_build, before_script] - # Seed the otherwise cold writable cache, keeping the cache uploaded below self-contained. - - mkdir -p .gradle/caches - - | - if [ -d .gradle-plugin-cache/caches/modules-2 ]; then - sudo chown -R 1001:1001 .gradle-plugin-cache - cp -a .gradle-plugin-cache/caches/modules-2 .gradle/caches/ - else - echo "Plugin cache is unavailable; continuing with a cold dependency cache." - fi - parallel: - matrix: - - GRADLE_TARGET: ":dd-java-agent:shadowJar :dd-trace-api:jar :dd-trace-ot:shadowJar" - CACHE_TYPE: "lib" - - GRADLE_TARGET: ":baseTest" - CACHE_TYPE: "base" - - GRADLE_TARGET: ":profilingTest" - CACHE_TYPE: "profiling" - - GRADLE_TARGET: ":instrumentationTest" - CACHE_TYPE: "inst" - - GRADLE_TARGET: ":instrumentationLatestDepTest" - CACHE_TYPE: "latestdep" - - GRADLE_TARGET: ":smokeTest" - CACHE_TYPE: "smoke" - - GRADLE_TARGET: "spotlessCheck" - CACHE_TYPE: "spotless" - GRADLE_MEMORY_MAX: "6G" - -publish-artifacts-to-s3: - image: registry.ddbuild.io/images/mirror/amazon/aws-cli:2.4.29 - stage: publish - needs: [ build ] - script: - - source upstream.env - - export VERSION="${UPSTREAM_TRACER_VERSION%~*}" # remove ~githash from the end of version - - aws s3 cp workspace/dd-java-agent/build/libs/dd-java-agent-${VERSION}.jar s3://dd-trace-java-builds/${CI_COMMIT_REF_NAME}/dd-java-agent.jar - - aws s3 cp workspace/dd-trace-api/build/libs/dd-trace-api-${VERSION}.jar s3://dd-trace-java-builds/${CI_COMMIT_REF_NAME}/dd-trace-api.jar - - aws s3 cp workspace/dd-trace-ot/build/libs/dd-trace-ot-${VERSION}.jar s3://dd-trace-java-builds/${CI_COMMIT_REF_NAME}/dd-trace-ot.jar - - aws s3 cp workspace/products/feature-flagging/feature-flagging-api/build/libs/dd-openfeature-${VERSION}.jar s3://dd-trace-java-builds/${CI_COMMIT_REF_NAME}/dd-openfeature.jar - - aws s3 cp workspace/dd-java-agent/build/libs/dd-java-agent-${VERSION}.jar s3://dd-trace-java-builds/${CI_PIPELINE_ID}/dd-java-agent.jar - - aws s3 cp workspace/dd-trace-api/build/libs/dd-trace-api-${VERSION}.jar s3://dd-trace-java-builds/${CI_PIPELINE_ID}/dd-trace-api.jar - - aws s3 cp workspace/dd-trace-ot/build/libs/dd-trace-ot-${VERSION}.jar s3://dd-trace-java-builds/${CI_PIPELINE_ID}/dd-trace-ot.jar - - aws s3 cp workspace/products/feature-flagging/feature-flagging-api/build/libs/dd-openfeature-${VERSION}.jar s3://dd-trace-java-builds/${CI_PIPELINE_ID}/dd-openfeature.jar - - | - cat << EOF > links.json - { - "S3 Links": [ - { - "external_link": { - "label": "Public Link to dd-java-agent.jar", - "url": "https://s3.us-east-1.amazonaws.com/dd-trace-java-builds/${CI_PIPELINE_ID}/dd-java-agent.jar" - } - }, - { - "external_link": { - "label": "Public Link to dd-openfeature.jar", - "url": "https://s3.us-east-1.amazonaws.com/dd-trace-java-builds/${CI_PIPELINE_ID}/dd-openfeature.jar" - } - } - ] - } - EOF - artifacts: - reports: - annotations: - - links.json - - -spotless: - extends: .gradle_build - stage: tests - needs: [] - variables: - GRADLE_MEMORY_MAX: 6G - CACHE_TYPE: "spotless" - script: - - ./gradlew --version - # test-published-dependencies's build file needs main version file - - ./gradlew spotlessCheck writeMainVersionFile $GRADLE_ARGS - - cd test-published-dependencies && ./gradlew spotlessCheck $GRADLE_ARGS - -check-instrumentation-naming: - extends: .gradle_build - stage: tests - needs: [ ] - script: - - ./gradlew --version - - ./gradlew checkInstrumentationNaming - -config-inversion-linter: - extends: .gradle_build - stage: tests - needs: [] - script: - - ./gradlew --version - - ./gradlew checkConfigurations - -test_published_artifacts: - extends: .gradle_build - image: ${BUILDER_IMAGE_REPO}:${BUILDER_IMAGE_VERSION_PREFIX}7 # Needs Java7 for some tests - stage: tests - needs: [ build ] - variables: - CACHE_TYPE: "lib" - script: - - mvn_local_repo=$(./mvnw help:evaluate -Dexpression=settings.localRepository -q -DforceStdout) - - rm -rf "${mvn_local_repo}/com/datadoghq" - - export GPG_PRIVATE_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.signing.gpg_private_key --with-decryption --query "Parameter.Value" --out text) - - export GPG_PASSWORD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.signing.gpg_passphrase --with-decryption --query "Parameter.Value" --out text) - - export GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xms2G -Xmx2G -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp'" - - ./gradlew publishToMavenLocal $GRADLE_ARGS - - cd test-published-dependencies - - printf '\norg.gradle.console=colored\n' >> gradle.properties - - export GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xms1G -Xmx1G -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp'" - - ./gradlew --version - - ./gradlew check --info $GRADLE_ARGS - after_script: - - *cgroup_info - - source .gitlab/gitlab-utils.sh - - gitlab_section_start "collect-reports" "Collecting reports" - - .gitlab/collect_reports.sh - - gitlab_section_end "collect-reports" - artifacts: - when: always - paths: - - ./check_reports - -validate_build: - extends: .gradle_build - stage: tests - needs: [ build ] - variables: - CACHE_TYPE: "lib" - script: - # Preserve the `build` job artifacts before Gradle rebuilds and overwrites them under - # workspace/**/build/libs, so jardiff can compare the rebuilt jars against them. - - mkdir -p reference-artifacts - - cp workspace/dd-java-agent/build/libs/*.jar reference-artifacts/ - - cp workspace/dd-trace-api/build/libs/*.jar reference-artifacts/ - - cp workspace/dd-trace-ot/build/libs/*.jar reference-artifacts/ - # Scheduled builds refresh dependency locks before creating the reference artifacts. - # Refresh them here as well so the candidate uses the same dependency resolution. - - if [ $CI_PIPELINE_SOURCE == "schedule" ] ; then ./gradlew resolveAndLockAll --write-locks $GRADLE_ARGS; fi - - ./gradlew --version - # This will run the shadowJar task and exercise the build cache, allowing to identify build-cache issues - # Keep both JAR sets for direct inspection when the comparison fails. - - | - if ! ./gradlew compareToReferenceJar -PjardiffReferenceDir="$CI_PROJECT_DIR/reference-artifacts" -PskipTests $GRADLE_ARGS; then - mkdir -p check_reports/jar-comparison-artifacts/reference check_reports/jar-comparison-artifacts/candidate - cp reference-artifacts/*.jar check_reports/jar-comparison-artifacts/reference/ - cp workspace/dd-java-agent/build/libs/*.jar check_reports/jar-comparison-artifacts/candidate/ - cp workspace/dd-trace-api/build/libs/*.jar check_reports/jar-comparison-artifacts/candidate/ - cp workspace/dd-trace-ot/build/libs/*.jar check_reports/jar-comparison-artifacts/candidate/ - exit 1 - fi - after_script: - - source .gitlab/gitlab-utils.sh - - gitlab_section_start "collect-reports" "Collecting reports" - - .gitlab/collect_reports.sh --destination ./check_reports - - gitlab_section_end "collect-reports" - artifacts: - when: always - paths: - - ./check_reports - - '.gradle/daemon/*/*.out.log' - -.check_job: - extends: .gradle_build - needs: [ build ] - stage: tests - variables: - CACHE_TYPE: "lib" - script: - - *gitlab_base_ref_params - - ./gradlew --version - - ./gradlew $GRADLE_TARGET -x spotlessCheck $GRADLE_PARAMS -PskipTests -PrunBuildSrcTests -Pslot=$CI_NODE_INDEX/$CI_NODE_TOTAL $GRADLE_ARGS - after_script: - - *container_info - - *cgroup_info - - source .gitlab/gitlab-utils.sh - - gitlab_section_start "collect-reports" "Collecting reports" - - .gitlab/collect_reports.sh --destination ./check_reports --move - - .gitlab/collect_results.sh - - gitlab_section_end "collect-reports" - artifacts: - when: always - paths: - - ./check_reports - - ./results - - '.gradle/daemon/*/*.out.log' - reports: - junit: results/*.xml - retry: - max: 2 - when: - - unknown_failure - - stuck_or_timeout_failure - - runner_system_failure - - unmet_prerequisites - - scheduler_failure - - data_integrity_failure - -check_build_src: - extends: .check_job - needs: [] - variables: - GRADLE_TARGET: ":buildSrc:build" - -check_base: - extends: .check_job - variables: - GRADLE_TARGET: ":baseCheck" - -check_inst: - extends: .check_job - parallel: 4 - variables: - GRADLE_TARGET: ":instrumentationCheck" - CACHE_TYPE: "inst" - -check_smoke: - extends: .check_job - parallel: 4 - variables: - GRADLE_TARGET: ":smokeCheck" - CACHE_TYPE: "smoke" - -check_profiling: - extends: .check_job - variables: - GRADLE_TARGET: ":profilingCheck" - -check_debugger: - extends: .check_job - variables: - GRADLE_TARGET: ":debuggerCheck" - -muzzle: - extends: .gradle_build - # needs:parallel:matrix limits this job to a specific build_tests combination. - # Keep matrix vars exact and in build_tests declaration order: - # https://docs.gitlab.com/ci/yaml/#needsparallelmatrix - needs: &needs_build_tests_inst - - job: build_tests - parallel: - matrix: - - GRADLE_TARGET: ":instrumentationTest" - CACHE_TYPE: "inst" - stage: tests - rules: - - if: '$CI_COMMIT_BRANCH =~ /^mq-working-branch-/' - when: never - - if: '$CI_COMMIT_BRANCH =~ /^gh-readonly-queue/' - when: never - - when: on_success - parallel: - matrix: - - CI_SPLIT: ["1/8", "2/8", "3/8", "4/8", "5/8", "6/8", "7/8", "8/8"] - variables: - CACHE_TYPE: "inst" - script: - - export SKIP_BUILDSCAN="true" - - ./gradlew --version - - ./gradlew :runMuzzle -Pslot=$CI_NODE_INDEX/$CI_NODE_TOTAL $GRADLE_ARGS - after_script: - - *container_info - - *cgroup_info - - *set_datadog_api_keys - - source .gitlab/gitlab-utils.sh - - gitlab_section_start "collect-reports" "Collecting reports" - - .gitlab/collect_reports.sh - - .gitlab/collect_results.sh - - .gitlab/upload_ciapp.sh $CACHE_TYPE - - gitlab_section_end "collect-reports" - artifacts: - when: always - paths: - - ./reports - - ./results - - '.gradle/daemon/*/*.out.log' - reports: - junit: results/*.xml - -muzzle-dep-report: - extends: .gradle_build - needs: *needs_build_tests_inst - stage: tests - rules: - - if: '$CI_COMMIT_BRANCH =~ /^mq-working-branch-/' - when: never - - if: '$CI_COMMIT_BRANCH =~ /^gh-readonly-queue/' - when: never - - when: on_success - variables: - CACHE_TYPE: "inst" - script: - - export SKIP_BUILDSCAN="true" - - ./gradlew --version - - ./gradlew generateMuzzleReport muzzleInstrumentationReport $GRADLE_ARGS - after_script: - - *container_info - - *cgroup_info - - .gitlab/collect_muzzle_deps.sh - artifacts: - when: always - paths: - - ./reports - - '.gradle/daemon/*/*.out.log' - -# In Gitlab, DD_* variables are set because the build runner is instrumented with Datadog telemetry -# To have a pristine environment for the tests, these variables are saved before the test run and restored afterwards .prepare_test_env: &prepare_test_env - export gitlabVariables=("DD_SERVICE" "DD_ENTITY_ID" "DD_SITE" "DD_ENV" "DD_DATACENTER" "DD_PARTITION" "DD_CLOUDPROVIDER") - '[ ! -e pretest.env ] || rm pretest.env' @@ -781,7 +181,6 @@ muzzle-dep-report: TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX: "registry.ddbuild.io/images/mirror/" JETTY_AVAILABLE_PROCESSORS: 4 # Jetty incorrectly calculates processor count in containers script: - - *gitlab_base_ref_params - > if [ "$PROFILE_TESTS" == "true" ] && [ "$testJvm" != "ibm8" ] && [ "$testJvm" != "oracle8" ]; then @@ -833,82 +232,6 @@ muzzle-dep-report: - scheduler_failure - data_integrity_failure -.test_job_amd64: - extends: .test_job_common - tags: [ "docker-in-docker:amd64" ] # use docker-in-docker runner for testcontainers - needs: [ build_tests ] - stage: tests - rules: - # Protected branches (master/mq/gh-readonly): all JVMs run unconditionally - - if: '$CI_COMMIT_BRANCH == "master"' - when: on_success - - if: '$CI_COMMIT_BRANCH =~ /^mq-working-branch-/' - when: on_success - - if: '$CI_COMMIT_BRANCH =~ /^gh-readonly-queue/' - when: on_success - # Enable for default test JVMs or for NON_DEFAULT_JVMS - - if: '$NON_DEFAULT_JVMS == "true"' - when: on_success - - if: '$CI_COMMIT_MESSAGE =~ /\[ci: NON_DEFAULT_JVMS\]/' - when: on_success - - if: '$testJvm =~ $DEFAULT_TEST_JVMS' - when: on_success - -.test_job_arm64: - extends: .test_job_common - tags: [ "docker-in-docker:arm64" ] - stage: tests-arm64 - # Use the amd64 build only as a compilation gate. Do not download its platform-specific artifacts. - needs: - - job: build - artifacts: false - variables: - DEFAULT_TEST_JVMS: /^(8|11|17|21|25|27|tip)$/ # Java 27 TODO: remove 27 after GA (tip will move to 27) - TEST_JVM_ARGS: "-Xshare:off" - rules: - # IBM 8 has no arm64 image published upstream. - - if: '$testJvm == "ibm8"' - when: never - # Oracle 8 is available on arm64 but too flaky to run in CI. - - if: '$testJvm == "oracle8"' - when: never - # arm64 tests are newly introduced to the merge queue and master. Keep them - # non-blocking (allow_failure) for now so we can collect stability stats and - # fix flaky/failing jobs without blocking the whole team. Remove allow_failure - # once the arm64 suite is proven stable. - - if: '$CI_COMMIT_BRANCH =~ /^mq-working-branch-/' - when: on_success - allow_failure: true - - if: '$CI_COMMIT_BRANCH == "master"' - when: on_success - allow_failure: true - # Enable non-default JVMs on demand. - - if: '$NON_DEFAULT_JVMS == "true"' - when: on_success - allow_failure: true - - if: '$CI_COMMIT_MESSAGE =~ /\[ci: NON_DEFAULT_JVMS\]/' - when: on_success - allow_failure: true - # Keep the default JVM subset available for manual runs on feature branches/PRs. - - if: '$testJvm =~ $DEFAULT_TEST_JVMS' - when: manual - allow_failure: true - cache: - - key: dependency-$CACHE_TYPE - paths: - - .gradle/wrapper - - .gradle/caches - - .gradle/notifications - - .mvn/caches - policy: pull - fallback_keys: - - dependency-base - - dependency-lib - unprotect: true - before_script: - - git config --global --add safe.directory "$CI_PROJECT_DIR" - - !reference [.gradle_build, before_script] - .test_job_with_test_agent_common: variables: CI_USE_TEST_AGENT: "true" @@ -927,590 +250,52 @@ muzzle-dep-report: - !reference [.test_job_common, script] - .gitlab/check_test_agent_results.sh -.test_job_amd64_with_test_agent: - extends: - - .test_job_amd64 - - .test_job_with_test_agent_common - -.test_job_arm64_with_test_agent: - extends: - - .test_job_arm64 - - .test_job_with_test_agent_common - -agent_integration_tests: - extends: .test_job_amd64 - tags: [ "arch:amd64" ] - variables: - testJvm: "8" - CI_AGENT_HOST: local-agent - GRADLE_TARGET: "traceAgentTest" - CACHE_TYPE: "base" - services: - - name: registry.ddbuild.io/images/mirror/datadog/agent:7.40.1 - alias: local-agent - variables: - DD_APM_ENABLED: "true" - DD_BIND_HOST: "0.0.0.0" - DD_HOSTNAME: "local-agent" - DD_API_KEY: "invalid_key_but_this_is_fine" - -test_base: - extends: .test_job_amd64 - # needs:parallel:matrix limits this job to a specific build_tests combination. - # Keep matrix vars exact and in build_tests declaration order: - # https://docs.gitlab.com/ci/yaml/#needsparallelmatrix - needs: - - job: build_tests - parallel: - matrix: - - GRADLE_TARGET: ":baseTest" - CACHE_TYPE: "base" - variables: - GRADLE_TARGET: ":baseTest" - CACHE_TYPE: "base" - parallel: - matrix: *test_matrix_4 - script: - - if [ "$testJvm" == "8" ]; then export GRADLE_PARAMS="-PskipFlakyTests -PcheckCoverage"; fi - - !reference [.test_job_common, script] - -test_base_arm64: - extends: .test_job_arm64 - variables: - GRADLE_TARGET: ":baseTest" - CACHE_TYPE: "base" - parallel: - matrix: *test_matrix_4 - # run coverage only on JVM 8, mirroring the amd64 test_base job - script: - - if [ "$testJvm" == "8" ]; then export GRADLE_PARAMS="$GRADLE_PARAMS -PcheckCoverage"; fi - - !reference [.test_job_common, script] - -test_inst: - extends: .test_job_amd64_with_test_agent - needs: *needs_build_tests_inst - variables: - <<: *tier_l_variables - GRADLE_TARGET: ":instrumentationTest" - CACHE_TYPE: "inst" - parallel: - matrix: *test_matrix_8 - -test_inst_arm64: - extends: .test_job_arm64_with_test_agent - variables: - <<: *tier_l_variables - GRADLE_TARGET: ":instrumentationTest" - CACHE_TYPE: "inst" - parallel: - matrix: *test_matrix_8 - -test_inst_latest: - extends: .test_job_amd64_with_test_agent - # needs:parallel:matrix limits this job to a specific build_tests combination. - # Keep matrix vars exact and in build_tests declaration order: - # https://docs.gitlab.com/ci/yaml/#needsparallelmatrix - needs: - - job: build_tests - parallel: - matrix: - - GRADLE_TARGET: ":instrumentationLatestDepTest" - CACHE_TYPE: "latestdep" - variables: - <<: *tier_l_variables - GRADLE_TARGET: ":instrumentationLatestDepTest" - CACHE_TYPE: "latestdep" - parallel: - matrix: - - testJvm: ["8", "17", "21", "25", "27", "tip"] # Java 27 TODO: remove 27 after GA (tip will move to 27) - # Gitlab doesn't support "parallel" and "parallel:matrix" at the same time - # This emulates "parallel" by including it in the matrix - CI_SPLIT: [ "1/6", "2/6", "3/6", "4/6", "5/6", "6/6"] - -test_inst_latest_arm64: - extends: .test_job_arm64_with_test_agent - variables: - <<: *tier_l_variables - GRADLE_TARGET: ":instrumentationLatestDepTest" - CACHE_TYPE: "latestdep" - parallel: - matrix: - - testJvm: ["8", "17", "21", "25", "27", "tip"] # Java 27 TODO: remove 27 after GA (tip will move to 27) - # Gitlab doesn't support "parallel" and "parallel:matrix" at the same time - # This emulates "parallel" by including it in the matrix - CI_SPLIT: [ "1/6", "2/6", "3/6", "4/6", "5/6", "6/6"] - -test_flaky: - extends: .test_job_amd64_with_test_agent - variables: - GRADLE_PARAMS: "-PrunFlakyTests" - CACHE_TYPE: "smoke" - testJvm: "8" - CONTINUE_ON_FAILURE: "true" - rules: - - *master_only - - if: $RUN_FLAKY_TESTS == "true" - when: on_success - parallel: - matrix: - - GRADLE_TARGET: [":baseTest", ":smokeTest", ":debuggerTest"] - # Gitlab doesn't support "parallel" and "parallel:matrix" at the same time - # This emulates "parallel" by including it in the matrix - CI_SPLIT: [ "1/4", "2/4", "3/4", "4/4" ] - -test_flaky_inst: - extends: .test_job_amd64 - needs: *needs_build_tests_inst - variables: - GRADLE_TARGET: ":instrumentationTest" - GRADLE_PARAMS: "-PrunFlakyTests" - CACHE_TYPE: "inst" - testJvm: "8" - CONTINUE_ON_FAILURE: "true" - rules: - - *master_only - - if: $RUN_FLAKY_TESTS == "true" - when: on_success - parallel: 6 - -test_profiling: - extends: .test_job_amd64 - # needs:parallel:matrix limits this job to a specific build_tests combination. - # Keep matrix vars exact and in build_tests declaration order: - # https://docs.gitlab.com/ci/yaml/#needsparallelmatrix - needs: - - job: build_tests - parallel: - matrix: - - GRADLE_TARGET: ":profilingTest" - CACHE_TYPE: "profiling" - variables: - GRADLE_TARGET: ":profilingTest" - CACHE_TYPE: "profiling" - parallel: - matrix: *test_matrix - -test_profiling_arm64: - extends: .test_job_arm64 - variables: - GRADLE_TARGET: ":profilingTest" - CACHE_TYPE: "profiling" - parallel: - matrix: *test_matrix - -# specific jvms list for debugger project because J9-based JVMs have issues with local vars -# so need to test at least against one J9-based JVM -test_debugger: - extends: .test_job_amd64 - variables: - GRADLE_TARGET: ":debuggerTest" - CACHE_TYPE: "base" - DEFAULT_TEST_JVMS: /^(8|11|17|21|25|27|tip|semeru8)$/ # Java 27 TODO: remove 27 after GA (tip will move to 27) - parallel: - matrix: *test_matrix - # avoid running coverage for semeru8, semeru11, semeru17 and ibm8 as some tests are disabled and therefore cannot reach the - # exepected coverage - script: - - if [[ "$testJvm" != "semeru8" && "$testJvm" != "semeru11" && "$testJvm" != "semeru17" && "$testJvm" != "ibm8" ]]; then export GRADLE_PARAMS="$GRADLE_PARAMS -PcheckCoverage"; fi - - !reference [.test_job_common, script] - -test_debugger_arm64: - extends: .test_job_arm64 - variables: - GRADLE_TARGET: ":debuggerTest" - CACHE_TYPE: "base" - parallel: - matrix: *test_matrix - # avoid running coverage for semeru8, semeru11 and semeru17 as some tests are disabled and therefore cannot reach the - # expected coverage (ibm8/oracle8 never run on arm64) - script: - - if [[ "$testJvm" != "semeru8" && "$testJvm" != "semeru11" && "$testJvm" != "semeru17" ]]; then export GRADLE_PARAMS="$GRADLE_PARAMS -PcheckCoverage"; fi - - !reference [.test_job_common, script] - -test_smoke: - extends: .test_job_amd64_with_test_agent - # needs:parallel:matrix limits this job to a specific build_tests combination. - # Keep matrix vars exact and in build_tests declaration order: - # https://docs.gitlab.com/ci/yaml/#needsparallelmatrix - needs: &needs_build_tests_smoke - - job: build_tests - parallel: - matrix: - - GRADLE_TARGET: ":smokeTest" - CACHE_TYPE: "smoke" - MAVEN_OPTS: "-Xms256M -Xmx1024M" - variables: - <<: *tier_l_variables - GRADLE_TARGET: "stageMainDist :smokeTest" - GRADLE_PARAMS: "-PskipFlakyTests" - CACHE_TYPE: "smoke" - parallel: - matrix: *test_matrix_8 - -test_smoke_arm64: - extends: .test_job_arm64_with_test_agent - variables: - <<: *tier_l_variables - GRADLE_TARGET: "stageMainDist :smokeTest" - GRADLE_PARAMS: "-PskipFlakyTests" - CACHE_TYPE: "smoke" - parallel: - matrix: *test_matrix_8 - -test_ssi_smoke: - extends: .test_job_amd64 - needs: *needs_build_tests_smoke - rules: - - if: $CI_COMMIT_BRANCH == "master" - when: on_success - - if: '$CI_COMMIT_BRANCH =~ /^mq-working-branch-/' - when: on_success - - if: '$CI_COMMIT_BRANCH =~ /^gh-readonly-queue/' - when: on_success - variables: - <<: *tier_l_variables - GRADLE_TARGET: "stageMainDist :smokeTest" - CACHE_TYPE: "smoke" - DD_INJECT_FORCE: "true" - DD_INJECTION_ENABLED: "tracer" - parallel: - matrix: *test_matrix_8 - -test_ssi_smoke_arm64: - extends: .test_job_arm64 - variables: - <<: *tier_l_variables - GRADLE_TARGET: "stageMainDist :smokeTest" - CACHE_TYPE: "smoke" - DD_INJECT_FORCE: "true" - DD_INJECTION_ENABLED: "tracer" - parallel: - matrix: *test_matrix_8 - -test_smoke_graalvm: - extends: .test_job_amd64 - needs: *needs_build_tests_smoke - tags: [ "arch:amd64" ] - variables: - <<: *tier_l_variables - GRADLE_TARGET: "stageMainDist :dd-smoke-test:spring-boot-3.0-native:test :dd-smoke-test:quarkus-native:test" - CACHE_TYPE: "smoke" - CI_NO_SPLIT: "true" - NON_DEFAULT_JVMS: "true" - parallel: - matrix: - - testJvm: ["graalvm17", "graalvm21", "graalvm25"] - -test_smoke_graalvm_arm64: - extends: .test_job_arm64 - tags: [ "arch:arm64" ] - variables: - <<: *tier_l_variables - GRADLE_TARGET: "stageMainDist :dd-smoke-test:spring-boot-3.0-native:test :dd-smoke-test:quarkus-native:test" - CACHE_TYPE: "smoke" - CI_NO_SPLIT: "true" - NON_DEFAULT_JVMS: "true" - parallel: - matrix: - - testJvm: ["graalvm17", "graalvm21", "graalvm25"] - -test_smoke_semeru8_debugger: - extends: .test_job_amd64 - needs: *needs_build_tests_smoke - tags: [ "arch:amd64" ] - variables: - GRADLE_TARGET: "stageMainDist dd-smoke-tests:debugger-integration-tests:test" - CACHE_TYPE: "smoke" - NON_DEFAULT_JVMS: "true" - testJvm: "semeru8" - -aggregate_test_counts: - image: ${BUILDER_IMAGE_REPO}:${BUILDER_IMAGE_VERSION_PREFIX}base - stage: test-summary - # Keep stage ordering, but prevent GitLab from downloading all previous-stage artifacts. - dependencies: [] - rules: - - if: '$POPULATE_CACHE' - when: never - - if: '$CI_COMMIT_BRANCH == "master"' - when: on_success - - if: '$CI_COMMIT_BRANCH =~ /^mq-working-branch-/' - when: on_success - - if: '$CI_COMMIT_BRANCH =~ /^gh-readonly-queue/' - when: on_success - - if: '$CI_COMMIT_BRANCH' - when: on_success - script: - - *set_datadog_api_keys - - export TEST_COUNTS_S3_PREFIX="test-counts/${CI_PIPELINE_ID}" - - mkdir -p ./test_counts_aggregate - - echo "Downloading test count files from s3://${TEST_COUNTS_S3_BUCKET}/${TEST_COUNTS_S3_PREFIX}/" - - aws s3 cp "s3://${TEST_COUNTS_S3_BUCKET}/${TEST_COUNTS_S3_PREFIX}/" ./test_counts_aggregate/ --recursive --exclude "*" --include "test_counts_*.json" --only-show-errors - - find ./test_counts_aggregate -name 'test_counts_*.json' -type f -print | sort - - .gitlab/aggregate_test_counts.sh ./test_counts_aggregate - artifacts: - when: always - paths: - - test_counts_aggregate/test_counts_*.json - - test_counts_summary.json - - test_counts_report.md - -deploy_to_profiling_backend: - stage: publish - needs: [ build ] - rules: - - if: '$POPULATE_CACHE' - when: never - - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' - when: on_success - - if: '$CI_COMMIT_TAG =~ /^v.*/' - when: on_success - - when: manual - allow_failure: true - trigger: - project: DataDog/profiling-backend - branch: dogfooding - variables: - UPSTREAM_PACKAGE_JOB: $BUILD_JOB_NAME - UPSTREAM_PACKAGE_JOB_ID: $BUILD_JOB_ID - UPSTREAM_PROJECT_ID: $CI_PROJECT_ID - UPSTREAM_PROJECT_NAME: $CI_PROJECT_NAME - UPSTREAM_PIPELINE_ID: $CI_PIPELINE_ID - UPSTREAM_BRANCH: $CI_COMMIT_BRANCH - UPSTREAM_TAG: $CI_COMMIT_TAG - -deploy_to_di_backend:manual: - stage: publish - needs: [ build ] - rules: - - if: '$POPULATE_CACHE' - when: never - - when: manual - allow_failure: true - trigger: - project: DataDog/debugger-demos - branch: main - variables: - UPSTREAM_PACKAGE_JOB: build - UPSTREAM_PROJECT_ID: $CI_PROJECT_ID - UPSTREAM_PROJECT_NAME: $CI_PROJECT_NAME - UPSTREAM_PIPELINE_ID: $CI_PIPELINE_ID - UPSTREAM_BRANCH: $CI_COMMIT_BRANCH - UPSTREAM_TAG: $CI_COMMIT_TAG - UPSTREAM_COMMIT_AUTHOR: $CI_COMMIT_AUTHOR - UPSTREAM_COMMIT_SHORT_SHA: $CI_COMMIT_SHORT_SHA - -deploy_to_reliability_env: - needs: [ build ] - -# If the deploy_to_maven_central job is re-run, re-trigger the deploy_artifacts_to_github job as well so that the artifacts match. -deploy_to_maven_central: - extends: .gradle_build - stage: publish-release-artifacts - needs: - - job: build - - job: system_tests - optional: true - variables: - CACHE_TYPE: "lib" - rules: - - if: '$POPULATE_CACHE' - when: never - - if: '$CI_COMMIT_BRANCH == "master"' - when: on_success - # Do not deploy release candidate versions - - if: '$CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+$/' - when: on_success - allow_failure: true - - when: manual - allow_failure: true - script: - - export MAVEN_CENTRAL_USERNAME=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.central_username --with-decryption --query "Parameter.Value" --out text) - - export MAVEN_CENTRAL_PASSWORD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.central_password --with-decryption --query "Parameter.Value" --out text) - - export GPG_PRIVATE_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.signing.gpg_private_key --with-decryption --query "Parameter.Value" --out text) - - export GPG_PASSWORD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.signing.gpg_passphrase --with-decryption --query "Parameter.Value" --out text) - - ./gradlew publishToSonatype closeSonatypeStagingRepository -PskipTests $GRADLE_ARGS - artifacts: - paths: - - 'workspace/dd-java-agent/build/libs/*.jar' - - 'workspace/dd-trace-api/build/libs/*.jar' - - 'workspace/dd-trace-ot/build/libs/*.jar' - -deploy_snapshot_with_ddprof_snapshot: +test_image_plugin: extends: .gradle_build - stage: publish - needs: [ build ] - variables: - CACHE_TYPE: "lib" - rules: - - if: '$POPULATE_CACHE' - when: never - # Manual trigger only - for testing with ddprof snapshot versions - - when: manual - allow_failure: true + stage: tests script: - - export MAVEN_CENTRAL_USERNAME=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.central_username --with-decryption --query "Parameter.Value" --out text) - - export MAVEN_CENTRAL_PASSWORD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.central_password --with-decryption --query "Parameter.Value" --out text) - - export GPG_PRIVATE_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.signing.gpg_private_key --with-decryption --query "Parameter.Value" --out text) - - export GPG_PASSWORD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.signing.gpg_passphrase --with-decryption --query "Parameter.Value" --out text) - - echo "Publishing dd-trace-java snapshot with ddprof snapshot dependency" - - ./gradlew -PbuildInfo.build.number=$CI_JOB_ID -PddprofUseSnapshot publishToSonatype -PskipTests $GRADLE_ARGS + - ./gradlew -p build-logic :testcontainers:test :testcontainers:validatePlugins $GRADLE_ARGS artifacts: - paths: - - 'workspace/dd-java-agent/build/libs/*.jar' - - 'workspace/dd-trace-api/build/libs/*.jar' - - 'workspace/dd-trace-ot/build/libs/*.jar' - -deploy_artifacts_to_github: - stage: publish-release-artifacts - image: registry.ddbuild.io/images/dd-octo-sts-ci-base:2025.06-1 - tags: [ "arch:amd64" ] - id_tokens: - DDOCTOSTS_ID_TOKEN: - aud: dd-octo-sts - rules: - - if: '$POPULATE_CACHE' - when: never - - if: '$CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+$/' - when: on_success - # Requires the deploy_to_maven_central job to have run first (the UP-TO-DATE gradle check across jobs is broken) - # This will deploy the artifacts built from the publishToSonatype task to the GitHub release - needs: - - job: deploy_to_maven_central - # The deploy_to_maven_central job is not run for release candidate versions - optional: true - before_script: - - dd-octo-sts version - - dd-octo-sts debug --scope DataDog/dd-trace-java --policy self.gitlab.release - - dd-octo-sts token --scope DataDog/dd-trace-java --policy self.gitlab.release > github-token.txt - script: - - gh auth login --with-token < github-token.txt - - gh auth status - - export VERSION=${CI_COMMIT_TAG##v} # remove "v" from front of tag to get version - - cp workspace/dd-java-agent/build/libs/dd-java-agent-${VERSION}.jar workspace/dd-java-agent/build/libs/dd-java-agent.jar # upload two filenames - - gh release upload --clobber --repo DataDog/dd-trace-java $CI_COMMIT_TAG workspace/dd-java-agent/build/libs/dd-java-agent.jar - - gh release upload --clobber --repo DataDog/dd-trace-java $CI_COMMIT_TAG workspace/dd-java-agent/build/libs/dd-java-agent-${VERSION}.jar - - gh release upload --clobber --repo DataDog/dd-trace-java $CI_COMMIT_TAG workspace/dd-trace-api/build/libs/dd-trace-api-${VERSION}.jar - - gh release upload --clobber --repo DataDog/dd-trace-java $CI_COMMIT_TAG workspace/dd-trace-ot/build/libs/dd-trace-ot-${VERSION}.jar - after_script: - - dd-octo-sts revoke -t $(cat github-token.txt) - retry: - max: 2 when: always - -requirements_json_test: - rules: - - when: on_success - variables: - REQUIREMENTS_BLOCK_JSON_PATH: "metadata/requirements-block.json" - REQUIREMENTS_ALLOW_JSON_PATH: "metadata/requirements-allow.json" - -package-oci: - needs: [ build ] - -override_verify_maven_central: - image: registry.ddbuild.io/images/base/gbi-ubuntu_2204:release - stage: publish - needs: [ ] - rules: - - if: '$POPULATE_CACHE' - when: never - - when: manual - allow_failure: true - script: - - touch OVERRIDE_MAVEN_VERIFY - cache: # Cache is used to signal between the override_verify_maven_central and verify_maven_central_deployment jobs - - key: $CI_PIPELINE_ID-OVERRIDE_SIGNAL - paths: - - OVERRIDE_MAVEN_VERIFY - policy: push - unprotect: true - -# Verify Maven Central deployment is publicly available before publishing OCI images -verify_maven_central_deployment: - image: registry.ddbuild.io/images/base/gbi-ubuntu_2204:release - stage: publish-release-artifacts - needs: [ deploy_to_maven_central ] - rules: - - if: '$POPULATE_CACHE' - when: never - - if: '$CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+$/' - when: on_success - cache: # Cache is used to signal between the override_verify_maven_central and verify_maven_central_deployment jobs - - key: $CI_PIPELINE_ID-OVERRIDE_SIGNAL - paths: - - OVERRIDE_MAVEN_VERIFY - policy: pull - unprotect: true - script: - - if [ -f OVERRIDE_MAVEN_VERIFY ]; then echo "SKIPPING MAVEN VERIFICATION"; exit 0; fi - - | - export VERSION=${CI_COMMIT_TAG##v} - ARTIFACT_URLS=( - "https://repo1.maven.org/maven2/com/datadoghq/dd-java-agent/${VERSION}/dd-java-agent-${VERSION}.jar" - "https://repo1.maven.org/maven2/com/datadoghq/dd-trace-api/${VERSION}/dd-trace-api-${VERSION}.jar" - "https://repo1.maven.org/maven2/com/datadoghq/dd-trace-ot/${VERSION}/dd-trace-ot-${VERSION}.jar" - ) - # Try once immediately (fast path on job retry), then every 5 mins for 45 mins. - TRY=0 - MAX_TRIES=10 - RETRY_DELAY=300 - while [ $TRY -lt $MAX_TRIES ]; do - ARTIFACTS_AVAILABLE=true - for URL in "${ARTIFACT_URLS[@]}"; do - if ! curl --location --fail --silent --show-error -I "$URL"; then - ARTIFACTS_AVAILABLE=false - break - fi - done - if [ "$ARTIFACTS_AVAILABLE" = true ]; then - break - fi - TRY=$((TRY + 1)) - if [ $TRY -eq $MAX_TRIES ]; then - echo "The release was not available after 45 mins. Manually re-run the job to try again." - exit 1 - fi - sleep $RETRY_DELAY - done - -publishing-gate: - stage: publish-release-artifacts - needs: - - job: verify_maven_central_deployment - optional: true # Required for releases only - -configure_system_tests: - variables: - SYSTEM_TESTS_REF: "main" # system tests are pinned on release branches only - SYSTEM_TESTS_SCENARIOS_GROUPS: "simple_onboarding,simple_onboarding_profiling,simple_onboarding_appsec,docker-ssi,lib-injection" - -create_key: - stage: generate-signing-key - when: manual - needs: [ ] - variables: - PROJECT_NAME: "dd-trace-java" - EXPORT_TO_KEYSERVER: "true" - image: $REGISTRY/ci/agent-key-management-tools/gpg:1 - script: - - /create.sh - artifacts: - expire_in: 13 mos paths: - - pubkeys - -validate_supported_configurations_v2_local_file: - extends: .validate_supported_configurations_v2_local_file - variables: - LOCAL_JSON_PATH: "metadata/supported-configurations.json" - BACKFILLED: "false" + - build-logic/testcontainers/build/reports/tests/ + reports: + junit: build-logic/testcontainers/build/test-results/test/*.xml -update_central_configurations_version_range_v2: - extends: .update_central_configurations_version_range_v2 +test_container_images: + extends: + - .test_job_common + - .test_job_with_test_agent_common + stage: tests + tags: ["docker-in-docker:amd64"] variables: - LOCAL_REPO_NAME: "dd-trace-java" - LOCAL_JSON_PATH: "metadata/supported-configurations.json" - LANGUAGE_NAME: "java" - MULTIPLE_RELEASE_LINES: "false" - SEND_ALIASES: "true" + CACHE_TYPE: "inst" + parallel: + matrix: + - GRADLE_TARGET: + - ":dd-java-agent:instrumentation:datastax-cassandra:datastax-cassandra-3.0:forkedTest" + - ":dd-java-agent:instrumentation:datastax-cassandra:datastax-cassandra-3.0:latestDepTest" + - ":dd-java-agent:instrumentation:datastax-cassandra:datastax-cassandra-3.0:test" + - ":dd-java-agent:instrumentation:datastax-cassandra:datastax-cassandra-3.8:forkedTest" + - ":dd-java-agent:instrumentation:datastax-cassandra:datastax-cassandra-3.8:latestDepTest" + - ":dd-java-agent:instrumentation:datastax-cassandra:datastax-cassandra-3.8:test" + - ":dd-java-agent:instrumentation:datastax-cassandra:datastax-cassandra-4.0:forkedTest" + - ":dd-java-agent:instrumentation:datastax-cassandra:datastax-cassandra-4.0:latestDepTest" + - ":dd-java-agent:instrumentation:datastax-cassandra:datastax-cassandra-4.0:test" + - ":dd-java-agent:instrumentation:google-pubsub-1.116:forkedTest" + - ":dd-java-agent:instrumentation:google-pubsub-1.116:latestDepForkedTest" + - ":dd-java-agent:instrumentation:google-pubsub-1.116:latestDepTest" + - ":dd-java-agent:instrumentation:google-pubsub-1.116:test" + - ":dd-java-agent:instrumentation:jdbc:forkedTest" + - ":dd-java-agent:instrumentation:jdbc:latestDepJava11Test" + - ":dd-java-agent:instrumentation:jdbc:latestDepTest" + - ":dd-java-agent:instrumentation:jdbc:oldH2Test" + - ":dd-java-agent:instrumentation:jdbc:oldPostgresTest" + - ":dd-java-agent:instrumentation:jdbc:test" + - ":dd-java-agent:instrumentation:vertx:vertx-mysql-client:vertx-mysql-client-3.9:forkedTest" + - ":dd-java-agent:instrumentation:vertx:vertx-mysql-client:vertx-mysql-client-3.9:latestDepForkedTest" + - ":dd-java-agent:instrumentation:vertx:vertx-mysql-client:vertx-mysql-client-4.0:forkedTest" + - ":dd-java-agent:instrumentation:vertx:vertx-mysql-client:vertx-mysql-client-4.0:latestDepForkedTest" + - ":dd-java-agent:instrumentation:vertx:vertx-pg-client:vertx-pg-client-4.0:forkedTest" + - ":dd-java-agent:instrumentation:vertx:vertx-pg-client:vertx-pg-client-4.0:latestDepForkedTest" + - ":dd-smoke-tests:websphere-jmx:test" diff --git a/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-3.0/build.gradle b/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-3.0/build.gradle index 9718e557981..72ff7d5d7e8 100644 --- a/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-3.0/build.gradle +++ b/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-3.0/build.gradle @@ -43,11 +43,6 @@ muzzle { } } -testJvmConstraints { - // Test use Cassandra 3 which requires Java 8. (Currently incompatible with Java 9.) - maxJavaVersion = JavaVersion.VERSION_1_8 -} - addTestSuiteForDir('latestDepTest', 'test') dependencies { diff --git a/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-3.8/build.gradle b/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-3.8/build.gradle index 5e74728b352..2f34b57f836 100644 --- a/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-3.8/build.gradle +++ b/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-3.8/build.gradle @@ -19,11 +19,6 @@ muzzle { } } -testJvmConstraints { - // Test use Cassandra 3 which requires Java 8. (Currently incompatible with Java 9.) - maxJavaVersion = JavaVersion.VERSION_1_8 -} - addTestSuiteForDir('latestDepTest', 'test') dependencies { diff --git a/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-4.0/build.gradle b/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-4.0/build.gradle index 5a87a224cd0..681c2ebb304 100644 --- a/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-4.0/build.gradle +++ b/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-4.0/build.gradle @@ -12,12 +12,6 @@ muzzle { } } -testJvmConstraints { - // TODO Java 17: The embedded cassandra deadlocks on start every time on Java 17 - // This can be changed to use test-containers - maxJavaVersion = JavaVersion.VERSION_15 -} - addTestSuiteForDir('latestDepTest', 'test') dependencies { diff --git a/dd-java-agent/instrumentation/jdbc/build.gradle b/dd-java-agent/instrumentation/jdbc/build.gradle index 722c6e60386..dc2622fdec5 100644 --- a/dd-java-agent/instrumentation/jdbc/build.gradle +++ b/dd-java-agent/instrumentation/jdbc/build.gradle @@ -90,7 +90,9 @@ tasks.named("check") { } tasks.named("latestDepJava11Test", Test) { - javaLauncher = getJavaLauncherFor(11) + if (!providers.gradleProperty('testJvm').isPresent()) { + javaLauncher = getJavaLauncherFor(11) + } } tasks.withType(GroovyCompile).configureEach { diff --git a/dd-smoke-tests/websphere-jmx/build.gradle b/dd-smoke-tests/websphere-jmx/build.gradle index b3ee93ac840..4ba5b2d2c58 100644 --- a/dd-smoke-tests/websphere-jmx/build.gradle +++ b/dd-smoke-tests/websphere-jmx/build.gradle @@ -12,7 +12,8 @@ dependencies { testJvmConstraints { // there is no need to run it multiple times since it runs on a container - maxJavaVersion = JavaVersion.VERSION_1_8 + minJavaVersion = JavaVersion.VERSION_25 + maxJavaVersion = JavaVersion.VERSION_25 } tasks.withType(Test).configureEach { From f8aab6a200aff4ce6c243db51e5aa89c7f4b081c Mon Sep 17 00:00:00 2001 From: Brice Dutheil Date: Thu, 24 Sep 2026 10:01:02 +0200 Subject: [PATCH 04/15] fix: configure standalone image plugin validation --- .gitlab-ci.yml | 2 +- build-logic/testcontainers/build.gradle.kts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 44caad57cbf..4d45afe03ad 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -254,7 +254,7 @@ test_image_plugin: extends: .gradle_build stage: tests script: - - ./gradlew -p build-logic :testcontainers:test :testcontainers:validatePlugins $GRADLE_ARGS + - JAVA_HOME=$JAVA_25_HOME ./gradlew -p build-logic :testcontainers:test :testcontainers:validatePlugins $GRADLE_ARGS artifacts: when: always paths: diff --git a/build-logic/testcontainers/build.gradle.kts b/build-logic/testcontainers/build.gradle.kts index 9668da207b4..e168f939a2a 100644 --- a/build-logic/testcontainers/build.gradle.kts +++ b/build-logic/testcontainers/build.gradle.kts @@ -16,8 +16,8 @@ kotlin { } } -val jib by configurations.creating -val conflictingBuildSrc by configurations.creating +val jib = configurations.create("jib") +val conflictingBuildSrc = configurations.create("conflictingBuildSrc") configurations.compileOnly { extendsFrom(jib) } dependencies { From b483d99883d0100539acb01726bb1a362a1b4f0c Mon Sep 17 00:00:00 2001 From: Brice Dutheil Date: Thu, 24 Sep 2026 10:56:40 +0200 Subject: [PATCH 05/15] fix: wire container inputs and limits lazily --- build-logic/testcontainers/README.md | 18 ++++ build-logic/testcontainers/build.gradle.kts | 4 + .../testcontainers/ContainerImageArguments.kt | 20 +++- .../TestcontainersLimitService.kt | 29 ++++++ .../testcontainers/TestcontainersPlugin.kt | 67 +++++++------ .../TestcontainersPluginTest.kt | 98 +++++++++++++++++-- build.gradle.kts | 1 + .../google-pubsub-1.116/build.gradle | 4 - .../instrumentation/jdbc/build.gradle | 4 - .../vertx-mysql-client-3.9/build.gradle | 4 - .../vertx-mysql-client-4.0/build.gradle | 4 - .../vertx-pg-client-4.0/build.gradle | 4 - dd-smoke-tests/websphere-jmx/build.gradle | 4 - gradle/java_no_deps.gradle | 12 +-- 14 files changed, 194 insertions(+), 79 deletions(-) create mode 100644 build-logic/testcontainers/src/main/kotlin/datadog/buildlogic/testcontainers/TestcontainersLimitService.kt diff --git a/build-logic/testcontainers/README.md b/build-logic/testcontainers/README.md index 0f729bdc431..a46d4d1b0c2 100644 --- a/build-logic/testcontainers/README.md +++ b/build-logic/testcontainers/README.md @@ -27,6 +27,24 @@ and `ForkedTest` companions receive the properties. A separate suite declare its own images with, for example, `integrationTestContainerImage(...)`. Run IDE tests through Gradle, or supply the named image properties explicitly. +Tasks with declared or inherited images also consume the shared `testcontainersLimit` +service. `testcontainersMaxParallelUsages` controls the build-wide quota (default: 2). +The legacy convention registers the same service for modules that still declare +`usesService(testcontainersLimit)` explicitly. Unrelated suites do not consume it. +The limit applies to whole test tasks, not individual test methods or containers. + +Configuration uses `configureEach` and a project-local provider rather than an +evaluation callback. Late declarations, configuration inheritance, resource directories +and task environment settings are captured when Gradle queries the provider. +Only that configuration is cached; registry resolution remains an execution-time input. +Isolated Projects has not been tested. + +The service references are declared on nested task inputs with `@ServiceReference`. +CI selectors running during configuration should check +`ContainerImageArguments.containers.isPresent` in the task's `jvmArgumentProviders`. +Gradle's internal `requiredServices.isServiceRequired` may not yet include these +inferred references; retain its existing check for legacy explicit service users. + The plugin resolves each effective tag once per build, when Gradle snapshots test inputs. An annotated JVM argument provider includes property names and immutable `registry/repository@sha256:...` values in the test fingerprint, then passes those diff --git a/build-logic/testcontainers/build.gradle.kts b/build-logic/testcontainers/build.gradle.kts index e168f939a2a..5add2447be6 100644 --- a/build-logic/testcontainers/build.gradle.kts +++ b/build-logic/testcontainers/build.gradle.kts @@ -51,6 +51,10 @@ tasks.test { gradlePlugin { plugins { + create("testcontainers-limit") { + id = "dd-trace-java.testcontainers-limit" + implementationClass = "datadog.buildlogic.testcontainers.TestcontainersLimitPlugin" + } create("testcontainers") { id = "dd-trace-java.testcontainers" implementationClass = "datadog.buildlogic.testcontainers.TestcontainersPlugin" diff --git a/build-logic/testcontainers/src/main/kotlin/datadog/buildlogic/testcontainers/ContainerImageArguments.kt b/build-logic/testcontainers/src/main/kotlin/datadog/buildlogic/testcontainers/ContainerImageArguments.kt index 0da3ad13134..62b00e0c117 100644 --- a/build-logic/testcontainers/src/main/kotlin/datadog/buildlogic/testcontainers/ContainerImageArguments.kt +++ b/build-logic/testcontainers/src/main/kotlin/datadog/buildlogic/testcontainers/ContainerImageArguments.kt @@ -1,22 +1,34 @@ package datadog.buildlogic.testcontainers import org.gradle.api.provider.Provider +import org.gradle.api.services.ServiceReference import org.gradle.api.tasks.Input import org.gradle.api.tasks.Internal +import org.gradle.api.tasks.Nested +import org.gradle.api.tasks.Optional import org.gradle.process.CommandLineArgumentProvider import java.io.File class ContainerImageArguments( + @get:Nested @get:Optional val containers: Provider, +) : CommandLineArgumentProvider { + override fun asArguments(): Iterable = + containers.orNull + ?.images + ?.map { (name, image) -> "-D$name=$image" } + .orEmpty() +} + +class ContainerImageInputs( @get:Internal val declarations: Map, @get:Internal val imageEnvironment: Map, @get:Internal val configurationFiles: List, - @get:Internal val resolver: Provider, -) : CommandLineArgumentProvider { + @get:ServiceReference("testContainerImageResolver") val resolver: Provider, + @get:ServiceReference("testcontainersLimit") val limit: Provider, +) { // Read during Test input snapshotting, after skip predicates and before cache lookup. // Only the service reference is serialized by the configuration cache; its memo is per build. @get:Input val images: Map get() = resolver.get().resolve(declarations, imageEnvironment, configurationFiles) - - override fun asArguments(): Iterable = images.map { (name, image) -> "-D$name=$image" } } diff --git a/build-logic/testcontainers/src/main/kotlin/datadog/buildlogic/testcontainers/TestcontainersLimitService.kt b/build-logic/testcontainers/src/main/kotlin/datadog/buildlogic/testcontainers/TestcontainersLimitService.kt new file mode 100644 index 00000000000..7b700a5fb1c --- /dev/null +++ b/build-logic/testcontainers/src/main/kotlin/datadog/buildlogic/testcontainers/TestcontainersLimitService.kt @@ -0,0 +1,29 @@ +package datadog.buildlogic.testcontainers + +import org.gradle.api.Plugin +import org.gradle.api.Project +import org.gradle.api.provider.Provider +import org.gradle.api.services.BuildService +import org.gradle.api.services.BuildServiceParameters + +/** Limits container test tasks across projects, including consumers of the legacy convention. */ +abstract class TestcontainersLimitService : BuildService { + companion object { + fun register(project: Project): Provider = + project.gradle.sharedServices.registerIfAbsent("testcontainersLimit", TestcontainersLimitService::class.java) { + maxParallelUsages.set( + project.providers + .gradleProperty("testcontainersMaxParallelUsages") + .map(String::toInt) + .orElse(2), + ) + } + } +} + +/** Keeps the legacy usesService(testcontainersLimit) DSL backed by the plugin's shared quota. */ +class TestcontainersLimitPlugin : Plugin { + override fun apply(project: Project) { + project.extensions.extraProperties.set("testcontainersLimit", TestcontainersLimitService.register(project)) + } +} diff --git a/build-logic/testcontainers/src/main/kotlin/datadog/buildlogic/testcontainers/TestcontainersPlugin.kt b/build-logic/testcontainers/src/main/kotlin/datadog/buildlogic/testcontainers/TestcontainersPlugin.kt index f72545ad9c9..b39c7752f26 100644 --- a/build-logic/testcontainers/src/main/kotlin/datadog/buildlogic/testcontainers/TestcontainersPlugin.kt +++ b/build-logic/testcontainers/src/main/kotlin/datadog/buildlogic/testcontainers/TestcontainersPlugin.kt @@ -36,6 +36,7 @@ class TestcontainersPlugin : Plugin { "testContainerImageResolver", ImageResolver::class.java, ) {} + val limit = TestcontainersLimitService.register(project) project.pluginManager.withPlugin("java") { val sourceSets = project.extensions.getByType() @@ -52,40 +53,44 @@ class TestcontainersPlugin : Plugin { }, ) } - // Suite inheritance is configured by build scripts after the plugin is applied. - project.afterEvaluate { - project.tasks.withType().configureEach { - val suiteName = if (name == "forkedTest") "test" else name.removeSuffix("ForkedTest") - val suite = - sourceSets.findByName(name) ?: sourceSets.findByName(suiteName) - ?: return@configureEach - val hierarchy = project.configurations.getByName(suite.implementationConfigurationName).hierarchy - val images = linkedMapOf() - val inheritedSourceSets = - sourceSets.filter { sourceSet -> - hierarchy.any { it.name == sourceSet.implementationConfigurationName } - } - inheritedSourceSets.forEach { sourceSet -> - declarations[sourceSet.name]?.forEach { (property, reference) -> - require(images.putIfAbsent(property, reference).let { it == null || it == reference }) { - "Conflicting container images for '$property' in $path" + project.tasks.withType().configureEach { + // ProviderFactory.provider snapshots project-local configuration for the configuration cache. + // Keep registry access in the nested input getter, so every build refreshes moving tags. + val containers = + project.providers.provider { + val suiteName = if (name == "forkedTest") "test" else name.removeSuffix("ForkedTest") + val suite = + sourceSets.findByName(name) ?: sourceSets.findByName(suiteName) + ?: return@provider null + val hierarchy = project.configurations.getByName(suite.implementationConfigurationName).hierarchy + val images = linkedMapOf() + val inheritedSourceSets = + sourceSets.filter { sourceSet -> + hierarchy.any { it.name == sourceSet.implementationConfigurationName } + } + inheritedSourceSets.forEach { sourceSet -> + declarations[sourceSet.name]?.forEach { (property, reference) -> + require(images.putIfAbsent(property, reference).let { it == null || it == reference }) { + "Conflicting container images for '$property' in $path" + } } } + if (images.isNotEmpty()) { + val configurationFiles = + listOf(File(System.getProperty("user.home"), ".testcontainers.properties")) + + inheritedSourceSets.flatMap { it.resources.srcDirs }.map { File(it, "testcontainers.properties") } + val imageEnvironment = + (project.providers.environmentVariablesPrefixedBy("TESTCONTAINERS_").get() + environment) + .filterKeys { + it == "TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX" || it == "TESTCONTAINERS_IMAGE_SUBSTITUTOR" || + (it.startsWith("TESTCONTAINERS_") && it.endsWith("_CONTAINER_IMAGE")) + }.mapValues { it.value.toString() } + ContainerImageInputs(images, imageEnvironment, configurationFiles, resolver, limit) + } else { + null + } } - if (images.isNotEmpty()) { - usesService(resolver) - val configurationFiles = - listOf(File(System.getProperty("user.home"), ".testcontainers.properties")) + - inheritedSourceSets.flatMap { it.resources.srcDirs }.map { File(it, "testcontainers.properties") } - val imageEnvironment = - (project.providers.environmentVariablesPrefixedBy("TESTCONTAINERS_").get() + environment) - .filterKeys { - it == "TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX" || it == "TESTCONTAINERS_IMAGE_SUBSTITUTOR" || - (it.startsWith("TESTCONTAINERS_") && it.endsWith("_CONTAINER_IMAGE")) - }.mapValues { it.value.toString() } - jvmArgumentProviders.add(ContainerImageArguments(images, imageEnvironment, configurationFiles, resolver)) - } - } + jvmArgumentProviders.add(ContainerImageArguments(containers)) } } } diff --git a/build-logic/testcontainers/src/test/kotlin/datadog/buildlogic/testcontainers/TestcontainersPluginTest.kt b/build-logic/testcontainers/src/test/kotlin/datadog/buildlogic/testcontainers/TestcontainersPluginTest.kt index 7ee486630b1..91388d24446 100644 --- a/build-logic/testcontainers/src/test/kotlin/datadog/buildlogic/testcontainers/TestcontainersPluginTest.kt +++ b/build-logic/testcontainers/src/test/kotlin/datadog/buildlogic/testcontainers/TestcontainersPluginTest.kt @@ -40,6 +40,7 @@ class TestcontainersPluginTest { try { fixture("127.0.0.1:${server.address.port}/library/cassandra:4") assertThat(run("help").task(":help")?.outcome).isEqualTo(TaskOutcome.SUCCESS) + assertThat(run("verifyServiceSelection").task(":verifyServiceSelection")?.outcome).isEqualTo(TaskOutcome.SUCCESS) assertThat(requests.get()).isZero() assertThat(run("test", "-PskipTests").task(":test")?.outcome).isEqualTo(TaskOutcome.SKIPPED) assertThat(requests.get()).isZero() @@ -102,15 +103,78 @@ class TestcontainersPluginTest { assertThat(changed.task(":test")?.outcome).isEqualTo(TaskOutcome.SUCCESS) assertThat(report()).contains("another.example/team/cassandra@$digest") - Files.createDirectories(directory.resolve("src/test/resources")) + directory.resolve("build.gradle").toFile().appendText( + """ + + sourceSets.test.resources.srcDirs = ['lateResources'] + tasks.named('test') { + environment 'TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX', 'task.example/' + } + """.trimIndent(), + ) + assertThat(run("test").task(":test")?.outcome).isEqualTo(TaskOutcome.SUCCESS) + assertThat(report()).contains("task.example/cassandra@$digest") + Files.createDirectories(directory.resolve("lateResources")) directory - .resolve("src/test/resources/testcontainers.properties") + .resolve("lateResources/testcontainers.properties") .toFile() .writeText("image.substitutor=untracked.CustomSubstitutor\n") assertThat(runner("test").buildAndFail().output) .contains("move image overrides into testContainerImage declarations") } + @Test + fun `container tasks share the legacy limit across projects and configuration cache reuse`() { + fixture("cassandra@sha256:${"a".repeat(64)}") + directory.resolve("settings.gradle").toFile().appendText("\ninclude('other')\n") + directory.resolve("gradle.properties").toFile().writeText("testcontainersMaxParallelUsages=1\n") + val buildFile = directory.resolve("build.gradle").toFile() + buildFile.appendText( + """ + + tasks.withType(Test).configureEach { + systemProperty('containerLock', '${directory.resolve("container.lock")}') + } + """.trimIndent(), + ) + val testFile = directory.resolve("src/test/java/ImageTest.java").toFile() + testFile.writeText( + """ + import org.junit.jupiter.api.Test; + import static org.junit.jupiter.api.Assertions.assertTrue; + public class ImageTest { + @Test public void respectsContainerLimit() throws Exception { + java.io.File lock = new java.io.File(System.getProperty("containerLock")); + assertTrue(lock.createNewFile(), "Container tasks exceeded the shared limit"); + try { Thread.sleep(1000); } finally { lock.delete(); } + } + } + """.trimIndent(), + ) + val other = directory.resolve("other") + Files.createDirectories(other.resolve("src/test/java")) + other.resolve("src/test/java/ImageTest.java").toFile().writeText(testFile.readText()) + // A legacy module still declares usesService explicitly and has no image declarations. + other.resolve("build.gradle").toFile().writeText( + buildFile + .readText() + .replace("id 'dd-trace-java.testcontainers'", "id 'dd-trace-java.testcontainers-limit'") + .replace("testContainerImage(image('cassandra@sha256:${"a".repeat(64)}', 'test.cassandra.image'))", "") + + """ + + tasks.named('test', Test) { usesService(testcontainersLimit) } + """.trimIndent(), + ) + val arguments = arrayOf(":test", ":other:test", "--parallel", "--rerun-tasks", "--no-build-cache") + val first = run(*arguments) + assertThat(first.task(":test")?.outcome).isEqualTo(TaskOutcome.SUCCESS) + assertThat(first.task(":other:test")?.outcome).isEqualTo(TaskOutcome.SUCCESS) + val reused = run(*arguments) + assertThat(reused.output).contains("Reusing configuration cache") + assertThat(reused.task(":test")?.outcome).isEqualTo(TaskOutcome.SUCCESS) + assertThat(reused.task(":other:test")?.outcome).isEqualTo(TaskOutcome.SUCCESS) + } + @Test fun `bearer authentication resolves manifests with an older HttpClient in buildSrc`() { val server = registry() @@ -218,13 +282,6 @@ class TestcontainersPluginTest { isolatedTest emptyTest } - configurations.latestDepTestImplementation.extendsFrom(configurations.testImplementation) - configurations.emptyTestImplementation.extendsFrom(configurations.testImplementation) - dependencies { - testImplementation files($junitClasspath) - isolatedTestImplementation files($junitClasspath) - testContainerImage(image('$image', 'test.cassandra.image')) - } tasks.register('latestDepTest', Test) { testClassesDirs = sourceSets.latestDepTest.output.classesDirs classpath = sourceSets.latestDepTest.runtimeClasspath @@ -246,6 +303,29 @@ class TestcontainersPluginTest { useJUnitPlatform() onlyIf { !skip.isPresent() } } + // Plugins must see declarations and inheritance added after tasks are realized. + tasks.named('test').get() + tasks.named('latestDepTest').get() + tasks.named('latestDepTestForkedTest').get() + configurations.latestDepTestImplementation.extendsFrom(configurations.testImplementation) + configurations.emptyTestImplementation.extendsFrom(configurations.testImplementation) + dependencies { + testImplementation files($junitClasspath) + isolatedTestImplementation files($junitClasspath) + testContainerImage(image('$image', 'test.cassandra.image')) + } + tasks.register('verifyServiceSelection') { + def selected = providers.provider { + ['test', 'latestDepTest', 'latestDepTestForkedTest', 'isolatedTest'].collectEntries { name -> + def task = tasks.named(name).get() + [(name): task.requiredServices.searchServices().any { it.name == 'testcontainersLimit' }] + } + } + inputs.property('selected', selected) + doLast { + assert inputs.properties.selected == [test: true, latestDepTest: true, latestDepTestForkedTest: true, isolatedTest: false] + } + } """.trimIndent(), ) Files.createDirectories(directory.resolve("src/test/java")) diff --git a/build.gradle.kts b/build.gradle.kts index 5f07b849668..f516b38c6f8 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -12,6 +12,7 @@ plugins { id("dd-trace-java.dump-hanged-test") id("dd-trace-java.gradle-debug") id("dd-trace-java.tracer-version") + id("dd-trace-java.testcontainers-limit") apply false alias(libs.plugins.shadow) apply false alias(libs.plugins.spotless) diff --git a/dd-java-agent/instrumentation/google-pubsub-1.116/build.gradle b/dd-java-agent/instrumentation/google-pubsub-1.116/build.gradle index a2b03630339..99f539589cc 100644 --- a/dd-java-agent/instrumentation/google-pubsub-1.116/build.gradle +++ b/dd-java-agent/instrumentation/google-pubsub-1.116/build.gradle @@ -27,7 +27,3 @@ dependencies { testImplementation project(":dd-java-agent:instrumentation:gax-1.4") latestDepTestImplementation group: 'com.google.cloud', name: 'google-cloud-pubsub', version: '+' } - -tasks.withType(Test).configureEach { - usesService(testcontainersLimit) -} diff --git a/dd-java-agent/instrumentation/jdbc/build.gradle b/dd-java-agent/instrumentation/jdbc/build.gradle index dc2622fdec5..f38c5760d93 100644 --- a/dd-java-agent/instrumentation/jdbc/build.gradle +++ b/dd-java-agent/instrumentation/jdbc/build.gradle @@ -104,10 +104,6 @@ tasks.withType(GroovyCompile).configureEach { ) } -tasks.withType(Test).configureEach { - usesService(testcontainersLimit) -} - tasks.named("latestDepJava11Test", Test) { testJvmConstraints { minJavaVersion = JavaVersion.VERSION_11 diff --git a/dd-java-agent/instrumentation/vertx/vertx-mysql-client/vertx-mysql-client-3.9/build.gradle b/dd-java-agent/instrumentation/vertx/vertx-mysql-client/vertx-mysql-client-3.9/build.gradle index 1f8423384f0..c1ec5a069c5 100644 --- a/dd-java-agent/instrumentation/vertx/vertx-mysql-client/vertx-mysql-client-3.9/build.gradle +++ b/dd-java-agent/instrumentation/vertx/vertx-mysql-client/vertx-mysql-client-3.9/build.gradle @@ -34,7 +34,3 @@ dependencies { latestDepTestImplementation group: 'io.vertx', name: 'vertx-mysql-client', version: '3.+' } - -tasks.withType(Test).configureEach { - usesService(testcontainersLimit) -} diff --git a/dd-java-agent/instrumentation/vertx/vertx-mysql-client/vertx-mysql-client-4.0/build.gradle b/dd-java-agent/instrumentation/vertx/vertx-mysql-client/vertx-mysql-client-4.0/build.gradle index c314173c643..727a607812a 100644 --- a/dd-java-agent/instrumentation/vertx/vertx-mysql-client/vertx-mysql-client-4.0/build.gradle +++ b/dd-java-agent/instrumentation/vertx/vertx-mysql-client/vertx-mysql-client-4.0/build.gradle @@ -35,7 +35,3 @@ dependencies { latestDepTestImplementation group: 'io.vertx', name: 'vertx-mysql-client', version: '4.+' } - -tasks.withType(Test).configureEach { - usesService(testcontainersLimit) -} diff --git a/dd-java-agent/instrumentation/vertx/vertx-pg-client/vertx-pg-client-4.0/build.gradle b/dd-java-agent/instrumentation/vertx/vertx-pg-client/vertx-pg-client-4.0/build.gradle index 0db168b53ad..a2d811abc10 100644 --- a/dd-java-agent/instrumentation/vertx/vertx-pg-client/vertx-pg-client-4.0/build.gradle +++ b/dd-java-agent/instrumentation/vertx/vertx-pg-client/vertx-pg-client-4.0/build.gradle @@ -55,7 +55,3 @@ configurations.named("latestDepTestRuntimeClasspath") { configurations.named("latestDepForkedTestRuntimeClasspath") { exclude group: 'com.ongres.scram', module: 'client' } - -tasks.withType(Test).configureEach { - usesService(testcontainersLimit) -} diff --git a/dd-smoke-tests/websphere-jmx/build.gradle b/dd-smoke-tests/websphere-jmx/build.gradle index 4ba5b2d2c58..ecf21ab32bb 100644 --- a/dd-smoke-tests/websphere-jmx/build.gradle +++ b/dd-smoke-tests/websphere-jmx/build.gradle @@ -15,7 +15,3 @@ testJvmConstraints { minJavaVersion = JavaVersion.VERSION_25 maxJavaVersion = JavaVersion.VERSION_25 } - -tasks.withType(Test).configureEach { - usesService(testcontainersLimit) -} diff --git a/gradle/java_no_deps.gradle b/gradle/java_no_deps.gradle index 46f77882407..288e3f0ebcc 100644 --- a/gradle/java_no_deps.gradle +++ b/gradle/java_no_deps.gradle @@ -1,8 +1,6 @@ import datadog.gradle.plugin.testJvmConstraints.TestJvmConstraintsExtension import datadog.gradle.plugin.testJvmConstraints.ProvideJvmArgsOnJvmLauncherVersion import groovy.transform.CompileStatic -import org.gradle.api.services.BuildService -import org.gradle.api.services.BuildServiceParameters import java.nio.file.Files import java.nio.file.LinkOption @@ -140,15 +138,7 @@ class TracerJavaExtension { } } def tracerJavaExtension = extensions.create(TracerJavaExtension.NAME, TracerJavaExtension, objects, providers, project) - - - -abstract class TestcontainersLimitService implements BuildService { -} - -ext.testcontainersLimit = gradle.sharedServices.registerIfAbsent("testcontainersLimit", TestcontainersLimitService) { - maxParallelUsages = project.findProperty("testcontainersMaxParallelUsages").toInteger() -} +apply plugin: 'dd-trace-java.testcontainers-limit' // Every project gets a forked test task for its default 'test' source set. addForkedTestTask('test') From 4ed465ca2d4bf6185a05190dd455fdd581501bce Mon Sep 17 00:00:00 2001 From: Brice Dutheil Date: Thu, 24 Sep 2026 11:31:55 +0200 Subject: [PATCH 06/15] fix: keep container limits separate from image inputs --- build-logic/testcontainers/README.md | 14 +--- build-logic/testcontainers/build.gradle.kts | 4 -- .../testcontainers/ContainerImageArguments.kt | 1 - .../TestcontainersLimitService.kt | 29 -------- .../testcontainers/TestcontainersPlugin.kt | 8 ++- .../TestcontainersPluginTest.kt | 72 +++++-------------- build.gradle.kts | 1 - .../google-pubsub-1.116/build.gradle | 4 ++ .../instrumentation/jdbc/build.gradle | 4 ++ .../vertx-mysql-client-3.9/build.gradle | 4 ++ .../vertx-mysql-client-4.0/build.gradle | 4 ++ .../vertx-pg-client-4.0/build.gradle | 4 ++ dd-smoke-tests/websphere-jmx/build.gradle | 4 ++ docs/how_to_smoke_test.md | 40 +++++++++++ docs/how_to_test.md | 48 +++++++++++++ gradle/java_no_deps.gradle | 12 +++- 16 files changed, 148 insertions(+), 105 deletions(-) delete mode 100644 build-logic/testcontainers/src/main/kotlin/datadog/buildlogic/testcontainers/TestcontainersLimitService.kt diff --git a/build-logic/testcontainers/README.md b/build-logic/testcontainers/README.md index a46d4d1b0c2..f0d6213f59f 100644 --- a/build-logic/testcontainers/README.md +++ b/build-logic/testcontainers/README.md @@ -27,11 +27,9 @@ and `ForkedTest` companions receive the properties. A separate suite declare its own images with, for example, `integrationTestContainerImage(...)`. Run IDE tests through Gradle, or supply the named image properties explicitly. -Tasks with declared or inherited images also consume the shared `testcontainersLimit` -service. `testcontainersMaxParallelUsages` controls the build-wide quota (default: 2). -The legacy convention registers the same service for modules that still declare -`usesService(testcontainersLimit)` explicitly. Unrelated suites do not consume it. -The limit applies to whole test tasks, not individual test methods or containers. +This plugin only fingerprints images. Container concurrency remains controlled by +the existing, explicit `usesService(testcontainersLimit)` declarations in module +build scripts; applying this plugin does not add or remove them. Configuration uses `configureEach` and a project-local provider rather than an evaluation callback. Late declarations, configuration inheritance, resource directories @@ -39,12 +37,6 @@ and task environment settings are captured when Gradle queries the provider. Only that configuration is cached; registry resolution remains an execution-time input. Isolated Projects has not been tested. -The service references are declared on nested task inputs with `@ServiceReference`. -CI selectors running during configuration should check -`ContainerImageArguments.containers.isPresent` in the task's `jvmArgumentProviders`. -Gradle's internal `requiredServices.isServiceRequired` may not yet include these -inferred references; retain its existing check for legacy explicit service users. - The plugin resolves each effective tag once per build, when Gradle snapshots test inputs. An annotated JVM argument provider includes property names and immutable `registry/repository@sha256:...` values in the test fingerprint, then passes those diff --git a/build-logic/testcontainers/build.gradle.kts b/build-logic/testcontainers/build.gradle.kts index 5add2447be6..e168f939a2a 100644 --- a/build-logic/testcontainers/build.gradle.kts +++ b/build-logic/testcontainers/build.gradle.kts @@ -51,10 +51,6 @@ tasks.test { gradlePlugin { plugins { - create("testcontainers-limit") { - id = "dd-trace-java.testcontainers-limit" - implementationClass = "datadog.buildlogic.testcontainers.TestcontainersLimitPlugin" - } create("testcontainers") { id = "dd-trace-java.testcontainers" implementationClass = "datadog.buildlogic.testcontainers.TestcontainersPlugin" diff --git a/build-logic/testcontainers/src/main/kotlin/datadog/buildlogic/testcontainers/ContainerImageArguments.kt b/build-logic/testcontainers/src/main/kotlin/datadog/buildlogic/testcontainers/ContainerImageArguments.kt index 62b00e0c117..44351ad7003 100644 --- a/build-logic/testcontainers/src/main/kotlin/datadog/buildlogic/testcontainers/ContainerImageArguments.kt +++ b/build-logic/testcontainers/src/main/kotlin/datadog/buildlogic/testcontainers/ContainerImageArguments.kt @@ -24,7 +24,6 @@ class ContainerImageInputs( @get:Internal val imageEnvironment: Map, @get:Internal val configurationFiles: List, @get:ServiceReference("testContainerImageResolver") val resolver: Provider, - @get:ServiceReference("testcontainersLimit") val limit: Provider, ) { // Read during Test input snapshotting, after skip predicates and before cache lookup. // Only the service reference is serialized by the configuration cache; its memo is per build. diff --git a/build-logic/testcontainers/src/main/kotlin/datadog/buildlogic/testcontainers/TestcontainersLimitService.kt b/build-logic/testcontainers/src/main/kotlin/datadog/buildlogic/testcontainers/TestcontainersLimitService.kt deleted file mode 100644 index 7b700a5fb1c..00000000000 --- a/build-logic/testcontainers/src/main/kotlin/datadog/buildlogic/testcontainers/TestcontainersLimitService.kt +++ /dev/null @@ -1,29 +0,0 @@ -package datadog.buildlogic.testcontainers - -import org.gradle.api.Plugin -import org.gradle.api.Project -import org.gradle.api.provider.Provider -import org.gradle.api.services.BuildService -import org.gradle.api.services.BuildServiceParameters - -/** Limits container test tasks across projects, including consumers of the legacy convention. */ -abstract class TestcontainersLimitService : BuildService { - companion object { - fun register(project: Project): Provider = - project.gradle.sharedServices.registerIfAbsent("testcontainersLimit", TestcontainersLimitService::class.java) { - maxParallelUsages.set( - project.providers - .gradleProperty("testcontainersMaxParallelUsages") - .map(String::toInt) - .orElse(2), - ) - } - } -} - -/** Keeps the legacy usesService(testcontainersLimit) DSL backed by the plugin's shared quota. */ -class TestcontainersLimitPlugin : Plugin { - override fun apply(project: Project) { - project.extensions.extraProperties.set("testcontainersLimit", TestcontainersLimitService.register(project)) - } -} diff --git a/build-logic/testcontainers/src/main/kotlin/datadog/buildlogic/testcontainers/TestcontainersPlugin.kt b/build-logic/testcontainers/src/main/kotlin/datadog/buildlogic/testcontainers/TestcontainersPlugin.kt index b39c7752f26..4c7807a67ef 100644 --- a/build-logic/testcontainers/src/main/kotlin/datadog/buildlogic/testcontainers/TestcontainersPlugin.kt +++ b/build-logic/testcontainers/src/main/kotlin/datadog/buildlogic/testcontainers/TestcontainersPlugin.kt @@ -36,7 +36,6 @@ class TestcontainersPlugin : Plugin { "testContainerImageResolver", ImageResolver::class.java, ) {} - val limit = TestcontainersLimitService.register(project) project.pluginManager.withPlugin("java") { val sourceSets = project.extensions.getByType() @@ -79,13 +78,16 @@ class TestcontainersPlugin : Plugin { val configurationFiles = listOf(File(System.getProperty("user.home"), ".testcontainers.properties")) + inheritedSourceSets.flatMap { it.resources.srcDirs }.map { File(it, "testcontainers.properties") } + // Track inherited values for configuration-cache invalidation, but use the task's + // effective environment below so explicit overrides and removals are respected. + project.providers.environmentVariablesPrefixedBy("TESTCONTAINERS_").get() val imageEnvironment = - (project.providers.environmentVariablesPrefixedBy("TESTCONTAINERS_").get() + environment) + environment .filterKeys { it == "TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX" || it == "TESTCONTAINERS_IMAGE_SUBSTITUTOR" || (it.startsWith("TESTCONTAINERS_") && it.endsWith("_CONTAINER_IMAGE")) }.mapValues { it.value.toString() } - ContainerImageInputs(images, imageEnvironment, configurationFiles, resolver, limit) + ContainerImageInputs(images, imageEnvironment, configurationFiles, resolver) } else { null } diff --git a/build-logic/testcontainers/src/test/kotlin/datadog/buildlogic/testcontainers/TestcontainersPluginTest.kt b/build-logic/testcontainers/src/test/kotlin/datadog/buildlogic/testcontainers/TestcontainersPluginTest.kt index 91388d24446..187483a11af 100644 --- a/build-logic/testcontainers/src/test/kotlin/datadog/buildlogic/testcontainers/TestcontainersPluginTest.kt +++ b/build-logic/testcontainers/src/test/kotlin/datadog/buildlogic/testcontainers/TestcontainersPluginTest.kt @@ -40,7 +40,6 @@ class TestcontainersPluginTest { try { fixture("127.0.0.1:${server.address.port}/library/cassandra:4") assertThat(run("help").task(":help")?.outcome).isEqualTo(TaskOutcome.SUCCESS) - assertThat(run("verifyServiceSelection").task(":verifyServiceSelection")?.outcome).isEqualTo(TaskOutcome.SUCCESS) assertThat(requests.get()).isZero() assertThat(run("test", "-PskipTests").task(":test")?.outcome).isEqualTo(TaskOutcome.SKIPPED) assertThat(requests.get()).isZero() @@ -124,55 +123,30 @@ class TestcontainersPluginTest { } @Test - fun `container tasks share the legacy limit across projects and configuration cache reuse`() { - fixture("cassandra@sha256:${"a".repeat(64)}") - directory.resolve("settings.gradle").toFile().appendText("\ninclude('other')\n") - directory.resolve("gradle.properties").toFile().writeText("testcontainersMaxParallelUsages=1\n") - val buildFile = directory.resolve("build.gradle").toFile() - buildFile.appendText( + fun `task environment removals override inherited Testcontainers settings`() { + val digest = "sha256:${"a".repeat(64)}" + fixture("cassandra@$digest") + directory.resolve("build.gradle").toFile().appendText( """ - tasks.withType(Test).configureEach { - systemProperty('containerLock', '${directory.resolve("container.lock")}') - } - """.trimIndent(), - ) - val testFile = directory.resolve("src/test/java/ImageTest.java").toFile() - testFile.writeText( - """ - import org.junit.jupiter.api.Test; - import static org.junit.jupiter.api.Assertions.assertTrue; - public class ImageTest { - @Test public void respectsContainerLimit() throws Exception { - java.io.File lock = new java.io.File(System.getProperty("containerLock")); - assertTrue(lock.createNewFile(), "Container tasks exceeded the shared limit"); - try { Thread.sleep(1000); } finally { lock.delete(); } - } + tasks.named('test') { + environment.remove('TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX') + environment.remove('TESTCONTAINERS_IMAGE_SUBSTITUTOR') } """.trimIndent(), ) - val other = directory.resolve("other") - Files.createDirectories(other.resolve("src/test/java")) - other.resolve("src/test/java/ImageTest.java").toFile().writeText(testFile.readText()) - // A legacy module still declares usesService explicitly and has no image declarations. - other.resolve("build.gradle").toFile().writeText( - buildFile - .readText() - .replace("id 'dd-trace-java.testcontainers'", "id 'dd-trace-java.testcontainers-limit'") - .replace("testContainerImage(image('cassandra@sha256:${"a".repeat(64)}', 'test.cassandra.image'))", "") + - """ - - tasks.named('test', Test) { usesService(testcontainersLimit) } - """.trimIndent(), - ) - val arguments = arrayOf(":test", ":other:test", "--parallel", "--rerun-tasks", "--no-build-cache") - val first = run(*arguments) + val environment = + System.getenv() + + mapOf( + "TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX" to "removed.example/team/", + "TESTCONTAINERS_IMAGE_SUBSTITUTOR" to "removed.CustomSubstitutor", + ) + val first = runner("test").withEnvironment(environment).build() assertThat(first.task(":test")?.outcome).isEqualTo(TaskOutcome.SUCCESS) - assertThat(first.task(":other:test")?.outcome).isEqualTo(TaskOutcome.SUCCESS) - val reused = run(*arguments) + assertThat(report()).contains("registry-1.docker.io/library/cassandra@$digest") + val reused = runner("test").withEnvironment(environment).build() assertThat(reused.output).contains("Reusing configuration cache") - assertThat(reused.task(":test")?.outcome).isEqualTo(TaskOutcome.SUCCESS) - assertThat(reused.task(":other:test")?.outcome).isEqualTo(TaskOutcome.SUCCESS) + assertThat(reused.task(":test")?.outcome).isEqualTo(TaskOutcome.UP_TO_DATE) } @Test @@ -314,18 +288,6 @@ class TestcontainersPluginTest { isolatedTestImplementation files($junitClasspath) testContainerImage(image('$image', 'test.cassandra.image')) } - tasks.register('verifyServiceSelection') { - def selected = providers.provider { - ['test', 'latestDepTest', 'latestDepTestForkedTest', 'isolatedTest'].collectEntries { name -> - def task = tasks.named(name).get() - [(name): task.requiredServices.searchServices().any { it.name == 'testcontainersLimit' }] - } - } - inputs.property('selected', selected) - doLast { - assert inputs.properties.selected == [test: true, latestDepTest: true, latestDepTestForkedTest: true, isolatedTest: false] - } - } """.trimIndent(), ) Files.createDirectories(directory.resolve("src/test/java")) diff --git a/build.gradle.kts b/build.gradle.kts index f516b38c6f8..5f07b849668 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -12,7 +12,6 @@ plugins { id("dd-trace-java.dump-hanged-test") id("dd-trace-java.gradle-debug") id("dd-trace-java.tracer-version") - id("dd-trace-java.testcontainers-limit") apply false alias(libs.plugins.shadow) apply false alias(libs.plugins.spotless) diff --git a/dd-java-agent/instrumentation/google-pubsub-1.116/build.gradle b/dd-java-agent/instrumentation/google-pubsub-1.116/build.gradle index 99f539589cc..a2b03630339 100644 --- a/dd-java-agent/instrumentation/google-pubsub-1.116/build.gradle +++ b/dd-java-agent/instrumentation/google-pubsub-1.116/build.gradle @@ -27,3 +27,7 @@ dependencies { testImplementation project(":dd-java-agent:instrumentation:gax-1.4") latestDepTestImplementation group: 'com.google.cloud', name: 'google-cloud-pubsub', version: '+' } + +tasks.withType(Test).configureEach { + usesService(testcontainersLimit) +} diff --git a/dd-java-agent/instrumentation/jdbc/build.gradle b/dd-java-agent/instrumentation/jdbc/build.gradle index f38c5760d93..dc2622fdec5 100644 --- a/dd-java-agent/instrumentation/jdbc/build.gradle +++ b/dd-java-agent/instrumentation/jdbc/build.gradle @@ -104,6 +104,10 @@ tasks.withType(GroovyCompile).configureEach { ) } +tasks.withType(Test).configureEach { + usesService(testcontainersLimit) +} + tasks.named("latestDepJava11Test", Test) { testJvmConstraints { minJavaVersion = JavaVersion.VERSION_11 diff --git a/dd-java-agent/instrumentation/vertx/vertx-mysql-client/vertx-mysql-client-3.9/build.gradle b/dd-java-agent/instrumentation/vertx/vertx-mysql-client/vertx-mysql-client-3.9/build.gradle index c1ec5a069c5..1f8423384f0 100644 --- a/dd-java-agent/instrumentation/vertx/vertx-mysql-client/vertx-mysql-client-3.9/build.gradle +++ b/dd-java-agent/instrumentation/vertx/vertx-mysql-client/vertx-mysql-client-3.9/build.gradle @@ -34,3 +34,7 @@ dependencies { latestDepTestImplementation group: 'io.vertx', name: 'vertx-mysql-client', version: '3.+' } + +tasks.withType(Test).configureEach { + usesService(testcontainersLimit) +} diff --git a/dd-java-agent/instrumentation/vertx/vertx-mysql-client/vertx-mysql-client-4.0/build.gradle b/dd-java-agent/instrumentation/vertx/vertx-mysql-client/vertx-mysql-client-4.0/build.gradle index 727a607812a..c314173c643 100644 --- a/dd-java-agent/instrumentation/vertx/vertx-mysql-client/vertx-mysql-client-4.0/build.gradle +++ b/dd-java-agent/instrumentation/vertx/vertx-mysql-client/vertx-mysql-client-4.0/build.gradle @@ -35,3 +35,7 @@ dependencies { latestDepTestImplementation group: 'io.vertx', name: 'vertx-mysql-client', version: '4.+' } + +tasks.withType(Test).configureEach { + usesService(testcontainersLimit) +} diff --git a/dd-java-agent/instrumentation/vertx/vertx-pg-client/vertx-pg-client-4.0/build.gradle b/dd-java-agent/instrumentation/vertx/vertx-pg-client/vertx-pg-client-4.0/build.gradle index a2d811abc10..0db168b53ad 100644 --- a/dd-java-agent/instrumentation/vertx/vertx-pg-client/vertx-pg-client-4.0/build.gradle +++ b/dd-java-agent/instrumentation/vertx/vertx-pg-client/vertx-pg-client-4.0/build.gradle @@ -55,3 +55,7 @@ configurations.named("latestDepTestRuntimeClasspath") { configurations.named("latestDepForkedTestRuntimeClasspath") { exclude group: 'com.ongres.scram', module: 'client' } + +tasks.withType(Test).configureEach { + usesService(testcontainersLimit) +} diff --git a/dd-smoke-tests/websphere-jmx/build.gradle b/dd-smoke-tests/websphere-jmx/build.gradle index ecf21ab32bb..4ba5b2d2c58 100644 --- a/dd-smoke-tests/websphere-jmx/build.gradle +++ b/dd-smoke-tests/websphere-jmx/build.gradle @@ -15,3 +15,7 @@ testJvmConstraints { minJavaVersion = JavaVersion.VERSION_25 maxJavaVersion = JavaVersion.VERSION_25 } + +tasks.withType(Test).configureEach { + usesService(testcontainersLimit) +} diff --git a/docs/how_to_smoke_test.md b/docs/how_to_smoke_test.md index 445b2565f43..ffb7e7f0f83 100644 --- a/docs/how_to_smoke_test.md +++ b/docs/how_to_smoke_test.md @@ -233,6 +233,46 @@ static final TestAgentBackend agent = AgentBackend.testAgentBuilder().retainAcro static final SmokeServerApp sender = /* ... */; ``` +## Container images + +For application containers or dependencies such as RabbitMQ, apply +`dd-trace-java.testcontainers` and declare the image in the smoke-test module: + +```groovy +plugins { + id 'dd-trace-java.module.smoke-test' + id 'dd-trace-java.testcontainers' +} + +dependencies { + testImplementation group: 'org.testcontainers', name: 'rabbitmq', version: libs.versions.testcontainers.get() + testImplementation group: 'org.testcontainers', name: 'junit-jupiter', version: libs.versions.testcontainers.get() + testContainerImage(image('rabbitmq:3.12-management-alpine', 'test.rabbitmq.image')) +} +``` + +Read the property when creating the container. Annotate the test class with +`@Testcontainers` so its `@Container` fields are started and stopped automatically: + +```java +@Container +private static final RabbitMQContainer RABBIT_MQ_CONTAINER = + new RabbitMQContainer( + DockerImageName.parse(System.getProperty("test.rabbitmq.image")) + .asCompatibleSubstituteFor("rabbitmq")); +``` + +Gradle fingerprints the resolved digest and passes that immutable image to the +test JVM. The property belongs to the test JVM; explicitly forward it if the +launched application needs it too. Continue using `placeholder(...)` for mapped +ports, which only become available after the container starts. + +Keep the existing `usesService(testcontainersLimit)` declarations on container +test tasks: the image plugin does not assign that service. It also does not +automatically track the test-agent backend's image or Testcontainers helper images. +See [Tests that use containers](./how_to_test.md#tests-that-use-containers) for +suite inheritance, cache behavior, IDE runs and concurrency guidance. + ## Choosing a backend The backend is the agent stand-in the app reports to. diff --git a/docs/how_to_test.md b/docs/how_to_test.md index a0bcefdb368..2d837094fb0 100644 --- a/docs/how_to_test.md +++ b/docs/how_to_test.md @@ -55,6 +55,54 @@ In order to identify such tests and avoid the continuous integration to fail, th > * using the `Retry` button from the job view: > ![Rerun workflow from failed](how_to_test/retry-failed-job.png) +## Tests that use containers + +Declare container images in the module's Gradle build so a changed image cannot +silently reuse cached test results: + +```groovy +plugins { + id 'dd-trace-java.testcontainers' +} + +dependencies { + testImplementation libs.testcontainers + testContainerImage(image('redis:7-alpine', 'test.redis.image')) +} +``` + +Use the supplied system property when constructing the container, without a tag +fallback. Keep the compatibility alias when using a Testcontainers module that +validates its image name: + +```java +DockerImageName image = DockerImageName.parse(System.getProperty("test.redis.image")) + .asCompatibleSubstituteFor("redis"); +GenericContainer redis = new GenericContainer<>(image).withExposedPorts(6379); +``` + +The plugin resolves tags to immutable registry digests before Gradle checks the +test cache, then passes those same image references to the test JVM. Unchanged +digests can reuse test results; changed digests select a different cache entry. +Tags are refreshed even when Gradle reuses its configuration cache. Resolution +failure stops the task rather than trusting an old result. + +Each source set has a `ContainerImage` declaration method. Images follow +`implementation` configuration inheritance and reach the matching test task and +its forked companions. For a separate suite, use its own declaration, such as +`integrationTestContainerImage(...)`. Declarations apply to the whole task, including +when `--tests` selects only some classes. Run IDE tests through Gradle, or supply +the image system properties explicitly. + +Image fingerprinting does not configure concurrency. Keep the existing explicit +`usesService(testcontainersLimit)` declarations; the service limits concurrent +test tasks, not individual containers. When adding container tests, configure the +service on the tasks that run them, including forked tasks where applicable. + +Only declared images are tracked; implicit helper images such as Ryuk are not. +See the [plugin reference](../build-logic/testcontainers/README.md) for registry +credentials, Docker Hub mirrors and supported image substitutions. + ## Running Tests You can run the whole project test suite using `./gradlew test` but expect it to take a certain time. diff --git a/gradle/java_no_deps.gradle b/gradle/java_no_deps.gradle index 288e3f0ebcc..46f77882407 100644 --- a/gradle/java_no_deps.gradle +++ b/gradle/java_no_deps.gradle @@ -1,6 +1,8 @@ import datadog.gradle.plugin.testJvmConstraints.TestJvmConstraintsExtension import datadog.gradle.plugin.testJvmConstraints.ProvideJvmArgsOnJvmLauncherVersion import groovy.transform.CompileStatic +import org.gradle.api.services.BuildService +import org.gradle.api.services.BuildServiceParameters import java.nio.file.Files import java.nio.file.LinkOption @@ -138,7 +140,15 @@ class TracerJavaExtension { } } def tracerJavaExtension = extensions.create(TracerJavaExtension.NAME, TracerJavaExtension, objects, providers, project) -apply plugin: 'dd-trace-java.testcontainers-limit' + + + +abstract class TestcontainersLimitService implements BuildService { +} + +ext.testcontainersLimit = gradle.sharedServices.registerIfAbsent("testcontainersLimit", TestcontainersLimitService) { + maxParallelUsages = project.findProperty("testcontainersMaxParallelUsages").toInteger() +} // Every project gets a forked test task for its default 'test' source set. addForkedTestTask('test') From c8ebb6790744a598e3627f8d08fc51afee48190b Mon Sep 17 00:00:00 2001 From: Brice Dutheil Date: Thu, 24 Sep 2026 12:32:34 +0200 Subject: [PATCH 07/15] fix: use dedicated container APIs and explain image resolution --- build-logic/testcontainers/README.md | 111 +++++++++++++----- .../test/groovy/CassandraClientTest.groovy | 8 +- .../test/groovy/CassandraClientTest.groovy | 8 +- .../test/groovy/CassandraClientTest.groovy | 2 +- docs/how_to_test.md | 16 ++- 5 files changed, 107 insertions(+), 38 deletions(-) diff --git a/build-logic/testcontainers/README.md b/build-logic/testcontainers/README.md index f0d6213f59f..26e5737ff9a 100644 --- a/build-logic/testcontainers/README.md +++ b/build-logic/testcontainers/README.md @@ -12,12 +12,17 @@ dependencies { } ``` -Read the property when constructing the container. Keep Testcontainers' compatibility -declaration when the image can come from a mirror: +Keep the dedicated container type and pass the property through its `DockerImageName` +constructor. The compatibility declaration lets it accept a mirrored image without +changing the resolved reference: ```java -DockerImageName.parse(System.getProperty("test.cassandra.image")) - .asCompatibleSubstituteFor("cassandra") +import org.testcontainers.cassandra.CassandraContainer; +import org.testcontainers.utility.DockerImageName; + +CassandraContainer container = new CassandraContainer( + DockerImageName.parse(System.getProperty("test.cassandra.image")) + .asCompatibleSubstituteFor("cassandra")); ``` Each source set gets a `ContainerImage` declaration method. Images follow @@ -31,40 +36,88 @@ This plugin only fingerprints images. Container concurrency remains controlled b the existing, explicit `usesService(testcontainersLimit)` declarations in module build scripts; applying this plugin does not add or remove them. -Configuration uses `configureEach` and a project-local provider rather than an -evaluation callback. Late declarations, configuration inheritance, resource directories +## Resolution and test execution + +For each selected test task with image declarations: + +1. Apply the task's effective Docker Hub prefix to image names without an explicit + registry. This selects the registry before resolving any digest. +2. Use Jib to fetch each tag's manifest from that registry and produce an immutable + `registry/repository@sha256:...` reference. Jib does not download image layers or + require a Docker daemon. Declarations already containing a digest skip this request. +3. Include the property names and resolved references in Gradle's test input + fingerprint, before the up-to-date and build-cache checks. +4. If the test needs to run, pass the same references as `-D=` + arguments to its JVM. Testcontainers then uses Docker to pull and start those + exact images. Cached test results require no container startup. + +The resolver shares each effective image's result within one build. The next build +resolves moving tags again, even when reusing the configuration cache. Unchanged +references allow `UP-TO-DATE`/`FROM-CACHE`; changed digests select a different cache +entry. The registry and repository are also part of the input, so switching hosts +changes the fingerprint even if both registries return the same digest. + +Moving tags need registry access even when Docker already has the image locally. +Resolution failure stops the test instead of trusting stale results. Unrelated +tasks and skipped tests perform no registry requests. + +Configuration uses `configureEach` and a project-local provider rather than +`afterEvaluate`. Late declarations, configuration inheritance, resource directories and task environment settings are captured when Gradle queries the provider. -Only that configuration is cached; registry resolution remains an execution-time input. -Isolated Projects has not been tested. - -The plugin resolves each effective tag once per build, when Gradle snapshots test -inputs. An annotated JVM argument provider includes property names and immutable -`registry/repository@sha256:...` values in the test fingerprint, then passes those -same values to the JVM. Unchanged images allow `UP-TO-DATE`/`FROM-CACHE`; changed -images select a different cache entry. Configuration-cache reuse still refreshes -tags. Unrelated tasks and skipped tests perform no registry requests. Resolution -failure stops the test instead of trusting stale results. - -Jib reads registry manifests without pulling layers or requiring a Docker daemon. -Its dependencies are relocated inside the plugin JAR so older libraries exported -by `buildSrc` cannot override its HTTP client. Tests load that JAR through an -included build with an older HttpClient on the `buildSrc` classpath. -Private registries use Docker's `config.json` and credential helpers, honoring -`DOCKER_CONFIG`. Remote registries require TLS; loopback registries also permit -local development certificates and HTTP. - -Docker Hub substitution honors `TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX`, then the -user's `.testcontainers.properties` and source-set `testcontainers.properties`. -Explicit registry names bypass this prefix, matching Testcontainers. Other mappings -belong in the declaration; JDBC's SQL Server declaration selects its CI mirror there. +Only that configuration is cached; the nested input getter resolves images before +test cache lookup on every build. Isolated Projects has not been tested. + +## Local registries and CI mirrors + +Without a prefix, the example declaration resolves `cassandra:4` from Docker Hub. +CI sets `TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX=registry.ddbuild.io/images/mirror/` +in the [test job configuration](../../.gitlab-ci.yml), giving this flow: + +```text +Local: cassandra:4 + -> registry-1.docker.io/library/cassandra@sha256:... + +CI: cassandra:4 + -> registry.ddbuild.io/images/mirror/cassandra:4 + -> registry.ddbuild.io/images/mirror/cassandra@sha256:... +``` + +CI fingerprints the mirror's content, which may differ from Docker Hub. The test +receives the fully qualified reference, so Testcontainers does not apply the prefix +again. This follows Testcontainers' [image substitution rules](https://java.testcontainers.org/features/image_name_substitution/). + +The prefix comes from the task's `TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX`, then +`hub.image.name.prefix` in the user's `.testcontainers.properties` and source-set +`testcontainers.properties`. Explicit task environment overrides and removals are +respected. Changing the inherited `TESTCONTAINERS_` environment variables invalidates +the configuration cache. + +Explicit registry names bypass the prefix, including explicit Docker Hub hosts. +For example, Pub/Sub's `gcr.io` and WebSphere's `icr.io` declarations retain their +registries. Other mappings belong in the declaration: +[JDBC's SQL Server declaration](../../dd-java-agent/instrumentation/jdbc/build.gradle) +selects `mcr.microsoft.com/mssql/server:latest` locally and +`registry.ddbuild.io/images/mirror/sqlserver:latest` when `CI` is present. This +replaces runtime substitution so Gradle fingerprints the image the test will use. + Custom image substitutors and `*.container.image` overrides in these configuration sources are rejected because they could replace the resolved digest at runtime. Image substitution supplied by dependency JAR resources is not supported. +Private registries use Docker's `config.json` and credential helpers from the Gradle +process, honoring `DOCKER_CONFIG`. Remote registries require TLS; loopback registries +also permit local development certificates and HTTP. + +Jib's dependencies are relocated inside the plugin JAR so older libraries exported +by `buildSrc` cannot override its HTTP client. Tests load that JAR through an +included build with an older HttpClient on the `buildSrc` classpath. + Only declared images are tracked. This migration covers Cassandra, Pub/Sub, JDBC, Vert.x MySQL/PostgreSQL and WebSphere fixtures. Other fixtures and implicit Testcontainers helpers such as Alpine/Ryuk require separate adoption. +## Verification + Run the hermetic registry and Gradle cache tests with: ```shell diff --git a/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-3.0/src/test/groovy/CassandraClientTest.groovy b/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-3.0/src/test/groovy/CassandraClientTest.groovy index 85d414bb5c3..88c820b9dc3 100644 --- a/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-3.0/src/test/groovy/CassandraClientTest.groovy +++ b/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-3.0/src/test/groovy/CassandraClientTest.groovy @@ -12,7 +12,7 @@ import datadog.trace.api.DDSpanTypes import datadog.trace.bootstrap.instrumentation.api.InstrumentationTags import datadog.trace.bootstrap.instrumentation.api.Tags import datadog.trace.core.DDSpan -import org.testcontainers.containers.CassandraContainer +import org.testcontainers.cassandra.CassandraContainer import org.testcontainers.utility.DockerImageName import spock.lang.Shared @@ -47,8 +47,12 @@ abstract class CassandraClientTest extends VersionedNamingTestBase { .asCompatibleSubstituteFor("cassandra") container = new CassandraContainer(image).withStartupTimeout(Duration.ofSeconds(120)) container.start() - cluster = container.getCluster() port = container.getMappedPort(9042) + cluster = Cluster.builder() + .addContactPoint(container.getHost()) + .withPort(port) + .withoutJMXReporting() + .build() // Looks like sometimes our requests fail because Cassandra takes to long to respond, // Increase this timeout as well to try to cope with this. cluster.getConfiguration().getSocketOptions().setReadTimeoutMillis(120000) diff --git a/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-3.8/src/test/groovy/CassandraClientTest.groovy b/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-3.8/src/test/groovy/CassandraClientTest.groovy index 85d414bb5c3..88c820b9dc3 100644 --- a/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-3.8/src/test/groovy/CassandraClientTest.groovy +++ b/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-3.8/src/test/groovy/CassandraClientTest.groovy @@ -12,7 +12,7 @@ import datadog.trace.api.DDSpanTypes import datadog.trace.bootstrap.instrumentation.api.InstrumentationTags import datadog.trace.bootstrap.instrumentation.api.Tags import datadog.trace.core.DDSpan -import org.testcontainers.containers.CassandraContainer +import org.testcontainers.cassandra.CassandraContainer import org.testcontainers.utility.DockerImageName import spock.lang.Shared @@ -47,8 +47,12 @@ abstract class CassandraClientTest extends VersionedNamingTestBase { .asCompatibleSubstituteFor("cassandra") container = new CassandraContainer(image).withStartupTimeout(Duration.ofSeconds(120)) container.start() - cluster = container.getCluster() port = container.getMappedPort(9042) + cluster = Cluster.builder() + .addContactPoint(container.getHost()) + .withPort(port) + .withoutJMXReporting() + .build() // Looks like sometimes our requests fail because Cassandra takes to long to respond, // Increase this timeout as well to try to cope with this. cluster.getConfiguration().getSocketOptions().setReadTimeoutMillis(120000) diff --git a/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-4.0/src/test/groovy/CassandraClientTest.groovy b/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-4.0/src/test/groovy/CassandraClientTest.groovy index fd3d735bfac..49cae522fdf 100644 --- a/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-4.0/src/test/groovy/CassandraClientTest.groovy +++ b/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-4.0/src/test/groovy/CassandraClientTest.groovy @@ -12,7 +12,7 @@ import datadog.trace.api.DDSpanTypes import datadog.trace.bootstrap.instrumentation.api.InstrumentationTags import datadog.trace.bootstrap.instrumentation.api.Tags import datadog.trace.core.DDSpan -import org.testcontainers.containers.CassandraContainer +import org.testcontainers.cassandra.CassandraContainer import org.testcontainers.utility.DockerImageName import spock.lang.Shared import spock.util.concurrent.BlockingVariable diff --git a/docs/how_to_test.md b/docs/how_to_test.md index 2d837094fb0..b349a0a2f55 100644 --- a/docs/how_to_test.md +++ b/docs/how_to_test.md @@ -67,20 +67,28 @@ plugins { dependencies { testImplementation libs.testcontainers + testImplementation group: 'com.redis.testcontainers', name: 'testcontainers-redis', version: '1.6.2' testContainerImage(image('redis:7-alpine', 'test.redis.image')) } ``` -Use the supplied system property when constructing the container, without a tag -fallback. Keep the compatibility alias when using a Testcontainers module that -validates its image name: +Keep the dedicated container type and use its `DockerImageName` constructor with +the supplied property, without a tag fallback. This example uses the same Redis +module as the repository's Redis tests. Keep the compatibility alias so container +types that validate their image name also accept CI mirrors: ```java +import com.redis.testcontainers.RedisContainer; +import org.testcontainers.utility.DockerImageName; + DockerImageName image = DockerImageName.parse(System.getProperty("test.redis.image")) .asCompatibleSubstituteFor("redis"); -GenericContainer redis = new GenericContainer<>(image).withExposedPorts(6379); +RedisContainer redis = new RedisContainer(image); ``` +Use `GenericContainer` for custom application images without a matching container +module, as in the [WebSphere smoke test](../dd-smoke-tests/websphere-jmx). + The plugin resolves tags to immutable registry digests before Gradle checks the test cache, then passes those same image references to the test JVM. Unchanged digests can reuse test results; changed digests select a different cache entry. From 2d4f0f97fca11f3661def3f584c18710fcd36001 Mon Sep 17 00:00:00 2001 From: Brice Dutheil Date: Thu, 24 Sep 2026 13:39:55 +0200 Subject: [PATCH 08/15] chore: Align Kotlin DSL usage --- build-logic/testcontainers/README.md | 28 +- build-logic/testcontainers/build.gradle.kts | 15 +- .../testcontainers/ContainerImages.kt | 48 ++ .../testcontainers/TestcontainersPlugin.kt | 25 +- .../testcontainers/RegistryExtension.kt | 98 ++++ .../TestcontainersPluginTest.kt | 422 ++++++++---------- docs/how_to_smoke_test.md | 15 +- docs/how_to_test.md | 22 +- 8 files changed, 392 insertions(+), 281 deletions(-) create mode 100644 build-logic/testcontainers/src/main/kotlin/datadog/buildlogic/testcontainers/ContainerImages.kt create mode 100644 build-logic/testcontainers/src/test/kotlin/datadog/buildlogic/testcontainers/RegistryExtension.kt diff --git a/build-logic/testcontainers/README.md b/build-logic/testcontainers/README.md index 26e5737ff9a..750941d8503 100644 --- a/build-logic/testcontainers/README.md +++ b/build-logic/testcontainers/README.md @@ -2,16 +2,23 @@ Apply this plugin only in modules that use containers: -```groovy +```kotlin +import datadog.buildlogic.testcontainers.image +import datadog.buildlogic.testcontainers.testContainerImage + plugins { - id 'dd-trace-java.testcontainers' + id("dd-trace-java.testcontainers") } dependencies { - testContainerImage(image('cassandra:4', 'test.cassandra.image')) + testContainerImage(image("cassandra:4", "test.cassandra.image")) } ``` +> [!NOTE] +> The imports expose the plugin's Kotlin functions in the build script; applying the +plugin alone does not import them. + Keep the dedicated container type and pass the property through its `DockerImageName` constructor. The compatibility declaration lets it accept a mirrored image without changing the resolved reference: @@ -25,13 +32,20 @@ CassandraContainer container = new CassandraContainer( .asCompatibleSubstituteFor("cassandra")); ``` -Each source set gets a `ContainerImage` declaration method. Images follow -`implementation` configuration inheritance, so a `latestDepTest` suite extending +`testContainerImage` declares an image for the `test` source set. For a separate +suite, import `datadog.buildlogic.testcontainers.containerImage`, declare its source +set first, then use `containerImage("integrationTest", image("redis:7-alpine", "test.redis.image"))` +inside `dependencies {}`. +Images follow `implementation` configuration inheritance, so a `latestDepTest` suite extending `testImplementation` inherits its images. The matching `Test` task, `forkedTest`, -and `ForkedTest` companions receive the properties. A separate suite can -declare its own images with, for example, `integrationTestContainerImage(...)`. +and `ForkedTest` companions receive the properties. Run IDE tests through Gradle, or supply the named image properties explicitly. +Groovy builds use `dependencies { testContainerImage(image('cassandra:4', +'test.cassandra.image')) }` without imports. Each source set gets a dynamic +`ContainerImage` method in Groovy; Kotlin uses `containerImage` for custom +source sets. Both DSLs use the same declarations and validation. + This plugin only fingerprints images. Container concurrency remains controlled by the existing, explicit `usesService(testcontainersLimit)` declarations in module build scripts; applying this plugin does not add or remove them. diff --git a/build-logic/testcontainers/build.gradle.kts b/build-logic/testcontainers/build.gradle.kts index e168f939a2a..1315f9c4122 100644 --- a/build-logic/testcontainers/build.gradle.kts +++ b/build-logic/testcontainers/build.gradle.kts @@ -1,3 +1,5 @@ +import org.gradle.internal.classpath.Instrumented.systemProperty + plugins { `java-gradle-plugin` `kotlin-dsl` @@ -5,17 +7,6 @@ plugins { alias(libs.plugins.shadow) } -java { - sourceCompatibility = JavaVersion.VERSION_1_8 - targetCompatibility = JavaVersion.VERSION_1_8 -} - -kotlin { - compilerOptions { - jvmTarget.set(org.jetbrains.kotlin.gradle.dsl.JvmTarget.JVM_1_8) - } -} - val jib = configurations.create("jib") val conflictingBuildSrc = configurations.create("conflictingBuildSrc") configurations.compileOnly { extendsFrom(jib) } @@ -64,6 +55,8 @@ testing { useJUnitJupiter(libs.versions.junit5) dependencies { implementation(libs.assertj.core) + implementation(libs.okhttp3.mockwebserver) + implementation("com.squareup.okhttp3:okhttp-tls:${libs.versions.okhttp3.testing.get()}") implementation(gradleTestKit()) } } diff --git a/build-logic/testcontainers/src/main/kotlin/datadog/buildlogic/testcontainers/ContainerImages.kt b/build-logic/testcontainers/src/main/kotlin/datadog/buildlogic/testcontainers/ContainerImages.kt new file mode 100644 index 00000000000..51dabfb0f99 --- /dev/null +++ b/build-logic/testcontainers/src/main/kotlin/datadog/buildlogic/testcontainers/ContainerImages.kt @@ -0,0 +1,48 @@ +package datadog.buildlogic.testcontainers + +import org.gradle.api.artifacts.dsl.DependencyHandler +import org.gradle.api.plugins.ExtensionAware + +/** Creates an image declaration for the test JVM property that consumes it. */ +fun image( + reference: String, + systemProperty: String, +): ContainerImage = ContainerImage(reference, systemProperty) + +/** Declares a container image for the test source set. */ +fun DependencyHandler.testContainerImage(image: ContainerImage) = containerImage("test", image) + +/** Declares a container image for an existing source set. */ +fun DependencyHandler.containerImage( + sourceSet: String, + image: ContainerImage, +) { + val images = (this as ExtensionAware).extensions.extraProperties.get("testContainerImages") as ContainerImages + images.add(sourceSet, image) +} + +internal class ContainerImages { + val declarations = mutableMapOf>() + + fun add( + sourceSet: String, + image: ContainerImage, + ) { + val images = requireNotNull(declarations[sourceSet]) { "Unknown container image source set '$sourceSet'" } + require(images.putIfAbsent(image.systemProperty, image.reference) == null) { + "Container image system property '${image.systemProperty}' is declared twice in $sourceSet" + } + } +} + +data class ContainerImage( + val reference: String, + val systemProperty: String, +) { + init { + require(reference.isNotBlank()) { "Container image reference must not be empty" } + require(systemProperty.matches(Regex("[A-Za-z0-9_.-]+"))) { + "Invalid container image system property: $systemProperty" + } + } +} diff --git a/build-logic/testcontainers/src/main/kotlin/datadog/buildlogic/testcontainers/TestcontainersPlugin.kt b/build-logic/testcontainers/src/main/kotlin/datadog/buildlogic/testcontainers/TestcontainersPlugin.kt index 4c7807a67ef..25247d9c9d6 100644 --- a/build-logic/testcontainers/src/main/kotlin/datadog/buildlogic/testcontainers/TestcontainersPlugin.kt +++ b/build-logic/testcontainers/src/main/kotlin/datadog/buildlogic/testcontainers/TestcontainersPlugin.kt @@ -13,21 +13,16 @@ import java.io.File /** Declares container images alongside the dependencies of the test suite that consumes them. */ class TestcontainersPlugin : Plugin { override fun apply(project: Project) { - val declarations = mutableMapOf>() + val containerImages = ContainerImages() val dependencyDsl = (project.dependencies as ExtensionAware).extensions.extraProperties + dependencyDsl.set("testContainerImages", containerImages) dependencyDsl.set( "image", object : Closure(null) { fun doCall( reference: String, systemProperty: String, - ): ContainerImage { - require(reference.isNotBlank()) { "Container image reference must not be empty" } - require(systemProperty.matches(Regex("[A-Za-z0-9_.-]+"))) { - "Invalid container image system property: $systemProperty" - } - return ContainerImage(reference, systemProperty) - } + ): ContainerImage = image(reference, systemProperty) }, ) @@ -40,14 +35,13 @@ class TestcontainersPlugin : Plugin { project.pluginManager.withPlugin("java") { val sourceSets = project.extensions.getByType() sourceSets.all { - val images = declarations.getOrPut(name) { linkedMapOf() } + containerImages.declarations.getOrPut(name) { linkedMapOf() } + val sourceSetName = name dependencyDsl.set( "${name}ContainerImage", object : Closure(null) { fun doCall(image: ContainerImage) { - require(images.putIfAbsent(image.systemProperty, image.reference) == null) { - "Container image system property '${image.systemProperty}' is declared twice in $name" - } + containerImages.add(sourceSetName, image) } }, ) @@ -68,7 +62,7 @@ class TestcontainersPlugin : Plugin { hierarchy.any { it.name == sourceSet.implementationConfigurationName } } inheritedSourceSets.forEach { sourceSet -> - declarations[sourceSet.name]?.forEach { (property, reference) -> + containerImages.declarations[sourceSet.name]?.forEach { (property, reference) -> require(images.putIfAbsent(property, reference).let { it == null || it == reference }) { "Conflicting container images for '$property' in $path" } @@ -97,8 +91,3 @@ class TestcontainersPlugin : Plugin { } } } - -data class ContainerImage( - val reference: String, - val systemProperty: String, -) diff --git a/build-logic/testcontainers/src/test/kotlin/datadog/buildlogic/testcontainers/RegistryExtension.kt b/build-logic/testcontainers/src/test/kotlin/datadog/buildlogic/testcontainers/RegistryExtension.kt new file mode 100644 index 00000000000..e61fe9c331e --- /dev/null +++ b/build-logic/testcontainers/src/test/kotlin/datadog/buildlogic/testcontainers/RegistryExtension.kt @@ -0,0 +1,98 @@ +package datadog.buildlogic.testcontainers + +import okhttp3.mockwebserver.Dispatcher +import okhttp3.mockwebserver.MockResponse +import okhttp3.mockwebserver.MockWebServer +import okhttp3.mockwebserver.RecordedRequest +import okhttp3.tls.HandshakeCertificates +import okhttp3.tls.HeldCertificate +import org.junit.jupiter.api.extension.AfterEachCallback +import org.junit.jupiter.api.extension.BeforeEachCallback +import org.junit.jupiter.api.extension.ExtensionContext +import java.net.InetAddress +import java.security.MessageDigest +import java.util.concurrent.atomic.AtomicInteger + +/** A local HTTPS registry with mutable manifests and optional bearer authentication. */ +class RegistryExtension : + BeforeEachCallback, + AfterEachCallback { + private val server = MockWebServer() + + @Volatile var imageVersion = 1 + + @Volatile var requireAuthentication = false + + @Volatile var unavailable = false + + val tokenRequests = AtomicInteger() + val authorizedRequests = AtomicInteger() + + val image: String + get() = "127.0.0.1:${server.port}/library/cassandra:4" + + val requestCount: Int + get() = server.requestCount + + val digest: String + get() = digest(manifest()) + + override fun beforeEach(context: ExtensionContext) { + val certificate = + HeldCertificate + .Builder() + .commonName("localhost") + .addSubjectAlternativeName("127.0.0.1") + .build() + val certificates = HandshakeCertificates.Builder().heldCertificate(certificate).build() + server.useHttps(certificates.sslSocketFactory(), false) + server.setDispatcher( + object : Dispatcher() { + override fun dispatch(request: RecordedRequest): MockResponse = + when { + unavailable -> { + MockResponse().setResponseCode(503) + } + + request.path!!.startsWith("/token") -> { + tokenRequests.incrementAndGet() + MockResponse().setHeader("Content-Type", "application/json").setBody("""{"token":"fixture-token"}""") + } + + !request.path!!.startsWith("/v2/") -> { + MockResponse().setResponseCode(404) + } + + requireAuthentication && request.getHeader("Authorization") != "Bearer fixture-token" -> { + MockResponse().setResponseCode(401).setHeader( + "WWW-Authenticate", + "Bearer realm=\"${server.url("/token")}\",service=\"fixture\",scope=\"repository:library/cassandra:pull\"", + ) + } + + else -> { + if (requireAuthentication) authorizedRequests.incrementAndGet() + val body = manifest() + MockResponse() + .setHeader("Content-Type", "application/vnd.oci.image.manifest.v1+json") + .setHeader("Docker-Content-Digest", digest(body)) + .setBody(body) + } + } + }, + ) + server.start(InetAddress.getByName("127.0.0.1"), 0) + } + + override fun afterEach(context: ExtensionContext) { + server.shutdown() + } + + private fun manifest() = + """{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json","config":{"mediaType":"application/vnd.oci.image.config.v1+json","digest":"${digest( + imageVersion.toString(), + )}","size":2},"layers":[]}""" + + private fun digest(body: String) = + "sha256:" + MessageDigest.getInstance("SHA-256").digest(body.toByteArray()).joinToString("") { "%02x".format(it) } +} diff --git a/build-logic/testcontainers/src/test/kotlin/datadog/buildlogic/testcontainers/TestcontainersPluginTest.kt b/build-logic/testcontainers/src/test/kotlin/datadog/buildlogic/testcontainers/TestcontainersPluginTest.kt index 187483a11af..55fd6f0e0c7 100644 --- a/build-logic/testcontainers/src/test/kotlin/datadog/buildlogic/testcontainers/TestcontainersPluginTest.kt +++ b/build-logic/testcontainers/src/test/kotlin/datadog/buildlogic/testcontainers/TestcontainersPluginTest.kt @@ -1,84 +1,67 @@ package datadog.buildlogic.testcontainers -import com.sun.net.httpserver.HttpsConfigurator -import com.sun.net.httpserver.HttpsServer import org.assertj.core.api.Assertions.assertThat import org.gradle.testkit.runner.GradleRunner import org.gradle.testkit.runner.TaskOutcome import org.junit.jupiter.api.Test +import org.junit.jupiter.api.extension.RegisterExtension import org.junit.jupiter.api.io.TempDir import java.io.File -import java.net.InetSocketAddress import java.nio.file.Files import java.nio.file.Path -import java.security.KeyStore -import java.security.MessageDigest import java.util.Properties -import java.util.concurrent.atomic.AtomicInteger -import java.util.concurrent.atomic.AtomicReference -import javax.net.ssl.KeyManagerFactory -import javax.net.ssl.SSLContext class TestcontainersPluginTest { @TempDir lateinit var directory: Path + @RegisterExtension + @JvmField + val registry = RegistryExtension() + @Test fun `moving images are refreshed before cache lookup including configuration cache reuse`() { - val manifests = AtomicReference(manifest("1")) - val requests = AtomicInteger() - val server = registry() - server.createContext("/v2/") { exchange -> - requests.incrementAndGet() - val body = manifests.get().toByteArray() - exchange.responseHeaders.add("Content-Type", "application/vnd.oci.image.manifest.v1+json") - exchange.responseHeaders.add("Docker-Content-Digest", digest(manifests.get())) - exchange.sendResponseHeaders(200, body.size.toLong()) - exchange.responseBody.use { it.write(body) } - } - server.start() - try { - fixture("127.0.0.1:${server.address.port}/library/cassandra:4") - assertThat(run("help").task(":help")?.outcome).isEqualTo(TaskOutcome.SUCCESS) - assertThat(requests.get()).isZero() - assertThat(run("test", "-PskipTests").task(":test")?.outcome).isEqualTo(TaskOutcome.SKIPPED) - assertThat(requests.get()).isZero() + fixture(registry.image) + assertThat(run("help").task(":help")?.outcome).isEqualTo(TaskOutcome.SUCCESS) + assertThat(registry.requestCount).isZero() + assertThat(run("test", "-PskipTests").task(":test")?.outcome).isEqualTo(TaskOutcome.SKIPPED) + assertThat(registry.requestCount).isZero() - assertThat(run("test").task(":test")?.outcome).isEqualTo(TaskOutcome.SUCCESS) - assertThat(report()).contains("library/cassandra@${digest(manifests.get())}") - val firstRequests = requests.get() - val warm = run("test") - assertThat(warm.output).contains("Reusing configuration cache") - assertThat(warm.task(":test")?.outcome).isEqualTo(TaskOutcome.UP_TO_DATE) - assertThat(requests.get()).isGreaterThan(firstRequests) + assertThat(run("test").task(":test")?.outcome).isEqualTo(TaskOutcome.SUCCESS) + assertThat(report()).contains("library/cassandra@${registry.digest}") + val firstRequests = registry.requestCount + val warm = run("test") + assertThat(warm.output).contains("Reusing configuration cache") + assertThat(warm.task(":test")?.outcome).isEqualTo(TaskOutcome.UP_TO_DATE) + assertThat(registry.requestCount).isGreaterThan(firstRequests) - manifests.set(manifest("2")) - val changed = run("test") - assertThat(changed.output).contains("Reusing configuration cache") - assertThat(changed.task(":test")?.outcome).isEqualTo(TaskOutcome.SUCCESS) - assertThat(report()).contains("library/cassandra@${digest(manifests.get())}") + registry.imageVersion = 2 + val changed = run("test") + assertThat(changed.output).contains("Reusing configuration cache") + assertThat(changed.task(":test")?.outcome).isEqualTo(TaskOutcome.SUCCESS) + assertThat(report()).contains("library/cassandra@${registry.digest}") - directory.resolve("build").toFile().deleteRecursively() - val restored = run("test") - assertThat(restored.output).contains("Reusing configuration cache") - assertThat(restored.task(":test")?.outcome).isEqualTo(TaskOutcome.FROM_CACHE) - assertThat(report()).contains("library/cassandra@${digest(manifests.get())}") + run("clean") + val restored = run("test") + assertThat(restored.output).contains("Reusing configuration cache") + assertThat(restored.task(":test")?.outcome).isEqualTo(TaskOutcome.FROM_CACHE) + assertThat(report()).contains("library/cassandra@${registry.digest}") - assertThat(run("latestDepTest", "latestDepTestForkedTest").task(":latestDepTestForkedTest")?.outcome) - .isIn(TaskOutcome.SUCCESS, TaskOutcome.FROM_CACHE) - assertThat(directory.resolve("build/test-results/latestDepTest/TEST-ImageTest.xml").toFile().readText()) - .contains("library/cassandra@${digest(manifests.get())}") - val requestsBeforeUnrelatedSuite = requests.get() - assertThat(run("isolatedTest").task(":isolatedTest")?.outcome).isEqualTo(TaskOutcome.SUCCESS) - assertThat(run("emptyTest").task(":emptyTest")?.outcome).isEqualTo(TaskOutcome.NO_SOURCE) - assertThat(requests.get()).isEqualTo(requestsBeforeUnrelatedSuite) + assertThat(run("latestDepTest", "latestDepTestForkedTest").task(":latestDepTestForkedTest")?.outcome) + .isIn(TaskOutcome.SUCCESS, TaskOutcome.FROM_CACHE) + assertThat(directory.resolve("build/test-results/latestDepTest/TEST-ImageTest.xml").toFile().readText()) + .contains("library/cassandra@${registry.digest}") + assertThat(run("declaredTest").task(":declaredTest")?.outcome).isIn(TaskOutcome.SUCCESS, TaskOutcome.FROM_CACHE) + assertThat(directory.resolve("build/test-results/declaredTest/TEST-ImageTest.xml").toFile().readText()) + .contains("library/cassandra@${registry.digest}") + val requestsBeforeUnrelatedSuite = registry.requestCount + assertThat(run("isolatedTest").task(":isolatedTest")?.outcome).isEqualTo(TaskOutcome.SUCCESS) + assertThat(run("emptyTest").task(":emptyTest")?.outcome).isEqualTo(TaskOutcome.NO_SOURCE) + assertThat(registry.requestCount).isEqualTo(requestsBeforeUnrelatedSuite) - server.stop(0) - assertThat(runner("test").buildAndFail().output) - .contains("Cannot resolve test container image") - } finally { - server.stop(0) - } + registry.unavailable = true + assertThat(runner("test").buildAndFail().output) + .contains("Cannot resolve test container image") } @Test @@ -102,12 +85,12 @@ class TestcontainersPluginTest { assertThat(changed.task(":test")?.outcome).isEqualTo(TaskOutcome.SUCCESS) assertThat(report()).contains("another.example/team/cassandra@$digest") - directory.resolve("build.gradle").toFile().appendText( + directory.resolve("build.gradle.kts").toFile().appendText( """ - sourceSets.test.resources.srcDirs = ['lateResources'] - tasks.named('test') { - environment 'TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX', 'task.example/' + sourceSets.test { resources.setSrcDirs(listOf("lateResources")) } + tasks.named("test") { + environment("TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX", "task.example/") } """.trimIndent(), ) @@ -126,12 +109,12 @@ class TestcontainersPluginTest { fun `task environment removals override inherited Testcontainers settings`() { val digest = "sha256:${"a".repeat(64)}" fixture("cassandra@$digest") - directory.resolve("build.gradle").toFile().appendText( + directory.resolve("build.gradle.kts").toFile().appendText( """ - tasks.named('test') { - environment.remove('TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX') - environment.remove('TESTCONTAINERS_IMAGE_SUBSTITUTOR') + tasks.named("test") { + environment.remove("TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX") + environment.remove("TESTCONTAINERS_IMAGE_SUBSTITUTOR") } """.trimIndent(), ) @@ -150,143 +133,153 @@ class TestcontainersPluginTest { } @Test - fun `bearer authentication resolves manifests with an older HttpClient in buildSrc`() { - val server = registry() - val tokenRequests = AtomicInteger() - val authorizedRequests = AtomicInteger() - server.createContext("/token") { exchange -> - tokenRequests.incrementAndGet() - val body = """{"token":"fixture-token"}""".toByteArray() - exchange.responseHeaders.add("Content-Type", "application/json") - exchange.sendResponseHeaders(200, body.size.toLong()) - exchange.responseBody.use { it.write(body) } - } - server.createContext("/v2/") { exchange -> - if (exchange.requestHeaders.getFirst("Authorization") != "Bearer fixture-token") { - exchange.responseHeaders.add( - "WWW-Authenticate", - "Bearer realm=\"https://127.0.0.1:${server.address.port}/token\",service=\"fixture\",scope=\"repository:library/cassandra:pull\"", - ) - exchange.sendResponseHeaders(401, -1) - exchange.close() - } else { - authorizedRequests.incrementAndGet() - val body = manifest("3").toByteArray() - exchange.responseHeaders.add("Content-Type", "application/vnd.oci.image.manifest.v1+json") - exchange.sendResponseHeaders(200, body.size.toLong()) - exchange.responseBody.use { it.write(body) } + fun `Groovy dependency extension methods are available for each source set`() { + val image = "cassandra@sha256:${"a".repeat(64)}" + fixture(image) + Files.delete(directory.resolve("build.gradle.kts")) + directory.resolve("build.gradle").toFile().writeText( + """ + plugins { + id 'java' + id 'dd-trace-java.testcontainers' } - } - server.start() - try { - fixture("127.0.0.1:${server.address.port}/library/cassandra:4") - // Reproduce the parent classloader supplied by Aether in the real buildSrc. - val httpClasspath = - System.getProperty("test.buildSrc.classpath").split(File.pathSeparator).joinToString(", ") { "'$it'" } - Files.createDirectories(directory.resolve("buildSrc/src/main/java")) - directory.resolve("buildSrc/src/main/java/BuildLogic.java").toFile().writeText("public class BuildLogic {}\n") - directory.resolve("buildSrc/build.gradle").toFile().writeText( - """ - plugins { id 'java' } - dependencies { implementation files($httpClasspath) } - """.trimIndent(), - ) - // Load as an included build instead of using TestKit's injected plugin classpath. - val metadata = Properties() - javaClass.classLoader.getResourceAsStream("plugin-under-test-metadata.properties")!!.use { metadata.load(it) } - val pluginClasspath = metadata.getProperty("implementation-classpath").split(File.pathSeparator).joinToString(", ") { "'$it'" } - Files.createDirectories(directory.resolve("plugin")) - directory.resolve("plugin/settings.gradle").toFile().writeText("rootProject.name = 'fixture-plugin'\n") - directory.resolve("plugin/build.gradle").toFile().writeText( - """ - plugins { id 'java-gradle-plugin' } - dependencies { implementation files($pluginClasspath) } - gradlePlugin { - plugins { - testcontainers { - id = 'dd-trace-java.testcontainers' - implementationClass = 'datadog.buildlogic.testcontainers.TestcontainersPlugin' - } + sourceSets { + integrationTest { java.srcDirs = sourceSets.test.java.srcDirs } + } + dependencies { + testImplementation files(${junitClasspath()}) + integrationTestImplementation files(${junitClasspath()}) + testContainerImage(image('$image', 'test.cassandra.image')) + integrationTestContainerImage(image('$image', 'test.cassandra.image')) + } + tasks.register('integrationTest', Test) { + testClassesDirs = sourceSets.integrationTest.output.classesDirs + classpath = sourceSets.integrationTest.runtimeClasspath + } + tasks.withType(Test).configureEach { useJUnitPlatform() } + """.trimIndent(), + ) + val result = run("test", "integrationTest") + assertThat(result.task(":test")?.outcome).isIn(TaskOutcome.SUCCESS, TaskOutcome.FROM_CACHE) + assertThat(result.task(":integrationTest")?.outcome).isIn(TaskOutcome.SUCCESS, TaskOutcome.FROM_CACHE) + assertThat(report()).contains("registry-1.docker.io/library/$image") + assertThat(directory.resolve("build/test-results/integrationTest/TEST-ImageTest.xml").toFile().readText()) + .contains("registry-1.docker.io/library/$image") + val reused = run("test", "integrationTest") + assertThat(reused.output).contains("Reusing configuration cache") + assertThat(reused.task(":test")?.outcome).isEqualTo(TaskOutcome.UP_TO_DATE) + assertThat(reused.task(":integrationTest")?.outcome).isEqualTo(TaskOutcome.UP_TO_DATE) + } + + @Test + fun `bearer authentication resolves manifests with an older HttpClient in buildSrc`() { + registry.requireAuthentication = true + fixture(registry.image) + // Reproduce the parent classloader supplied by Aether in the real buildSrc. + val httpClasspath = + System.getProperty("test.buildSrc.classpath").split(File.pathSeparator).joinToString(", ") { "\"$it\"" } + Files.createDirectories(directory.resolve("buildSrc/src/main/java")) + directory.resolve("buildSrc/src/main/java/BuildLogic.java").toFile().writeText("public class BuildLogic {}\n") + directory.resolve("buildSrc/build.gradle.kts").toFile().writeText( + """ + plugins { java } + dependencies { implementation(files($httpClasspath)) } + """.trimIndent(), + ) + // Load as an included build instead of using TestKit's injected plugin classpath. + val metadata = Properties() + javaClass.classLoader.getResourceAsStream("plugin-under-test-metadata.properties")!!.use { metadata.load(it) } + val pluginClasspath = metadata.getProperty("implementation-classpath").split(File.pathSeparator).joinToString(", ") { "\"$it\"" } + Files.createDirectories(directory.resolve("plugin")) + directory.resolve("plugin/settings.gradle.kts").toFile().writeText("rootProject.name = \"fixture-plugin\"\n") + directory.resolve("plugin/build.gradle.kts").toFile().writeText( + """ + plugins { `java-gradle-plugin` } + dependencies { implementation(files($pluginClasspath)) } + gradlePlugin { + plugins { + create("testcontainers") { + id = "dd-trace-java.testcontainers" + implementationClass = "datadog.buildlogic.testcontainers.TestcontainersPlugin" } } - """.trimIndent(), - ) - val settings = directory.resolve("settings.gradle").toFile() - settings.writeText("pluginManagement { includeBuild('plugin') }\n" + settings.readText()) - assertThat(runner("test", injectPluginClasspath = false).build().task(":test")?.outcome).isEqualTo(TaskOutcome.SUCCESS) - assertThat(tokenRequests.get()).isPositive() - assertThat(authorizedRequests.get()).isPositive() - assertThat(report()).contains(digest(manifest("3"))) - } finally { - server.stop(0) - } + } + """.trimIndent(), + ) + val settings = directory.resolve("settings.gradle.kts").toFile() + settings.writeText("pluginManagement { includeBuild(\"plugin\") }\n" + settings.readText()) + assertThat(runner("test", injectPluginClasspath = false).build().task(":test")?.outcome).isEqualTo(TaskOutcome.SUCCESS) + assertThat(registry.tokenRequests.get()).isPositive() + assertThat(registry.authorizedRequests.get()).isPositive() + assertThat(report()).contains(registry.digest) } private fun fixture(image: String) { - directory.resolve("settings.gradle").toFile().writeText( + directory.resolve("settings.gradle.kts").toFile().writeText( """ - rootProject.name = 'container-image-fixture' - buildCache { local { directory = file('cache') } } + rootProject.name = "container-image-fixture" + buildCache { local { directory = file("cache") } } """.trimIndent(), ) - val junitClasspath = - listOf( - "org.junit.jupiter.api.Test", - "org.junit.jupiter.engine.JupiterTestEngine", - "org.junit.platform.engine.TestEngine", - "org.junit.platform.launcher.Launcher", - "org.junit.platform.commons.JUnitException", - "org.opentest4j.AssertionFailedError", - ).joinToString(", ") { - "'${Path.of( - Class - .forName(it) - .protectionDomain.codeSource.location - .toURI(), - )}'" - } - directory.resolve("build.gradle").toFile().writeText( + directory.resolve("build.gradle.kts").toFile().writeText( """ + import datadog.buildlogic.testcontainers.containerImage + import datadog.buildlogic.testcontainers.image + import datadog.buildlogic.testcontainers.testContainerImage + plugins { - id 'java' - id 'dd-trace-java.testcontainers' + java + id("dd-trace-java.testcontainers") } - sourceSets { - latestDepTest { java.srcDirs = sourceSets.test.java.srcDirs } - isolatedTest - emptyTest + val latestDepTest = sourceSets.create("latestDepTest") { + java.setSrcDirs(sourceSets.test.get().java.srcDirs) + } + val declaredTest = sourceSets.create("declaredTest") { + java.setSrcDirs(sourceSets.test.get().java.srcDirs) } - tasks.register('latestDepTest', Test) { - testClassesDirs = sourceSets.latestDepTest.output.classesDirs - classpath = sourceSets.latestDepTest.runtimeClasspath + val isolatedTest = sourceSets.create("isolatedTest") + val emptyTest = sourceSets.create("emptyTest") + tasks.register("latestDepTest") { + testClassesDirs = latestDepTest.output.classesDirs + classpath = latestDepTest.runtimeClasspath } - tasks.register('latestDepTestForkedTest', Test) { - testClassesDirs = sourceSets.latestDepTest.output.classesDirs - classpath = sourceSets.latestDepTest.runtimeClasspath + tasks.register("latestDepTestForkedTest") { + testClassesDirs = latestDepTest.output.classesDirs + classpath = latestDepTest.runtimeClasspath } - tasks.register('isolatedTest', Test) { - testClassesDirs = sourceSets.isolatedTest.output.classesDirs - classpath = sourceSets.isolatedTest.runtimeClasspath + tasks.register("declaredTest") { + testClassesDirs = declaredTest.output.classesDirs + classpath = declaredTest.runtimeClasspath } - tasks.register('emptyTest', Test) { - testClassesDirs = sourceSets.emptyTest.output.classesDirs - classpath = sourceSets.emptyTest.runtimeClasspath + tasks.register("isolatedTest") { + testClassesDirs = isolatedTest.output.classesDirs + classpath = isolatedTest.runtimeClasspath } - def skip = providers.gradleProperty('skipTests') - tasks.withType(Test).configureEach { + tasks.register("emptyTest") { + testClassesDirs = emptyTest.output.classesDirs + classpath = emptyTest.runtimeClasspath + } + tasks.withType().configureEach { useJUnitPlatform() - onlyIf { !skip.isPresent() } + val skip = providers.gradleProperty("skipTests") + onlyIf { !skip.isPresent } } // Plugins must see declarations and inheritance added after tasks are realized. - tasks.named('test').get() - tasks.named('latestDepTest').get() - tasks.named('latestDepTestForkedTest').get() - configurations.latestDepTestImplementation.extendsFrom(configurations.testImplementation) - configurations.emptyTestImplementation.extendsFrom(configurations.testImplementation) + tasks.named("test").get() + tasks.named("latestDepTest").get() + tasks.named("latestDepTestForkedTest").get() + configurations.named(latestDepTest.implementationConfigurationName) { + extendsFrom(configurations.testImplementation.get()) + } + configurations.named(emptyTest.implementationConfigurationName) { + extendsFrom(configurations.testImplementation.get()) + } dependencies { - testImplementation files($junitClasspath) - isolatedTestImplementation files($junitClasspath) - testContainerImage(image('$image', 'test.cassandra.image')) + testImplementation(files(${junitClasspath()})) + add(declaredTest.implementationConfigurationName, files(${junitClasspath()})) + add(isolatedTest.implementationConfigurationName, files(${junitClasspath()})) + testContainerImage(image("$image", "test.cassandra.image")) + containerImage("declaredTest", image("$image", "test.cassandra.image")) } """.trimIndent(), ) @@ -318,6 +311,23 @@ class TestcontainersPluginTest { ) } + private fun junitClasspath() = + listOf( + "org.junit.jupiter.api.Test", + "org.junit.jupiter.engine.JupiterTestEngine", + "org.junit.platform.engine.TestEngine", + "org.junit.platform.launcher.Launcher", + "org.junit.platform.commons.JUnitException", + "org.opentest4j.AssertionFailedError", + ).joinToString(", ") { + "\"${Path.of( + Class + .forName(it) + .protectionDomain.codeSource.location + .toURI(), + )}\"" + } + private fun runner( vararg arguments: String, injectPluginClasspath: Boolean = true, @@ -337,52 +347,4 @@ class TestcontainersPluginTest { private fun run(vararg arguments: String) = runner(*arguments).build() private fun report() = directory.resolve("build/test-results/test/TEST-ImageTest.xml").toFile().readText() - - private fun manifest(character: String) = - """{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json","config":{"mediaType":"application/vnd.oci.image.config.v1+json","digest":"sha256:${character.repeat( - 64, - )}","size":2},"layers":[]}""" - - private fun registry(): HttpsServer { - val keystore = directory.resolve("registry.p12") - val keytool = Path.of(System.getProperty("java.home"), "bin", "keytool").toString() - val process = - ProcessBuilder( - keytool, - "-genkeypair", - "-alias", - "registry", - "-keyalg", - "RSA", - "-keystore", - keystore.toString(), - "-storepass", - "fixture", - "-keypass", - "fixture", - "-dname", - "CN=localhost", - "-validity", - "1", - "-noprompt", - ).redirectErrorStream(true).start() - val output = process.inputStream.bufferedReader().readText() - check(process.waitFor() == 0) { output } - val keys = KeyStore.getInstance("PKCS12") - Files.newInputStream(keystore).use { keys.load(it, "fixture".toCharArray()) } - val manager = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()) - manager.init(keys, "fixture".toCharArray()) - val context = SSLContext.getInstance("TLS") - context.init(manager.keyManagers, null, null) - return HttpsServer.create(InetSocketAddress("127.0.0.1", 0), 0).apply { - httpsConfigurator = HttpsConfigurator(context) - } - } - - private fun digest(body: String) = - "sha256:" + - MessageDigest - .getInstance("SHA-256") - .digest(body.toByteArray()) - .joinToString("") { "%02x".format(it) } } diff --git a/docs/how_to_smoke_test.md b/docs/how_to_smoke_test.md index ffb7e7f0f83..2fbbfcec774 100644 --- a/docs/how_to_smoke_test.md +++ b/docs/how_to_smoke_test.md @@ -238,16 +238,19 @@ static final SmokeServerApp sender = /* ... */; For application containers or dependencies such as RabbitMQ, apply `dd-trace-java.testcontainers` and declare the image in the smoke-test module: -```groovy +```kotlin +import datadog.buildlogic.testcontainers.image +import datadog.buildlogic.testcontainers.testContainerImage + plugins { - id 'dd-trace-java.module.smoke-test' - id 'dd-trace-java.testcontainers' + id("dd-trace-java.module.smoke-test") + id("dd-trace-java.testcontainers") } dependencies { - testImplementation group: 'org.testcontainers', name: 'rabbitmq', version: libs.versions.testcontainers.get() - testImplementation group: 'org.testcontainers', name: 'junit-jupiter', version: libs.versions.testcontainers.get() - testContainerImage(image('rabbitmq:3.12-management-alpine', 'test.rabbitmq.image')) + testImplementation("org.testcontainers:rabbitmq:${libs.versions.testcontainers.get()}") + testImplementation("org.testcontainers:junit-jupiter:${libs.versions.testcontainers.get()}") + testContainerImage(image("rabbitmq:3.12-management-alpine", "test.rabbitmq.image")) } ``` diff --git a/docs/how_to_test.md b/docs/how_to_test.md index b349a0a2f55..0400a23e47e 100644 --- a/docs/how_to_test.md +++ b/docs/how_to_test.md @@ -60,15 +60,18 @@ In order to identify such tests and avoid the continuous integration to fail, th Declare container images in the module's Gradle build so a changed image cannot silently reuse cached test results: -```groovy +```kotlin +import datadog.buildlogic.testcontainers.image +import datadog.buildlogic.testcontainers.testContainerImage + plugins { - id 'dd-trace-java.testcontainers' + id("dd-trace-java.testcontainers") } dependencies { - testImplementation libs.testcontainers - testImplementation group: 'com.redis.testcontainers', name: 'testcontainers-redis', version: '1.6.2' - testContainerImage(image('redis:7-alpine', 'test.redis.image')) + testImplementation(libs.testcontainers) + testImplementation("com.redis.testcontainers:testcontainers-redis:1.6.2") + testContainerImage(image("redis:7-alpine", "test.redis.image")) } ``` @@ -95,10 +98,11 @@ digests can reuse test results; changed digests select a different cache entry. Tags are refreshed even when Gradle reuses its configuration cache. Resolution failure stops the task rather than trusting an old result. -Each source set has a `ContainerImage` declaration method. Images follow -`implementation` configuration inheritance and reach the matching test task and -its forked companions. For a separate suite, use its own declaration, such as -`integrationTestContainerImage(...)`. Declarations apply to the whole task, including +Images declared with `testContainerImage` belong to the `test` source set, follow +`implementation` configuration inheritance and reach the matching test task and its forked companions. For a +separate suite, import `datadog.buildlogic.testcontainers.containerImage`, declare +its source set first, then use `containerImage("integrationTest", image(...))` in +`dependencies {}`. Declarations apply to the whole task, including when `--tests` selects only some classes. Run IDE tests through Gradle, or supply the image system properties explicitly. From a33de72ec4f23d1e12117312d6710354b2b14d8d Mon Sep 17 00:00:00 2001 From: Brice Dutheil Date: Thu, 24 Sep 2026 16:22:34 +0200 Subject: [PATCH 09/15] chore: Use Gradle configuration to dicover automatically --- build-logic/testcontainers/README.md | 201 +++++++------- .../testcontainers/ContainerImages.kt | 53 ++-- .../testcontainers/ImageResolver.kt | 37 ++- .../testcontainers/TestcontainersPlugin.kt | 84 +++--- .../TestcontainersPluginTest.kt | 260 ++++++++++++++++-- docs/how_to_smoke_test.md | 14 +- docs/how_to_test.md | 52 ++-- 7 files changed, 476 insertions(+), 225 deletions(-) diff --git a/build-logic/testcontainers/README.md b/build-logic/testcontainers/README.md index 750941d8503..52786aa6ffb 100644 --- a/build-logic/testcontainers/README.md +++ b/build-logic/testcontainers/README.md @@ -4,7 +4,6 @@ Apply this plugin only in modules that use containers: ```kotlin import datadog.buildlogic.testcontainers.image -import datadog.buildlogic.testcontainers.testContainerImage plugins { id("dd-trace-java.testcontainers") @@ -16,12 +15,13 @@ dependencies { ``` > [!NOTE] -> The imports expose the plugin's Kotlin functions in the build script; applying the -plugin alone does not import them. +> `testContainerImage` is a configuration. +> Note the `image` function import. -Keep the dedicated container type and pass the property through its `DockerImageName` -constructor. The compatibility declaration lets it accept a mirrored image without -changing the resolved reference: +Use `GenericContainer` or the dedicated container type and pass the relevant +system property through its `DockerImageName` constructor. Note the compatibility +declaration that let it accept a mirrored image without changing the resolved +reference: ```java import org.testcontainers.cassandra.CassandraContainer; @@ -32,108 +32,125 @@ CassandraContainer container = new CassandraContainer( .asCompatibleSubstituteFor("cassandra")); ``` -`testContainerImage` declares an image for the `test` source set. For a separate -suite, import `datadog.buildlogic.testcontainers.containerImage`, declare its source -set first, then use `containerImage("integrationTest", image("redis:7-alpine", "test.redis.image"))` -inside `dependencies {}`. -Images follow `implementation` configuration inheritance, so a `latestDepTest` suite extending -`testImplementation` inherits its images. The matching `Test` task, `forkedTest`, -and `ForkedTest` companions receive the properties. -Run IDE tests through Gradle, or supply the named image properties explicitly. +`testContainerImage` is a dependency-scope configuration: it stores declarations +and is neither resolvable nor consumable. `image("image", "system property name")` +creates a Gradle module dependency. +Note that these declarations stay outside Java classpaths. Before test task runs, the plugin +resolves the images digest. -Groovy builds use `dependencies { testContainerImage(image('cassandra:4', -'test.cassandra.image')) }` without imports. Each source set gets a dynamic -`ContainerImage` method in Groovy; Kotlin uses `containerImage` for custom -source sets. Both DSLs use the same declarations and validation. +The plugin identifies a test task's source sets from its `testClassesDirs` and +reads their image configurations. Forked or differently named tasks running the +same compiled tests therefore inherit the same images. Additional images can be +declared in `ContainerImage`. +In practice the plugin automatically creates an image configuration for each `*Implementation` +configuration, including those added by JVM Test Suites. For example: -This plugin only fingerprints images. Container concurrency remains controlled by -the existing, explicit `usesService(testcontainersLimit)` declarations in module -build scripts; applying this plugin does not add or remove them. - -## Resolution and test execution +```kotlin +testing { + suites { + register("integrationTest") { + useJUnitJupiter() + project.dependencies { + add("integrationTestContainerImage", image("redis:7-alpine", "test.redis.image")) + } + } + } +} +``` -For each selected test task with image declarations: - -1. Apply the task's effective Docker Hub prefix to image names without an explicit - registry. This selects the registry before resolving any digest. -2. Use Jib to fetch each tag's manifest from that registry and produce an immutable - `registry/repository@sha256:...` reference. Jib does not download image layers or - require a Docker daemon. Declarations already containing a digest skip this request. -3. Include the property names and resolved references in Gradle's test input - fingerprint, before the up-to-date and build-cache checks. -4. If the test needs to run, pass the same references as `-D=` - arguments to its JVM. Testcontainers then uses Docker to pull and start those - exact images. Cached test results require no container startup. - -The resolver shares each effective image's result within one build. The next build -resolves moving tags again, even when reusing the configuration cache. Unchanged -references allow `UP-TO-DATE`/`FROM-CACHE`; changed digests select a different cache -entry. The registry and repository are also part of the input, so switching hosts -changes the fingerprint even if both registries return the same digest. - -Moving tags need registry access even when Docker already has the image locally. -Resolution failure stops the test instead of trusting stale results. Unrelated -tasks and skipped tests perform no registry requests. - -Configuration uses `configureEach` and a project-local provider rather than -`afterEvaluate`. Late declarations, configuration inheritance, resource directories -and task environment settings are captured when Gradle queries the provider. -Only that configuration is cached; the nested input getter resolves images before -test cache lookup on every build. Isolated Projects has not been tested. +With the repository's legacy Groovy `addTestSuite` helper, declare the image in +the project's `dependencies` block: -## Local registries and CI mirrors +```groovy +addTestSuite('integrationTest') -Without a prefix, the example declaration resolves `cassandra:4` from Docker Hub. -CI sets `TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX=registry.ddbuild.io/images/mirror/` -in the [test job configuration](../../.gitlab-ci.yml), giving this flow: +dependencies { + integrationTestContainerImage(image('redis:7-alpine', 'test.redis.image')) +} +``` -```text -Local: cassandra:4 - -> registry-1.docker.io/library/cassandra@sha256:... +Automatic inheritance follows Java dependency configurations: if +`latestDepTestImplementation` extends `testImplementation`, its tests also read +`testContainerImage`. The plugin maps each configuration in that hierarchy to its +image counterpart. It also reads parents declared with `extendsFrom` on image +configurations: -CI: cassandra:4 - -> registry.ddbuild.io/images/mirror/cassandra:4 - -> registry.ddbuild.io/images/mirror/cassandra@sha256:... +```kotlin +val databaseImages = configurations.dependencyScope("databaseImages") +configurations.testContainerImage { + extendsFrom(databaseImages.get()) +} +dependencies { + add(databaseImages.name, image("postgres:16-alpine", "test.postgres.image")) +} ``` -CI fingerprints the mirror's content, which may differ from Docker Hub. The test -receives the fully qualified reference, so Testcontainers does not apply the prefix -again. This follows Testcontainers' [image substitution rules](https://java.testcontainers.org/features/image_name_substitution/). +The plugin ensures there’s only one image per system property for a given test task. +Declaring both `image("redis:7", "test.redis.image")` and `image("redis:8", "test.redis.image")` +for that task, directly or through inherited configurations, will fail. -The prefix comes from the task's `TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX`, then -`hub.image.name.prefix` in the user's `.testcontainers.properties` and source-set -`testcontainers.properties`. Explicit task environment overrides and removals are -respected. Changing the inherited `TESTCONTAINERS_` environment variables invalidates -the configuration cache. +Groovy DSL is declared the same `dependencies { testContainerImage(image('cassandra:4', 'test.cassandra.image')) }`. -Explicit registry names bypass the prefix, including explicit Docker Hub hosts. -For example, Pub/Sub's `gcr.io` and WebSphere's `icr.io` declarations retain their -registries. Other mappings belong in the declaration: -[JDBC's SQL Server declaration](../../dd-java-agent/instrumentation/jdbc/build.gradle) -selects `mcr.microsoft.com/mssql/server:latest` locally and -`registry.ddbuild.io/images/mirror/sqlserver:latest` when `CI` is present. This -replaces runtime substitution so Gradle fingerprints the image the test will use. +## Resolution and test execution -Custom image substitutors and `*.container.image` overrides in these configuration -sources are rejected because they could replace the resolved digest at runtime. -Image substitution supplied by dependency JAR resources is not supported. +The plugin resolves declared tags to immutable reference like `registry/repository@sha256:...` +before Gradle checks the build-cache for reusable test results. These references are +both test inputs and system property values, so actual code in the tests use the exact +images. -Private registries use Docker's `config.json` and credential helpers from the Gradle -process, honoring `DOCKER_CONFIG`. Remote registries require TLS; loopback registries -also permit local development certificates and HTTP. +Like when java dependencies are declared with `+`, the tags are resolved on each invocation, +including when reusing the configuration cache. With a build cache this means unchanged +references allow to safely reuse test results, while Gradle is blind when the image is +resolving within test code. -Jib's dependencies are relocated inside the plugin JAR so older libraries exported -by `buildSrc` cannot override its HTTP client. Tests load that JAR through an -included build with an older HttpClient on the `buildSrc` classpath. +> [!NOTE] +> Only explicitly declared images can be tracked. Implicit Testcontainers helpers such as +> Alpine/Ryuk are not included. This can't be avoided. -Only declared images are tracked. This migration covers Cassandra, Pub/Sub, -JDBC, Vert.x MySQL/PostgreSQL and WebSphere fixtures. Other fixtures and implicit -Testcontainers helpers such as Alpine/Ryuk require separate adoption. +## Local registries and CI mirrors -## Verification +This plugin honors `TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX` environment variable. -Run the hermetic registry and Gradle cache tests with: +Locally, when asking for `cassandra:4` it is resolved from Docker Hub. +However, on [CI](../../.gitlab-ci.yml), the env var `TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX=...` is set, +so the plugin resolves and fingerprints from the prefix, instead, e.g.: -```shell -build-brief ./gradlew -p build-logic :testcontainers:test :testcontainers:validatePlugins +```text +Local: registry-1.docker.io/library/cassandra@sha256:... +CI: registry.ddbuild.io/images/mirror/cassandra@sha256:... ``` + +More precisely: +- Prefix precedence: the test task's `TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX`, then + `hub.image.name.prefix` in `~/.testcontainers.properties`, then + `testcontainers.properties` in test classpath directories. +- Explicit registry hosts bypass the prefix. Other mappings belong in the image + declaration, as in [JDBC's SQL Server example](../../dd-java-agent/instrumentation/jdbc/build.gradle). +- Private registries use Docker's `config.json` and credential helpers, honoring + `DOCKER_CONFIG`. Remote registries require TLS; loopback registries also allow + local development certificates and HTTP. + +## Testcontainers configuration restrictions + +Testcontainers supports [configuration properties and environment variables](https://java.testcontainers.org/features/configuration/) +that change its runtime behavior. In particular, its [image-substitution settings](https://java.testcontainers.org/features/image_name_substitution/) +can replace the images used when containers start. + +> [!IMPORTANT] +> This plugin forbids custom image substitutors (`image.substitutor`) and all +> `*.container.image` overrides: **the test task fails when these settings are +> detected**. Runtime replacements could make the tested image differ from the one +> in Gradle's cache key. This restriction includes helper-image overrides such as +> `ryuk.container.image`. + +The check covers environment variables, `~/.testcontainers.properties`, and +`testcontainers.properties` in test classpath directories. The Docker Hub prefix +described above remains supported. + +For declared test images, choose the replacement directly in the declaration, +for example `image("registry.example/redis:7", "test.redis.image")`. This keeps the +image used by the test consistent with the image in Gradle's cache key. + +The plugin does not inspect `testcontainers.properties` inside dependency JARs. +Testcontainers can still load substitutions from those files; they escape this +check and can invalidate cache correctness. diff --git a/build-logic/testcontainers/src/main/kotlin/datadog/buildlogic/testcontainers/ContainerImages.kt b/build-logic/testcontainers/src/main/kotlin/datadog/buildlogic/testcontainers/ContainerImages.kt index 51dabfb0f99..fed8c34dcb0 100644 --- a/build-logic/testcontainers/src/main/kotlin/datadog/buildlogic/testcontainers/ContainerImages.kt +++ b/build-logic/testcontainers/src/main/kotlin/datadog/buildlogic/testcontainers/ContainerImages.kt @@ -1,48 +1,27 @@ package datadog.buildlogic.testcontainers +import org.gradle.api.InvalidUserDataException +import org.gradle.api.artifacts.ExternalModuleDependency import org.gradle.api.artifacts.dsl.DependencyHandler -import org.gradle.api.plugins.ExtensionAware +import org.gradle.api.attributes.Attribute + +internal const val IMAGE_DEPENDENCY_GROUP = "datadog.container-image" +internal val IMAGE_REFERENCE = Attribute.of("datadog.container-image.reference", String::class.java) /** Creates an image declaration for the test JVM property that consumes it. */ -fun image( +fun DependencyHandler.image( reference: String, systemProperty: String, -): ContainerImage = ContainerImage(reference, systemProperty) - -/** Declares a container image for the test source set. */ -fun DependencyHandler.testContainerImage(image: ContainerImage) = containerImage("test", image) - -/** Declares a container image for an existing source set. */ -fun DependencyHandler.containerImage( - sourceSet: String, - image: ContainerImage, -) { - val images = (this as ExtensionAware).extensions.extraProperties.get("testContainerImages") as ContainerImages - images.add(sourceSet, image) -} - -internal class ContainerImages { - val declarations = mutableMapOf>() - - fun add( - sourceSet: String, - image: ContainerImage, - ) { - val images = requireNotNull(declarations[sourceSet]) { "Unknown container image source set '$sourceSet'" } - require(images.putIfAbsent(image.systemProperty, image.reference) == null) { - "Container image system property '${image.systemProperty}' is declared twice in $sourceSet" - } +): ExternalModuleDependency { + if (reference.isBlank()) { + throw InvalidUserDataException("Container image reference must not be empty") + } + if (!systemProperty.matches(Regex("[A-Za-z0-9_.-]+"))) { + throw InvalidUserDataException("Invalid container image system property: $systemProperty") } -} -data class ContainerImage( - val reference: String, - val systemProperty: String, -) { - init { - require(reference.isNotBlank()) { "Container image reference must not be empty" } - require(systemProperty.matches(Regex("[A-Za-z0-9_.-]+"))) { - "Invalid container image system property: $systemProperty" - } + // These logical dependencies are read from declaration-only configurations, never resolved by Gradle. + return (create("$IMAGE_DEPENDENCY_GROUP:$systemProperty") as ExternalModuleDependency).apply { + attributes { attribute(IMAGE_REFERENCE, reference) } } } diff --git a/build-logic/testcontainers/src/main/kotlin/datadog/buildlogic/testcontainers/ImageResolver.kt b/build-logic/testcontainers/src/main/kotlin/datadog/buildlogic/testcontainers/ImageResolver.kt index b91de5be102..d348b4e6b85 100644 --- a/build-logic/testcontainers/src/main/kotlin/datadog/buildlogic/testcontainers/ImageResolver.kt +++ b/build-logic/testcontainers/src/main/kotlin/datadog/buildlogic/testcontainers/ImageResolver.kt @@ -7,6 +7,7 @@ import com.google.cloud.tools.jib.frontend.CredentialRetrieverFactory import com.google.cloud.tools.jib.http.FailoverHttpClient import com.google.cloud.tools.jib.registry.RegistryClient import org.gradle.api.GradleException +import org.gradle.api.InvalidUserDataException import org.gradle.api.services.BuildService import org.gradle.api.services.BuildServiceParameters import java.io.File @@ -14,7 +15,9 @@ import java.nio.file.Paths import java.util.Properties import java.util.concurrent.ConcurrentHashMap -/** Resolves manifests without a Docker daemon or image-layer downloads. Memoized for this build only. */ +/** + * Resolves manifests without a Docker daemon or image-layer downloads. Memoized for this build only. + */ abstract class ImageResolver : BuildService { private val resolved = ConcurrentHashMap() @@ -36,9 +39,12 @@ abstract class ImageResolver : BuildService { environment.any { (key, value) -> (key == "TESTCONTAINERS_IMAGE_SUBSTITUTOR" || key.endsWith("_CONTAINER_IMAGE")) && value.isNotBlank() } - check(!customSubstitution) { - "Custom Testcontainers image substitutions can replace a fingerprinted digest; move image overrides into testContainerImage declarations" + if (customSubstitution) { + throw InvalidUserDataException( + "Custom Testcontainers image substitutions can replace a fingerprinted digest; move image overrides into testContainerImage declarations", + ) } + val prefix = environment["TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX"]?.takeIf { it.isNotEmpty() } ?: configuration.getProperty("hub.image.name.prefix", "") @@ -57,6 +63,31 @@ abstract class ImageResolver : BuildService { return resolved.computeIfAbsent(effective) { resolveManifest(it) } } + /** + * Image digest resolution. + * + * **Why jib?** + * + * Jib provides a daemonless registry client that retrieves the manifest without downloading layers. + * However, it relies on Jib non-public API. But this is preferable to maintaining registry authentication + * code in this case. + * [Jib Core](https://github.com/GoogleContainerTools/jib/blob/master/jib-core/README.md) + * [RegistryClient](https://github.com/GoogleContainerTools/jib/blob/master/jib-core/src/main/java/com/google/cloud/tools/jib/registry/RegistryClient.java) + * + * Considered alternatives: + * - Testcontainers uses docker-java under the hood, it reuses the local image cache + * or pull the complete image, and returns the image name rather than the remote manifest digest. + * That is too late and too expensive for Gradle input fingerprinting. + * [RemoteDockerImage](https://github.com/testcontainers/testcontainers-java/blob/1.21.4/core/src/main/java/org/testcontainers/images/RemoteDockerImage.java) + * - Docker’s `/distribution/{name}/json` endpoint can obtain the registry digest without + * downloading layers. But it still requires a running Docker daemon, and docker-java 3.7.1 + * has no typed command for it. [Docker Engine API](https://docs.docker.com/reference/api/engine/version/v1.40/) + * - A direct HTTP implementation is possible, but it must implement registry naming, + * OCI/Docker media types, bearer-token challenges, basic authentication, Docker credential helpers, + * redirects, TLS, and digest validation. + * [Registry authentication](https://docs.docker.com/reference/api/registry/auth/) + * [Registry API specification](https://github.com/distribution/distribution/blob/main/docs/content/spec/api.md) + */ private fun resolveManifest(reference: String): String { try { val image = ImageReference.parse(reference) diff --git a/build-logic/testcontainers/src/main/kotlin/datadog/buildlogic/testcontainers/TestcontainersPlugin.kt b/build-logic/testcontainers/src/main/kotlin/datadog/buildlogic/testcontainers/TestcontainersPlugin.kt index 25247d9c9d6..36099333799 100644 --- a/build-logic/testcontainers/src/main/kotlin/datadog/buildlogic/testcontainers/TestcontainersPlugin.kt +++ b/build-logic/testcontainers/src/main/kotlin/datadog/buildlogic/testcontainers/TestcontainersPlugin.kt @@ -1,9 +1,10 @@ package datadog.buildlogic.testcontainers import groovy.lang.Closure +import org.gradle.api.InvalidUserDataException import org.gradle.api.Plugin import org.gradle.api.Project -import org.gradle.api.plugins.ExtensionAware +import org.gradle.api.artifacts.ExternalModuleDependency import org.gradle.api.tasks.SourceSetContainer import org.gradle.api.tasks.testing.Test import org.gradle.kotlin.dsl.getByType @@ -13,16 +14,29 @@ import java.io.File /** Declares container images alongside the dependencies of the test suite that consumes them. */ class TestcontainersPlugin : Plugin { override fun apply(project: Project) { - val containerImages = ContainerImages() - val dependencyDsl = (project.dependencies as ExtensionAware).extensions.extraProperties - dependencyDsl.set("testContainerImages", containerImages) - dependencyDsl.set( + val configurations = project.configurations + // dependencyScope express exactly what is needed it is equivalent to + // configurations.register("testContainerImage") { + // isCanBeResolved = false + // isCanBeConsumed = false + // } + // Additionally, it is lazy. + configurations.dependencyScope("testContainerImage") + configurations.named { it.endsWith("Implementation") }.all { + val imageConfigurationName = "${name.removeSuffix("Implementation")}ContainerImage" + if (imageConfigurationName !in configurations.names) { + configurations.dependencyScope(imageConfigurationName) + } + } + + // Groovy needs a bridge to call the Kotlin receiver extension with the same syntax. + project.dependencies.extensions.extraProperties.set( "image", - object : Closure(null) { + object : Closure(null) { fun doCall( reference: String, systemProperty: String, - ): ContainerImage = image(reference, systemProperty) + ): ExternalModuleDependency = project.dependencies.image(reference, systemProperty) }, ) @@ -34,44 +48,46 @@ class TestcontainersPlugin : Plugin { project.pluginManager.withPlugin("java") { val sourceSets = project.extensions.getByType() - sourceSets.all { - containerImages.declarations.getOrPut(name) { linkedMapOf() } - val sourceSetName = name - dependencyDsl.set( - "${name}ContainerImage", - object : Closure(null) { - fun doCall(image: ContainerImage) { - containerImages.add(sourceSetName, image) - } - }, - ) - } + project.tasks.withType().configureEach { // ProviderFactory.provider snapshots project-local configuration for the configuration cache. // Keep registry access in the nested input getter, so every build refreshes moving tags. val containers = project.providers.provider { - val suiteName = if (name == "forkedTest") "test" else name.removeSuffix("ForkedTest") - val suite = - sourceSets.findByName(name) ?: sourceSets.findByName(suiteName) - ?: return@provider null - val hierarchy = project.configurations.getByName(suite.implementationConfigurationName).hierarchy + val testClasses = testClassesDirs.files + val implementationNames = + sourceSets + .filter { suite -> + suite.output.classesDirs.files + .any(testClasses::contains) + }.map { it.implementationConfigurationName } + .ifEmpty { listOf("${name}Implementation") } + val hierarchy = implementationNames.flatMap { configurations.findByName(it)?.hierarchy.orEmpty() } + val imageConfigurationNames = + setOf("${name}ContainerImage") + + hierarchy.map { "${it.name.removeSuffix("Implementation")}ContainerImage" } + val images = linkedMapOf() - val inheritedSourceSets = - sourceSets.filter { sourceSet -> - hierarchy.any { it.name == sourceSet.implementationConfigurationName } - } - inheritedSourceSets.forEach { sourceSet -> - containerImages.declarations[sourceSet.name]?.forEach { (property, reference) -> - require(images.putIfAbsent(property, reference).let { it == null || it == reference }) { - "Conflicting container images for '$property' in $path" + imageConfigurationNames.mapNotNull(configurations::findByName).forEach { configuration -> + configuration.allDependencies.forEach { dependency -> + val reference = (dependency as? ExternalModuleDependency)?.attributes?.getAttribute(IMAGE_REFERENCE) + if (dependency.group != IMAGE_DEPENDENCY_GROUP || reference == null) { + throw InvalidUserDataException("Use image(reference, systemProperty) for dependencies in ${configuration.name}") + } + + val previous = images.putIfAbsent(dependency.name, reference) + if (previous != null && previous != reference) { + throw InvalidUserDataException("Conflicting container images for '${dependency.name}' in $path") } } } + if (images.isNotEmpty()) { val configurationFiles = listOf(File(System.getProperty("user.home"), ".testcontainers.properties")) + - inheritedSourceSets.flatMap { it.resources.srcDirs }.map { File(it, "testcontainers.properties") } + // Keep output directories that processResources has not created yet. + classpath.files.map { File(it, "testcontainers.properties") }.sortedBy { it.toURI().toString() } + // Track inherited values for configuration-cache invalidation, but use the task's // effective environment below so explicit overrides and removals are respected. project.providers.environmentVariablesPrefixedBy("TESTCONTAINERS_").get() @@ -81,11 +97,13 @@ class TestcontainersPlugin : Plugin { it == "TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX" || it == "TESTCONTAINERS_IMAGE_SUBSTITUTOR" || (it.startsWith("TESTCONTAINERS_") && it.endsWith("_CONTAINER_IMAGE")) }.mapValues { it.value.toString() } + ContainerImageInputs(images, imageEnvironment, configurationFiles, resolver) } else { null } } + jvmArgumentProviders.add(ContainerImageArguments(containers)) } } diff --git a/build-logic/testcontainers/src/test/kotlin/datadog/buildlogic/testcontainers/TestcontainersPluginTest.kt b/build-logic/testcontainers/src/test/kotlin/datadog/buildlogic/testcontainers/TestcontainersPluginTest.kt index 55fd6f0e0c7..6572d1545a2 100644 --- a/build-logic/testcontainers/src/test/kotlin/datadog/buildlogic/testcontainers/TestcontainersPluginTest.kt +++ b/build-logic/testcontainers/src/test/kotlin/datadog/buildlogic/testcontainers/TestcontainersPluginTest.kt @@ -22,40 +22,74 @@ class TestcontainersPluginTest { @Test fun `moving images are refreshed before cache lookup including configuration cache reuse`() { fixture(registry.image) + assertThat(run("help").task(":help")?.outcome).isEqualTo(TaskOutcome.SUCCESS) assertThat(registry.requestCount).isZero() + assertThat(run("test", "-PskipTests").task(":test")?.outcome).isEqualTo(TaskOutcome.SKIPPED) assertThat(registry.requestCount).isZero() assertThat(run("test").task(":test")?.outcome).isEqualTo(TaskOutcome.SUCCESS) - assertThat(report()).contains("library/cassandra@${registry.digest}") + assertThat(testReport()).content().contains("library/cassandra@${registry.digest}") + val firstRequests = registry.requestCount val warm = run("test") + assertThat(warm.output).contains("Reusing configuration cache") assertThat(warm.task(":test")?.outcome).isEqualTo(TaskOutcome.UP_TO_DATE) assertThat(registry.requestCount).isGreaterThan(firstRequests) registry.imageVersion = 2 val changed = run("test") + assertThat(changed.output).contains("Reusing configuration cache") assertThat(changed.task(":test")?.outcome).isEqualTo(TaskOutcome.SUCCESS) - assertThat(report()).contains("library/cassandra@${registry.digest}") + assertThat(testReport()).content().contains("library/cassandra@${registry.digest}") run("clean") val restored = run("test") + assertThat(restored.output).contains("Reusing configuration cache") assertThat(restored.task(":test")?.outcome).isEqualTo(TaskOutcome.FROM_CACHE) - assertThat(report()).contains("library/cassandra@${registry.digest}") + assertThat(testReport()).content().contains("library/cassandra@${registry.digest}") + + val suites = run("forkedTest", "latestDepTest", "latestDepTestForkedTest", "latestDepForkedTest", "replayedTests") - assertThat(run("latestDepTest", "latestDepTestForkedTest").task(":latestDepTestForkedTest")?.outcome) + assertThat(suites.task(":replayedTests")?.outcome) .isIn(TaskOutcome.SUCCESS, TaskOutcome.FROM_CACHE) - assertThat(directory.resolve("build/test-results/latestDepTest/TEST-ImageTest.xml").toFile().readText()) + assertThat(testReport("forkedTest", "ImageForkedTest")) + .content() + .contains("library/cassandra@${registry.digest}") + assertThat(testReport("forkedTest", "ImageTest")).doesNotExist() + assertThat(testReport(testClass = "ImageForkedTest")).doesNotExist() + assertThat(testReport("latestDepTest")) + .content() + .contains("library/cassandra@${registry.digest}") + assertThat(testReport("latestDepTestForkedTest", "ImageForkedTest")) + .content() .contains("library/cassandra@${registry.digest}") + assertThat(testReport("replayedTests")) + .content() + .contains("library/cassandra@${registry.digest}") + assertThat(testReport("latestDepForkedTest", "ImageForkedTest")) + .content() + .contains("library/cassandra@${registry.digest}") + + val reusedSuites = run("forkedTest", "latestDepTest", "latestDepTestForkedTest", "latestDepForkedTest", "replayedTests") + + assertThat(reusedSuites.output).contains("Reusing configuration cache") + assertThat(reusedSuites.task(":forkedTest")?.outcome).isEqualTo(TaskOutcome.UP_TO_DATE) + assertThat(reusedSuites.task(":latestDepTestForkedTest")?.outcome).isEqualTo(TaskOutcome.UP_TO_DATE) + assertThat(reusedSuites.task(":replayedTests")?.outcome).isEqualTo(TaskOutcome.UP_TO_DATE) + assertThat(run("declaredTest").task(":declaredTest")?.outcome).isIn(TaskOutcome.SUCCESS, TaskOutcome.FROM_CACHE) - assertThat(directory.resolve("build/test-results/declaredTest/TEST-ImageTest.xml").toFile().readText()) + assertThat(testReport("declaredTest")) + .content() .contains("library/cassandra@${registry.digest}") + val requestsBeforeUnrelatedSuite = registry.requestCount assertThat(run("isolatedTest").task(":isolatedTest")?.outcome).isEqualTo(TaskOutcome.SUCCESS) + assertThat(run("testForkedTest").task(":testForkedTest")?.outcome).isIn(TaskOutcome.SUCCESS, TaskOutcome.FROM_CACHE) assertThat(run("emptyTest").task(":emptyTest")?.outcome).isEqualTo(TaskOutcome.NO_SOURCE) assertThat(registry.requestCount).isEqualTo(requestsBeforeUnrelatedSuite) @@ -64,26 +98,134 @@ class TestcontainersPluginTest { .contains("Cannot resolve test container image") } + @Test + fun `image configurations support shared parents and tests without matching source sets`() { + val image = "cassandra@sha256:${"a".repeat(64)}" + val extraImage = "redis@sha256:${"b".repeat(64)}" + fixture(image) + directory.resolve("build.gradle.kts").toFile().appendText( + """ + + val databaseImages = configurations.dependencyScope("databaseImages") + val sharedImplementation = configurations.dependencyScope("sharedImplementation") + + configurations.named("sharedContainerImage") { + extendsFrom(databaseImages.get()) + } + configurations.testContainerImage { dependencies.clear() } + configurations.testImplementation { extendsFrom(sharedImplementation.get()) } + configurations.dependencyScope("databaseCheckContainerImage") { + extendsFrom(databaseImages.get()) + } + + dependencies { + add(databaseImages.name, image("$image", "test.cassandra.image")) + add("databaseCheckContainerImage", image("$extraImage", "test.redis.image")) + } + + tasks.register("databaseCheck") { + testClassesDirs = sourceSets.test.get().output.classesDirs + classpath = sourceSets.test.get().runtimeClasspath + } + """.trimIndent(), + ) + + val first = run("test", "databaseCheck") + + assertThat(first.task(":test")?.outcome).isEqualTo(TaskOutcome.SUCCESS) + assertThat(first.task(":databaseCheck")?.outcome).isIn(TaskOutcome.SUCCESS, TaskOutcome.FROM_CACHE) + assertThat(testReport()).content().contains("registry-1.docker.io/library/$image") + assertThat(testReport("databaseCheck")) + .content() + .contains("registry-1.docker.io/library/$image") + .contains("registry-1.docker.io/library/$extraImage") + + val reused = run("test", "databaseCheck") + + assertThat(reused.output).contains("Reusing configuration cache") + assertThat(reused.task(":test")?.outcome).isEqualTo(TaskOutcome.UP_TO_DATE) + assertThat(reused.task(":databaseCheck")?.outcome).isEqualTo(TaskOutcome.UP_TO_DATE) + assertThat(registry.requestCount).isZero() + + run("databaseCheck") + Files.createDirectories(directory.resolve("src/test/resources")) + directory + .resolve("src/test/resources/testcontainers.properties") + .toFile() + .writeText("image.substitutor=untracked.CustomSubstitutor\n") + + val failed = runner("databaseCheck").buildAndFail() + + assertThat(failed.output) + .contains("Reusing configuration cache") + .contains("org.gradle.api.InvalidUserDataException: Custom Testcontainers image substitutions") + .contains("move image overrides into testContainerImage declarations") + } + + @Test + fun `tasks running multiple source sets reject conflicting inherited images`() { + fixture("cassandra@sha256:${"a".repeat(64)}") + directory.resolve("build.gradle.kts").toFile().appendText( + """ + + configurations.named("declaredTestContainerImage") { dependencies.clear() } + dependencies { + add("declaredTestContainerImage", image("cassandra@sha256:${"b".repeat(64)}", "test.cassandra.image")) + } + + tasks.register("combinedTests") { + testClassesDirs = files(sourceSets.test.get().output.classesDirs, sourceSets["declaredTest"].output.classesDirs) + classpath = sourceSets.test.get().runtimeClasspath + sourceSets["declaredTest"].runtimeClasspath + } + """.trimIndent(), + ) + + assertThat(runner("combinedTests").buildAndFail().output) + .contains("org.gradle.api.InvalidUserDataException: Conflicting container images for 'test.cassandra.image'") + assertThat(registry.requestCount).isZero() + } + + @Test + fun `different image attributes for the same property are rejected`() { + fixture("cassandra@sha256:${"a".repeat(64)}") + directory.resolve("build.gradle.kts").toFile().appendText( + """ + + dependencies { + testContainerImage(image("cassandra@sha256:${"b".repeat(64)}", "test.cassandra.image")) + } + """.trimIndent(), + ) + + assertThat(runner("test").buildAndFail().output) + .contains("org.gradle.api.InvalidUserDataException: Conflicting container images for 'test.cassandra.image'") + assertThat(registry.requestCount).isZero() + } + @Test fun `hub prefix is resolved and pinned references require no registry`() { val digest = "sha256:${"a".repeat(64)}" fixture("cassandra@$digest") + val result = runner("test") .withEnvironment( System.getenv() + ("TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX" to "mirror.example/team/"), ).build() + assertThat(result.task(":test")?.outcome).isEqualTo(TaskOutcome.SUCCESS) - assertThat(report()).contains("mirror.example/team/cassandra@$digest") + assertThat(testReport()).content().contains("mirror.example/team/cassandra@$digest") + val changed = runner("test") .withEnvironment( System.getenv() + ("TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX" to "another.example/team/"), ).build() + assertThat(changed.task(":test")?.outcome).isEqualTo(TaskOutcome.SUCCESS) - assertThat(report()).contains("another.example/team/cassandra@$digest") + assertThat(testReport()).content().contains("another.example/team/cassandra@$digest") directory.resolve("build.gradle.kts").toFile().appendText( """ @@ -94,13 +236,16 @@ class TestcontainersPluginTest { } """.trimIndent(), ) + assertThat(run("test").task(":test")?.outcome).isEqualTo(TaskOutcome.SUCCESS) - assertThat(report()).contains("task.example/cassandra@$digest") + assertThat(testReport()).content().contains("task.example/cassandra@$digest") + Files.createDirectories(directory.resolve("lateResources")) directory .resolve("lateResources/testcontainers.properties") .toFile() .writeText("image.substitutor=untracked.CustomSubstitutor\n") + assertThat(runner("test").buildAndFail().output) .contains("move image overrides into testContainerImage declarations") } @@ -124,16 +269,20 @@ class TestcontainersPluginTest { "TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX" to "removed.example/team/", "TESTCONTAINERS_IMAGE_SUBSTITUTOR" to "removed.CustomSubstitutor", ) + val first = runner("test").withEnvironment(environment).build() + assertThat(first.task(":test")?.outcome).isEqualTo(TaskOutcome.SUCCESS) - assertThat(report()).contains("registry-1.docker.io/library/cassandra@$digest") + assertThat(testReport()).content().contains("registry-1.docker.io/library/cassandra@$digest") + val reused = runner("test").withEnvironment(environment).build() + assertThat(reused.output).contains("Reusing configuration cache") assertThat(reused.task(":test")?.outcome).isEqualTo(TaskOutcome.UP_TO_DATE) } @Test - fun `Groovy dependency extension methods are available for each source set`() { + fun `one can write Groovy DSL dependency`() { val image = "cassandra@sha256:${"a".repeat(64)}" fixture(image) Files.delete(directory.resolve("build.gradle.kts")) @@ -143,15 +292,18 @@ class TestcontainersPluginTest { id 'java' id 'dd-trace-java.testcontainers' } + sourceSets { integrationTest { java.srcDirs = sourceSets.test.java.srcDirs } } + dependencies { testImplementation files(${junitClasspath()}) integrationTestImplementation files(${junitClasspath()}) testContainerImage(image('$image', 'test.cassandra.image')) integrationTestContainerImage(image('$image', 'test.cassandra.image')) } + tasks.register('integrationTest', Test) { testClassesDirs = sourceSets.integrationTest.output.classesDirs classpath = sourceSets.integrationTest.runtimeClasspath @@ -159,13 +311,18 @@ class TestcontainersPluginTest { tasks.withType(Test).configureEach { useJUnitPlatform() } """.trimIndent(), ) + val result = run("test", "integrationTest") + assertThat(result.task(":test")?.outcome).isIn(TaskOutcome.SUCCESS, TaskOutcome.FROM_CACHE) assertThat(result.task(":integrationTest")?.outcome).isIn(TaskOutcome.SUCCESS, TaskOutcome.FROM_CACHE) - assertThat(report()).contains("registry-1.docker.io/library/$image") - assertThat(directory.resolve("build/test-results/integrationTest/TEST-ImageTest.xml").toFile().readText()) + assertThat(testReport()).content().contains("registry-1.docker.io/library/$image") + assertThat(testReport("integrationTest")) + .content() .contains("registry-1.docker.io/library/$image") + val reused = run("test", "integrationTest") + assertThat(reused.output).contains("Reusing configuration cache") assertThat(reused.task(":test")?.outcome).isEqualTo(TaskOutcome.UP_TO_DATE) assertThat(reused.task(":integrationTest")?.outcome).isEqualTo(TaskOutcome.UP_TO_DATE) @@ -175,9 +332,14 @@ class TestcontainersPluginTest { fun `bearer authentication resolves manifests with an older HttpClient in buildSrc`() { registry.requireAuthentication = true fixture(registry.image) + // Reproduce the parent classloader supplied by Aether in the real buildSrc. + val httpClasspathFiles = + System.getProperty("test.buildSrc.classpath").split(File.pathSeparator).map(::File) + assertThat(httpClasspathFiles).allSatisfy { assertThat(it).isFile() } + assertThat(httpClasspathFiles).anySatisfy { assertThat(it.name).startsWith("httpclient-4.3.5") } val httpClasspath = - System.getProperty("test.buildSrc.classpath").split(File.pathSeparator).joinToString(", ") { "\"$it\"" } + httpClasspathFiles.joinToString(", ") { "\"$it\"" } Files.createDirectories(directory.resolve("buildSrc/src/main/java")) directory.resolve("buildSrc/src/main/java/BuildLogic.java").toFile().writeText("public class BuildLogic {}\n") directory.resolve("buildSrc/build.gradle.kts").toFile().writeText( @@ -186,6 +348,7 @@ class TestcontainersPluginTest { dependencies { implementation(files($httpClasspath)) } """.trimIndent(), ) + // Load as an included build instead of using TestKit's injected plugin classpath. val metadata = Properties() javaClass.classLoader.getResourceAsStream("plugin-under-test-metadata.properties")!!.use { metadata.load(it) } @@ -208,10 +371,11 @@ class TestcontainersPluginTest { ) val settings = directory.resolve("settings.gradle.kts").toFile() settings.writeText("pluginManagement { includeBuild(\"plugin\") }\n" + settings.readText()) + assertThat(runner("test", injectPluginClasspath = false).build().task(":test")?.outcome).isEqualTo(TaskOutcome.SUCCESS) assertThat(registry.tokenRequests.get()).isPositive() assertThat(registry.authorizedRequests.get()).isPositive() - assertThat(report()).contains(registry.digest) + assertThat(testReport()).content().contains(registry.digest) } private fun fixture(image: String) { @@ -223,85 +387,137 @@ class TestcontainersPluginTest { ) directory.resolve("build.gradle.kts").toFile().writeText( """ - import datadog.buildlogic.testcontainers.containerImage import datadog.buildlogic.testcontainers.image - import datadog.buildlogic.testcontainers.testContainerImage plugins { java id("dd-trace-java.testcontainers") } + val latestDepTest = sourceSets.create("latestDepTest") { java.setSrcDirs(sourceSets.test.get().java.srcDirs) } + val latestDepForkedTest = sourceSets.create("latestDepForkedTest") { + java.setSrcDirs(sourceSets.test.get().java.srcDirs) + } val declaredTest = sourceSets.create("declaredTest") { java.setSrcDirs(sourceSets.test.get().java.srcDirs) } val isolatedTest = sourceSets.create("isolatedTest") val emptyTest = sourceSets.create("emptyTest") + + tasks.register("forkedTest") { + testClassesDirs = sourceSets.test.get().output.classesDirs + classpath = sourceSets.test.get().runtimeClasspath + } + tasks.register("latestDepTest") { testClassesDirs = latestDepTest.output.classesDirs classpath = latestDepTest.runtimeClasspath } + tasks.register("latestDepTestForkedTest") { testClassesDirs = latestDepTest.output.classesDirs classpath = latestDepTest.runtimeClasspath } + + tasks.register("replayedTests") { + testClassesDirs = latestDepTest.output.classesDirs + classpath = latestDepTest.runtimeClasspath + } + + tasks.register("latestDepForkedTest") { + testClassesDirs = latestDepForkedTest.output.classesDirs + classpath = latestDepForkedTest.runtimeClasspath + } + tasks.register("declaredTest") { testClassesDirs = declaredTest.output.classesDirs classpath = declaredTest.runtimeClasspath } + tasks.register("isolatedTest") { testClassesDirs = isolatedTest.output.classesDirs classpath = isolatedTest.runtimeClasspath } + + tasks.register("testForkedTest") { + testClassesDirs = isolatedTest.output.classesDirs + classpath = isolatedTest.runtimeClasspath + } + tasks.register("emptyTest") { testClassesDirs = emptyTest.output.classesDirs classpath = emptyTest.runtimeClasspath } + tasks.withType().configureEach { useJUnitPlatform() val skip = providers.gradleProperty("skipTests") onlyIf { !skip.isPresent } + + // Match dd-trace-java.configure-tests: split regular and forked test classes. + if (name.startsWith("forkedTest") || name.endsWith("ForkedTest")) { + setExcludes(emptyList()) + setIncludes(listOf("**/*ForkedTest*")) + forkEvery = 1 + } else { + exclude("**/*ForkedTest*") + failOnNoDiscoveredTests = false + } } + // Plugins must see declarations and inheritance added after tasks are realized. tasks.named("test").get() tasks.named("latestDepTest").get() tasks.named("latestDepTestForkedTest").get() + tasks.named("replayedTests").get() configurations.named(latestDepTest.implementationConfigurationName) { extendsFrom(configurations.testImplementation.get()) } + configurations.named(latestDepForkedTest.implementationConfigurationName) { + extendsFrom(configurations.getByName(latestDepTest.implementationConfigurationName)) + } configurations.named(emptyTest.implementationConfigurationName) { extendsFrom(configurations.testImplementation.get()) } + dependencies { testImplementation(files(${junitClasspath()})) add(declaredTest.implementationConfigurationName, files(${junitClasspath()})) add(isolatedTest.implementationConfigurationName, files(${junitClasspath()})) testContainerImage(image("$image", "test.cassandra.image")) - containerImage("declaredTest", image("$image", "test.cassandra.image")) + add("declaredTestContainerImage", image("$image", "test.cassandra.image")) } """.trimIndent(), ) + Files.createDirectories(directory.resolve("src/test/java")) directory.resolve("src/test/java/ImageTest.java").toFile().writeText( """ import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertTrue; + public class ImageTest { @Test public void imageIsPinned() { String image = System.getProperty("test.cassandra.image"); assertTrue(image.matches(".+@sha256:[a-f0-9]{64}"), image); System.out.println(image); + System.out.println(System.getProperty("test.redis.image", "")); } } """.trimIndent(), ) + directory.resolve("src/test/java/ImageForkedTest.java").toFile().writeText( + "public class ImageForkedTest extends ImageTest {}\n", + ) + Files.createDirectories(directory.resolve("src/isolatedTest/java")) directory.resolve("src/isolatedTest/java/PlainTest.java").toFile().writeText( """ import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertNull; + public class PlainTest { @Test public void hasNoContainerDependency() { assertNull(System.getProperty("test.cassandra.image")); @@ -309,6 +525,9 @@ class TestcontainersPluginTest { } """.trimIndent(), ) + directory.resolve("src/isolatedTest/java/PlainForkedTest.java").toFile().writeText( + "public class PlainForkedTest extends PlainTest {}\n", + ) } private fun junitClasspath() = @@ -346,5 +565,8 @@ class TestcontainersPluginTest { private fun run(vararg arguments: String) = runner(*arguments).build() - private fun report() = directory.resolve("build/test-results/test/TEST-ImageTest.xml").toFile().readText() + private fun testReport( + task: String = "test", + testClass: String = "ImageTest", + ) = directory.resolve("build/test-results/$task/TEST-$testClass.xml").toFile() } diff --git a/docs/how_to_smoke_test.md b/docs/how_to_smoke_test.md index 2fbbfcec774..55e0484e59c 100644 --- a/docs/how_to_smoke_test.md +++ b/docs/how_to_smoke_test.md @@ -240,7 +240,6 @@ For application containers or dependencies such as RabbitMQ, apply ```kotlin import datadog.buildlogic.testcontainers.image -import datadog.buildlogic.testcontainers.testContainerImage plugins { id("dd-trace-java.module.smoke-test") @@ -265,16 +264,9 @@ private static final RabbitMQContainer RABBIT_MQ_CONTAINER = .asCompatibleSubstituteFor("rabbitmq")); ``` -Gradle fingerprints the resolved digest and passes that immutable image to the -test JVM. The property belongs to the test JVM; explicitly forward it if the -launched application needs it too. Continue using `placeholder(...)` for mapped -ports, which only become available after the container starts. - -Keep the existing `usesService(testcontainersLimit)` declarations on container -test tasks: the image plugin does not assign that service. It also does not -automatically track the test-agent backend's image or Testcontainers helper images. -See [Tests that use containers](./how_to_test.md#tests-that-use-containers) for -suite inheritance, cache behavior, IDE runs and concurrency guidance. +With this plugin Gradle can now fingerprint the resolved image digest and passes +the immutable image to the test. Also, see the [plugin reference](../build-logic/testcontainers/README.md) +for inheritance and shared configuration examples. ## Choosing a backend diff --git a/docs/how_to_test.md b/docs/how_to_test.md index 0400a23e47e..03bc8d7d487 100644 --- a/docs/how_to_test.md +++ b/docs/how_to_test.md @@ -57,12 +57,18 @@ In order to identify such tests and avoid the continuous integration to fail, th ## Tests that use containers +> [!IMPORTANT] +> Don't use image name in Test Container constructors like `new CassandraContainer("cassandra:4")` , +> or `new GenericContainer("icr.io/appcafe/websphere-traditional:latest")`. Tags can be moved. +> Also, these are not properly tracked as _test_ task inputs and as such can't be fingerprinted. +> Instead, use the `dd-trace-java.testcontainers` plugin to declare these as dependencies, +> it will resolve the actual image digest before running the test. + Declare container images in the module's Gradle build so a changed image cannot silently reuse cached test results: ```kotlin import datadog.buildlogic.testcontainers.image -import datadog.buildlogic.testcontainers.testContainerImage plugins { id("dd-trace-java.testcontainers") @@ -75,10 +81,10 @@ dependencies { } ``` -Keep the dedicated container type and use its `DockerImageName` constructor with -the supplied property, without a tag fallback. This example uses the same Redis -module as the repository's Redis tests. Keep the compatibility alias so container -types that validate their image name also accept CI mirrors: +Use `GenericContainer` or the dedicated container type and use its `DockerImageName` constructor +overload with the relevant system property, **without a fallback value**. The compatibility +declaration is important to let _testcontainer_ know it should accept it as a mirrored image. +For example with Redis: ```java import com.redis.testcontainers.RedisContainer; @@ -89,31 +95,17 @@ DockerImageName image = DockerImageName.parse(System.getProperty("test.redis.ima RedisContainer redis = new RedisContainer(image); ``` -Use `GenericContainer` for custom application images without a matching container -module, as in the [WebSphere smoke test](../dd-smoke-tests/websphere-jmx). - -The plugin resolves tags to immutable registry digests before Gradle checks the -test cache, then passes those same image references to the test JVM. Unchanged -digests can reuse test results; changed digests select a different cache entry. -Tags are refreshed even when Gradle reuses its configuration cache. Resolution -failure stops the task rather than trusting an old result. - -Images declared with `testContainerImage` belong to the `test` source set, follow -`implementation` configuration inheritance and reach the matching test task and its forked companions. For a -separate suite, import `datadog.buildlogic.testcontainers.containerImage`, declare -its source set first, then use `containerImage("integrationTest", image(...))` in -`dependencies {}`. Declarations apply to the whole task, including -when `--tests` selects only some classes. Run IDE tests through Gradle, or supply -the image system properties explicitly. - -Image fingerprinting does not configure concurrency. Keep the existing explicit -`usesService(testcontainersLimit)` declarations; the service limits concurrent -test tasks, not individual containers. When adding container tests, configure the -service on the tasks that run them, including forked tasks where applicable. - -Only declared images are tracked; implicit helper images such as Ryuk are not. -See the [plugin reference](../build-logic/testcontainers/README.md) for registry -credentials, Docker Hub mirrors and supported image substitutions. +Essentially the plugin resolves tags to immutable registry digests before Gradle checks +whether test results are up to date or cached. Resolution failure stops the test task +earlier, rather than within the tests. + +The plugin feed the system property to both `test` and `forkedTest` tasks. + +`testContainerImage` is a "companion" for the `testImplementation` configuration. +The plugin automatically creates an image configuration for each `*Implementation` +configuration. For example, `integrationTestImplementation` gets +`integrationTestContainerImage`. Also, see the [plugin reference](../build-logic/testcontainers/README.md) +for inheritance and shared configuration examples. ## Running Tests From 79d40a3f8a854e4328904baff2ef022c9326481b Mon Sep 17 00:00:00 2001 From: Brice Dutheil Date: Thu, 24 Sep 2026 17:42:21 +0200 Subject: [PATCH 10/15] chore: Simplify setup and testing --- build-logic/testcontainers/build.gradle.kts | 22 ++----- .../TestcontainersPluginTest.kt | 59 +------------------ 2 files changed, 8 insertions(+), 73 deletions(-) diff --git a/build-logic/testcontainers/build.gradle.kts b/build-logic/testcontainers/build.gradle.kts index 1315f9c4122..597b606ed86 100644 --- a/build-logic/testcontainers/build.gradle.kts +++ b/build-logic/testcontainers/build.gradle.kts @@ -1,44 +1,35 @@ -import org.gradle.internal.classpath.Instrumented.systemProperty - plugins { `java-gradle-plugin` `kotlin-dsl` - `jvm-test-suite` alias(libs.plugins.shadow) } -val jib = configurations.create("jib") -val conflictingBuildSrc = configurations.create("conflictingBuildSrc") -configurations.compileOnly { extendsFrom(jib) } - -dependencies { - jib("com.google.cloud.tools:jib-core:0.28.2") - conflictingBuildSrc("org.apache.httpcomponents:httpclient:4.3.5") +val jib = configurations.register("jib") { + dependencies.add(project.dependencies.create("com.google.cloud.tools:jib-core:0.28.2")) } +configurations.compileOnly { extendsFrom(jib) } // buildSrc exports an older HttpClient through the parent classloader. Keep Jib's // dependencies private, including when this plugin is consumed as an included build. tasks.shadowJar { - configurations = listOf(jib) + configurations.add(jib) enableAutoRelocation = true relocationPrefix = "datadog.buildlogic.testcontainers.internal" mergeServiceFiles() } configurations.apiElements { outgoing.artifacts.clear() + outgoing.variants.clear() outgoing.artifact(tasks.shadowJar) } configurations.runtimeElements { outgoing.artifacts.clear() + outgoing.variants.clear() outgoing.artifact(tasks.shadowJar) } tasks.pluginUnderTestMetadata { pluginClasspath.setFrom(tasks.shadowJar) } -tasks.test { - inputs.files(conflictingBuildSrc) - systemProperty("test.buildSrc.classpath", conflictingBuildSrc.asPath) -} gradlePlugin { plugins { @@ -57,7 +48,6 @@ testing { implementation(libs.assertj.core) implementation(libs.okhttp3.mockwebserver) implementation("com.squareup.okhttp3:okhttp-tls:${libs.versions.okhttp3.testing.get()}") - implementation(gradleTestKit()) } } } diff --git a/build-logic/testcontainers/src/test/kotlin/datadog/buildlogic/testcontainers/TestcontainersPluginTest.kt b/build-logic/testcontainers/src/test/kotlin/datadog/buildlogic/testcontainers/TestcontainersPluginTest.kt index 6572d1545a2..28c79cc62db 100644 --- a/build-logic/testcontainers/src/test/kotlin/datadog/buildlogic/testcontainers/TestcontainersPluginTest.kt +++ b/build-logic/testcontainers/src/test/kotlin/datadog/buildlogic/testcontainers/TestcontainersPluginTest.kt @@ -6,10 +6,8 @@ import org.gradle.testkit.runner.TaskOutcome import org.junit.jupiter.api.Test import org.junit.jupiter.api.extension.RegisterExtension import org.junit.jupiter.api.io.TempDir -import java.io.File import java.nio.file.Files import java.nio.file.Path -import java.util.Properties class TestcontainersPluginTest { @TempDir @@ -328,56 +326,6 @@ class TestcontainersPluginTest { assertThat(reused.task(":integrationTest")?.outcome).isEqualTo(TaskOutcome.UP_TO_DATE) } - @Test - fun `bearer authentication resolves manifests with an older HttpClient in buildSrc`() { - registry.requireAuthentication = true - fixture(registry.image) - - // Reproduce the parent classloader supplied by Aether in the real buildSrc. - val httpClasspathFiles = - System.getProperty("test.buildSrc.classpath").split(File.pathSeparator).map(::File) - assertThat(httpClasspathFiles).allSatisfy { assertThat(it).isFile() } - assertThat(httpClasspathFiles).anySatisfy { assertThat(it.name).startsWith("httpclient-4.3.5") } - val httpClasspath = - httpClasspathFiles.joinToString(", ") { "\"$it\"" } - Files.createDirectories(directory.resolve("buildSrc/src/main/java")) - directory.resolve("buildSrc/src/main/java/BuildLogic.java").toFile().writeText("public class BuildLogic {}\n") - directory.resolve("buildSrc/build.gradle.kts").toFile().writeText( - """ - plugins { java } - dependencies { implementation(files($httpClasspath)) } - """.trimIndent(), - ) - - // Load as an included build instead of using TestKit's injected plugin classpath. - val metadata = Properties() - javaClass.classLoader.getResourceAsStream("plugin-under-test-metadata.properties")!!.use { metadata.load(it) } - val pluginClasspath = metadata.getProperty("implementation-classpath").split(File.pathSeparator).joinToString(", ") { "\"$it\"" } - Files.createDirectories(directory.resolve("plugin")) - directory.resolve("plugin/settings.gradle.kts").toFile().writeText("rootProject.name = \"fixture-plugin\"\n") - directory.resolve("plugin/build.gradle.kts").toFile().writeText( - """ - plugins { `java-gradle-plugin` } - dependencies { implementation(files($pluginClasspath)) } - gradlePlugin { - plugins { - create("testcontainers") { - id = "dd-trace-java.testcontainers" - implementationClass = "datadog.buildlogic.testcontainers.TestcontainersPlugin" - } - } - } - """.trimIndent(), - ) - val settings = directory.resolve("settings.gradle.kts").toFile() - settings.writeText("pluginManagement { includeBuild(\"plugin\") }\n" + settings.readText()) - - assertThat(runner("test", injectPluginClasspath = false).build().task(":test")?.outcome).isEqualTo(TaskOutcome.SUCCESS) - assertThat(registry.tokenRequests.get()).isPositive() - assertThat(registry.authorizedRequests.get()).isPositive() - assertThat(testReport()).content().contains(registry.digest) - } - private fun fixture(image: String) { directory.resolve("settings.gradle.kts").toFile().writeText( """ @@ -547,13 +495,10 @@ class TestcontainersPluginTest { )}\"" } - private fun runner( - vararg arguments: String, - injectPluginClasspath: Boolean = true, - ) = GradleRunner + private fun runner(vararg arguments: String) = GradleRunner .create() .withProjectDir(directory.toFile()) - .apply { if (injectPluginClasspath) withPluginClasspath() } + .withPluginClasspath() .withArguments( *arguments, "--build-cache", From 7b229f75b5189e63a13b777ddd7fa105bcae5ae5 Mon Sep 17 00:00:00 2001 From: Brice Dutheil Date: Thu, 24 Sep 2026 18:12:43 +0200 Subject: [PATCH 11/15] fix: track remaining test container images --- .gitlab-ci.yml | 98 +++++++++++++++++++ .../proxy-repositories.init.gradle.kts | 10 +- .../testcontainers/RegistryExtension.kt | 21 +--- .../repository-proxy.init.gradle.kts | 1 + .../aerospike-4.0/build.gradle | 2 + .../aerospike4/AerospikeBaseTest.groovy | 3 +- .../aws-java-dynamodb-2.0/build.gradle | 2 + .../src/test/groovy/DynamoDbClientTest.groovy | 2 +- .../aws-java-eventbridge-2.0/build.gradle | 2 + .../test/groovy/EventBridgeClientTest.groovy | 2 +- .../aws-java/aws-java-s3-2.0/build.gradle | 2 + .../src/test/groovy/S3ClientTest.groovy | 2 +- .../aws-java/aws-java-sdk-2.2/build.gradle | 6 ++ .../groovy/PayloadTaggingTest.groovy | 3 +- .../aws-java/aws-java-sfn-2.0/build.gradle | 2 + .../src/test/groovy/SfnClientTest.groovy | 2 +- .../aws-java/aws-java-sns-1.0/build.gradle | 2 + .../src/test/groovy/SnsClientTest.groovy | 2 +- .../aws-java/aws-java-sns-2.0/build.gradle | 2 + .../src/test/groovy/SnsClientTest.groovy | 2 +- .../couchbase/couchbase-3.1/build.gradle | 8 ++ .../test/groovy/CouchbaseClient31Test.groovy | 6 +- .../couchbase/couchbase-3.2/build.gradle | 8 ++ .../test/groovy/CouchbaseClient32Test.groovy | 6 +- .../RemoteJDBCInstrumentationTest.groovy | 2 +- .../lettuce/lettuce-5.0/build.gradle | 2 + .../test/groovy/Lettuce5ClientTestBase.groovy | 16 +-- .../src/test/java/Lettuce5ClusterTest.java | 4 +- .../test/java/Lettuce5MasterReplicaTest.java | 4 +- .../testFixtures/groovy/MongoBaseTest.groovy | 9 +- .../mongo-driver-3.1/build.gradle | 2 + .../mongo-driver-3.4/build.gradle | 2 + .../mongo-driver-3.6/build.gradle | 2 + .../mongo-driver-3.8/build.gradle | 2 + .../mongo-driver-4.0/build.gradle | 2 + .../mongo-test-async-3.3/build.gradle | 3 + .../mongo-test-core-3.1/build.gradle | 4 +- .../mongo-test-core-3.7/build.gradle | 3 + .../mongo-test-sync-3.10/build.gradle | 3 + .../rabbitmq-amqp-2.7/build.gradle | 2 + .../groovy/ReactorRabbitMQTest.groovy | 5 +- .../src/test/groovy/RabbitMQTest.groovy | 5 +- .../redisson/redisson-2.0.0/build.gradle | 2 + .../src/test/groovy/RedissonClientTest.groovy | 4 +- .../redisson/redisson-2.3.0/build.gradle | 2 + .../src/test/groovy/RedissonClientTest.groovy | 4 +- .../redisson/redisson-3.10.3/build.gradle | 2 + .../src/test/groovy/RedissonClientTest.groovy | 4 +- .../spring/spring-rabbit-1.5/build.gradle | 2 + .../src/test/groovy/SpringAmqpTest.groovy | 5 +- .../spymemcached-2.10/build.gradle | 2 + .../spymemcached/SpymemcachedTest.groovy | 3 +- .../src/test/java/TestDatabases.java | 8 +- .../src/test/java/TestDatabases.java | 8 +- .../src/test/java/TestDatabases.java | 8 +- .../vertx-redis-client-3.9/build.gradle | 2 + .../src/test/groovy/VertxRedisTestBase.groovy | 5 +- .../spring-boot-rabbit/build.gradle | 3 + .../smoketest/SpringBootRabbitSmokeTest.java | 4 +- dd-smoke-tests/springboot-mongo/build.gradle | 3 + .../SpringBootMongoIntegrationTest.groovy | 4 +- dd-trace-core/build.gradle | 4 + .../TracerConnectionReliabilityTest.java | 3 +- .../java/AbstractTraceAgentTest.java | 4 +- .../metrics/MetricsIntegrationTest.java | 4 +- 65 files changed, 279 insertions(+), 79 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4d45afe03ad..2f3a3c13a97 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -270,6 +270,7 @@ test_container_images: tags: ["docker-in-docker:amd64"] variables: CACHE_TYPE: "inst" + CI_NO_SPLIT: "true" parallel: matrix: - GRADLE_TARGET: @@ -298,4 +299,101 @@ test_container_images: - ":dd-java-agent:instrumentation:vertx:vertx-mysql-client:vertx-mysql-client-4.0:latestDepForkedTest" - ":dd-java-agent:instrumentation:vertx:vertx-pg-client:vertx-pg-client-4.0:forkedTest" - ":dd-java-agent:instrumentation:vertx:vertx-pg-client:vertx-pg-client-4.0:latestDepForkedTest" + - ":dd-java-agent:instrumentation:aerospike-4.0:forkedTest" + - ":dd-java-agent:instrumentation:aerospike-4.0:latest7DepForkedTest" + - ":dd-java-agent:instrumentation:aerospike-4.0:latestDepForkedTest" + - ":dd-java-agent:instrumentation:aws-java:aws-java-dynamodb-2.0:latestDepTest" + - ":dd-java-agent:instrumentation:aws-java:aws-java-dynamodb-2.0:test" + - ":dd-java-agent:instrumentation:aws-java:aws-java-eventbridge-2.0:latestDepTest" + - ":dd-java-agent:instrumentation:aws-java:aws-java-eventbridge-2.0:test" + - ":dd-java-agent:instrumentation:aws-java:aws-java-s3-2.0:latestDepTest" + - ":dd-java-agent:instrumentation:aws-java:aws-java-s3-2.0:test" + - ":dd-java-agent:instrumentation:aws-java:aws-java-sdk-2.2:latestPayloadTaggingForkedTest" + - ":dd-java-agent:instrumentation:aws-java:aws-java-sdk-2.2:payloadTaggingTestForkedTest" + - ":dd-java-agent:instrumentation:aws-java:aws-java-sfn-2.0:latestDepTest" + - ":dd-java-agent:instrumentation:aws-java:aws-java-sfn-2.0:test" + - ":dd-java-agent:instrumentation:aws-java:aws-java-sns-1.0:forkedTest" + - ":dd-java-agent:instrumentation:aws-java:aws-java-sns-1.0:latestDepForkedTest" + - ":dd-java-agent:instrumentation:aws-java:aws-java-sns-1.0:latestDepTest" + - ":dd-java-agent:instrumentation:aws-java:aws-java-sns-1.0:test" + - ":dd-java-agent:instrumentation:aws-java:aws-java-sns-2.0:forkedTest" + - ":dd-java-agent:instrumentation:aws-java:aws-java-sns-2.0:latestDepForkedTest" + - ":dd-java-agent:instrumentation:aws-java:aws-java-sns-2.0:latestDepTest" + - ":dd-java-agent:instrumentation:aws-java:aws-java-sns-2.0:test" + - ":dd-java-agent:instrumentation:lettuce:lettuce-5.0:forkedTest" + - ":dd-java-agent:instrumentation:lettuce:lettuce-5.0:latestDepForkedTest" + - ":dd-java-agent:instrumentation:lettuce:lettuce-5.0:latestDepTest" + - ":dd-java-agent:instrumentation:lettuce:lettuce-5.0:lettuce51Test" + - ":dd-java-agent:instrumentation:lettuce:lettuce-5.0:lettuce60Test" + - ":dd-java-agent:instrumentation:lettuce:lettuce-5.0:lettuce61Test" + - ":dd-java-agent:instrumentation:lettuce:lettuce-5.0:lettuce62Test" + - ":dd-java-agent:instrumentation:lettuce:lettuce-5.0:test" + - ":dd-java-agent:instrumentation:spymemcached-2.10:forkedTest" + - ":dd-java-agent:instrumentation:spymemcached-2.10:latestDepForkedTest" + - ":dd-java-agent:instrumentation:spymemcached-2.10:latestDepTest" + - ":dd-java-agent:instrumentation:spymemcached-2.10:test" + - ":dd-java-agent:instrumentation:couchbase:couchbase-3.1:forkedTest" + - ":dd-java-agent:instrumentation:couchbase:couchbase-3.1:latestDepTest" + - ":dd-java-agent:instrumentation:couchbase:couchbase-3.1:test" + - ":dd-java-agent:instrumentation:couchbase:couchbase-3.2:forkedTest" + - ":dd-java-agent:instrumentation:couchbase:couchbase-3.2:latestDepTest" + - ":dd-java-agent:instrumentation:couchbase:couchbase-3.2:test" + - ":dd-java-agent:instrumentation:mongo:mongo-driver:mongo-driver-3:mongo-driver-3.1:forkedTest" + - ":dd-java-agent:instrumentation:mongo:mongo-driver:mongo-driver-3:mongo-driver-3.1:latestDepTest" + - ":dd-java-agent:instrumentation:mongo:mongo-driver:mongo-driver-3:mongo-driver-3.1:test" + - ":dd-java-agent:instrumentation:mongo:mongo-driver:mongo-driver-3:mongo-driver-3.4:forkedTest" + - ":dd-java-agent:instrumentation:mongo:mongo-driver:mongo-driver-3:mongo-driver-3.4:latestDepTest" + - ":dd-java-agent:instrumentation:mongo:mongo-driver:mongo-driver-3:mongo-driver-3.4:test" + - ":dd-java-agent:instrumentation:mongo:mongo-driver:mongo-driver-3:mongo-driver-3.6:forkedTest" + - ":dd-java-agent:instrumentation:mongo:mongo-driver:mongo-driver-3:mongo-driver-3.6:latestDepTest" + - ":dd-java-agent:instrumentation:mongo:mongo-driver:mongo-driver-3:mongo-driver-3.6:test" + - ":dd-java-agent:instrumentation:mongo:mongo-driver:mongo-driver-3:mongo-driver-3.8:forkedTest" + - ":dd-java-agent:instrumentation:mongo:mongo-driver:mongo-driver-3:mongo-driver-3.8:latestDepTest" + - ":dd-java-agent:instrumentation:mongo:mongo-driver:mongo-driver-3:mongo-driver-3.8:test" + - ":dd-java-agent:instrumentation:mongo:mongo-driver:mongo-driver-4.0:forkedTest" + - ":dd-java-agent:instrumentation:mongo:mongo-driver:mongo-driver-4.0:latestDepForkedTest" + - ":dd-java-agent:instrumentation:mongo:mongo-driver:mongo-driver-4.0:latestDepTest" + - ":dd-java-agent:instrumentation:mongo:mongo-driver:mongo-driver-4.0:mongo410ForkedTest" + - ":dd-java-agent:instrumentation:mongo:mongo-driver:mongo-driver-4.0:mongo410Test" + - ":dd-java-agent:instrumentation:mongo:mongo-driver:mongo-driver-4.0:mongo43ForkedTest" + - ":dd-java-agent:instrumentation:mongo:mongo-driver:mongo-driver-4.0:mongo43Test" + - ":dd-java-agent:instrumentation:mongo:mongo-driver:mongo-driver-4.0:test" + - ":dd-java-agent:instrumentation:mongo:mongo-test:mongo-test-async-3.3:forkedTest" + - ":dd-java-agent:instrumentation:mongo:mongo-test:mongo-test-async-3.3:latestDepTest" + - ":dd-java-agent:instrumentation:mongo:mongo-test:mongo-test-async-3.3:test" + - ":dd-java-agent:instrumentation:mongo:mongo-test:mongo-test-core-3.1:forkedTest" + - ":dd-java-agent:instrumentation:mongo:mongo-test:mongo-test-core-3.1:latestDepTest" + - ":dd-java-agent:instrumentation:mongo:mongo-test:mongo-test-core-3.1:test" + - ":dd-java-agent:instrumentation:mongo:mongo-test:mongo-test-core-3.7:forkedTest" + - ":dd-java-agent:instrumentation:mongo:mongo-test:mongo-test-core-3.7:latestDepTest" + - ":dd-java-agent:instrumentation:mongo:mongo-test:mongo-test-core-3.7:test" + - ":dd-java-agent:instrumentation:mongo:mongo-test:mongo-test-sync-3.10:forkedTest" + - ":dd-java-agent:instrumentation:mongo:mongo-test:mongo-test-sync-3.10:latestDepTest" + - ":dd-java-agent:instrumentation:mongo:mongo-test:mongo-test-sync-3.10:test" + - ":dd-java-agent:instrumentation:rabbitmq-amqp-2.7:forkedTest" + - ":dd-java-agent:instrumentation:rabbitmq-amqp-2.7:latestDepTest" + - ":dd-java-agent:instrumentation:rabbitmq-amqp-2.7:latestReactorTest" + - ":dd-java-agent:instrumentation:rabbitmq-amqp-2.7:reactorTest" + - ":dd-java-agent:instrumentation:rabbitmq-amqp-2.7:test" + - ":dd-java-agent:instrumentation:redisson:redisson-2.0.0:forkedTest" + - ":dd-java-agent:instrumentation:redisson:redisson-2.0.0:latestDepForkedTest" + - ":dd-java-agent:instrumentation:redisson:redisson-2.0.0:latestDepTest" + - ":dd-java-agent:instrumentation:redisson:redisson-2.0.0:test" + - ":dd-java-agent:instrumentation:redisson:redisson-2.3.0:forkedTest" + - ":dd-java-agent:instrumentation:redisson:redisson-2.3.0:latestDepForkedTest" + - ":dd-java-agent:instrumentation:redisson:redisson-2.3.0:latestDepTest" + - ":dd-java-agent:instrumentation:redisson:redisson-2.3.0:test" + - ":dd-java-agent:instrumentation:redisson:redisson-3.10.3:forkedTest" + - ":dd-java-agent:instrumentation:redisson:redisson-3.10.3:latestDepForkedTest" + - ":dd-java-agent:instrumentation:redisson:redisson-3.10.3:latestDepTest" + - ":dd-java-agent:instrumentation:redisson:redisson-3.10.3:test" + - ":dd-java-agent:instrumentation:spring:spring-rabbit-1.5:latestDepTest" + - ":dd-java-agent:instrumentation:spring:spring-rabbit-1.5:test" + - ":dd-java-agent:instrumentation:vertx:vertx-redis-client:vertx-redis-client-3.9:forkedTest" + - ":dd-java-agent:instrumentation:vertx:vertx-redis-client:vertx-redis-client-3.9:latestDepForkedTest" + - ":dd-java-agent:instrumentation:vertx:vertx-redis-client:vertx-redis-client-3.9:redis4xForkedTest" + - ":dd-trace-core:test" + - ":dd-trace-core:traceAgentTest" + - ":dd-smoke-tests:spring-boot-rabbit:test" + - ":dd-smoke-tests:springboot-mongo:test" - ":dd-smoke-tests:websphere-jmx:test" diff --git a/build-logic/smoke-test/src/main/resources/datadog/buildlogic/smoketest/proxy-repositories.init.gradle.kts b/build-logic/smoke-test/src/main/resources/datadog/buildlogic/smoketest/proxy-repositories.init.gradle.kts index 4d059fd9645..552577d09e1 100644 --- a/build-logic/smoke-test/src/main/resources/datadog/buildlogic/smoketest/proxy-repositories.init.gradle.kts +++ b/build-logic/smoke-test/src/main/resources/datadog/buildlogic/smoketest/proxy-repositories.init.gradle.kts @@ -1,12 +1,12 @@ import org.gradle.api.Action import org.gradle.api.Project +import org.gradle.kotlin.dsl.* import org.gradle.api.artifacts.dsl.RepositoryHandler import org.gradle.api.artifacts.repositories.MavenArtifactRepository import org.gradle.api.initialization.Settings import java.net.URI -// Requires Gradle 6.8+ and assumes project repositories are allowed by the target build. - +// Requires Gradle 6.8+ for Settings.providers and Settings.dependencyResolutionManagement. gradle.beforeSettings(Action { val gradlePluginProxy = providers.gradleProperty("gradlePluginProxy").orNull val mavenRepositoryProxy = providers.gradleProperty("mavenRepositoryProxy").orNull @@ -23,6 +23,8 @@ gradle.beforeSettings(Action { withType(MavenArtifactRepository::class.java).configureEach { // A repository declared without a URL has a null one until Gradle validates it; leave it be // so the nested build reports that itself instead of failing inside this init script. + // See https://github.com/gradle/gradle/issues/37612 + @Suppress("UNNECESSARY_SAFE_CALL") val repositoryUrl = url?.toString()?.trimEnd('/') if (repositoryUrl != null && repositoryUrl in mavenCentralUrls) { url = URI(proxy) @@ -33,6 +35,8 @@ gradle.beforeSettings(Action { fun RepositoryHandler.removeDuplicateMavenProxy() { val proxyUrl = mavenRepositoryProxy?.takeIf { it.isNotBlank() }?.trimEnd('/') ?: return + // see https://github.com/gradle/gradle/issues/37612 + @Suppress("UNNECESSARY_SAFE_CALL") val proxies = withType(MavenArtifactRepository::class.java) .filter { it.url?.toString()?.trimEnd('/') == proxyUrl } // Keep the injected repository: it is the only one known to be unrestricted, since a declared @@ -92,7 +96,7 @@ gradle.beforeSettings(Action { } }) - gradle.afterProject(Action { + gradle.afterProject(Action { repositories.removeDuplicateMavenProxy() }) }) diff --git a/build-logic/testcontainers/src/test/kotlin/datadog/buildlogic/testcontainers/RegistryExtension.kt b/build-logic/testcontainers/src/test/kotlin/datadog/buildlogic/testcontainers/RegistryExtension.kt index e61fe9c331e..26624a1a202 100644 --- a/build-logic/testcontainers/src/test/kotlin/datadog/buildlogic/testcontainers/RegistryExtension.kt +++ b/build-logic/testcontainers/src/test/kotlin/datadog/buildlogic/testcontainers/RegistryExtension.kt @@ -11,9 +11,8 @@ import org.junit.jupiter.api.extension.BeforeEachCallback import org.junit.jupiter.api.extension.ExtensionContext import java.net.InetAddress import java.security.MessageDigest -import java.util.concurrent.atomic.AtomicInteger -/** A local HTTPS registry with mutable manifests and optional bearer authentication. */ +/** A local HTTPS registry with mutable manifests. */ class RegistryExtension : BeforeEachCallback, AfterEachCallback { @@ -21,13 +20,8 @@ class RegistryExtension : @Volatile var imageVersion = 1 - @Volatile var requireAuthentication = false - @Volatile var unavailable = false - val tokenRequests = AtomicInteger() - val authorizedRequests = AtomicInteger() - val image: String get() = "127.0.0.1:${server.port}/library/cassandra:4" @@ -54,24 +48,11 @@ class RegistryExtension : MockResponse().setResponseCode(503) } - request.path!!.startsWith("/token") -> { - tokenRequests.incrementAndGet() - MockResponse().setHeader("Content-Type", "application/json").setBody("""{"token":"fixture-token"}""") - } - !request.path!!.startsWith("/v2/") -> { MockResponse().setResponseCode(404) } - requireAuthentication && request.getHeader("Authorization") != "Bearer fixture-token" -> { - MockResponse().setResponseCode(401).setHeader( - "WWW-Authenticate", - "Bearer realm=\"${server.url("/token")}\",service=\"fixture\",scope=\"repository:library/cassandra:pull\"", - ) - } - else -> { - if (requireAuthentication) authorizedRequests.incrementAndGet() val body = manifest() MockResponse() .setHeader("Content-Type", "application/vnd.oci.image.manifest.v1+json") diff --git a/buildSrc/src/test/resources/repository-proxy.init.gradle.kts b/buildSrc/src/test/resources/repository-proxy.init.gradle.kts index f4b18536966..08c3e8ef446 100644 --- a/buildSrc/src/test/resources/repository-proxy.init.gradle.kts +++ b/buildSrc/src/test/resources/repository-proxy.init.gradle.kts @@ -5,6 +5,7 @@ import org.gradle.api.artifacts.repositories.MavenArtifactRepository import org.gradle.api.initialization.Settings import java.net.URI +// Requires Gradle 6.8+ for Settings.dependencyResolutionManagement. // Routes the public repositories declared by TestKit builds (see GradleFixture) through the // mirrors CI configures, so buildSrc tests do not hit Maven Central and get rate limited. // diff --git a/dd-java-agent/instrumentation/aerospike-4.0/build.gradle b/dd-java-agent/instrumentation/aerospike-4.0/build.gradle index b67da684832..87de14495cd 100644 --- a/dd-java-agent/instrumentation/aerospike-4.0/build.gradle +++ b/dd-java-agent/instrumentation/aerospike-4.0/build.gradle @@ -1,5 +1,6 @@ plugins { id 'dd-trace-java.module.instrumentation' + id 'dd-trace-java.testcontainers' } muzzle { @@ -30,6 +31,7 @@ tasks.named("latestDepForkedTest", Test) { dependencies { compileOnly group: 'com.aerospike', name: 'aerospike-client', version: '4.0.0' + testContainerImage(image('aerospike:ce-6.2.0.2', 'test.aerospike.image')) testImplementation group: 'com.aerospike', name: 'aerospike-client', version: '4.0.0' testImplementation libs.testcontainers diff --git a/dd-java-agent/instrumentation/aerospike-4.0/src/test/groovy/datadog/trace/instrumentation/aerospike4/AerospikeBaseTest.groovy b/dd-java-agent/instrumentation/aerospike-4.0/src/test/groovy/datadog/trace/instrumentation/aerospike4/AerospikeBaseTest.groovy index 746f57577c8..274da89bc4b 100644 --- a/dd-java-agent/instrumentation/aerospike-4.0/src/test/groovy/datadog/trace/instrumentation/aerospike4/AerospikeBaseTest.groovy +++ b/dd-java-agent/instrumentation/aerospike-4.0/src/test/groovy/datadog/trace/instrumentation/aerospike4/AerospikeBaseTest.groovy @@ -11,6 +11,7 @@ import datadog.trace.api.DDSpanTypes import datadog.trace.bootstrap.instrumentation.api.Tags import datadog.trace.core.DDSpan import org.testcontainers.containers.GenericContainer +import org.testcontainers.utility.DockerImageName import spock.lang.Shared abstract class AerospikeBaseTest extends VersionedNamingTestBase { @@ -26,7 +27,7 @@ abstract class AerospikeBaseTest extends VersionedNamingTestBase { def setup() throws Exception { // Linux arm64 supported since `ce-6.2.0.2` - aerospike = new GenericContainer('aerospike:ce-6.2.0.2') + aerospike = new GenericContainer(DockerImageName.parse(System.getProperty('test.aerospike.image'))) .withExposedPorts(3000) // proto-fd-max default is 15000, but container default is 1024. // see: https://aerospike.com/docs/database/reference/config#service__proto-fd-max diff --git a/dd-java-agent/instrumentation/aws-java/aws-java-dynamodb-2.0/build.gradle b/dd-java-agent/instrumentation/aws-java/aws-java-dynamodb-2.0/build.gradle index 7e419d43131..7ff7c472ef1 100644 --- a/dd-java-agent/instrumentation/aws-java/aws-java-dynamodb-2.0/build.gradle +++ b/dd-java-agent/instrumentation/aws-java/aws-java-dynamodb-2.0/build.gradle @@ -1,5 +1,6 @@ plugins { id 'dd-trace-java.module.instrumentation' + id 'dd-trace-java.testcontainers' } muzzle { @@ -16,6 +17,7 @@ addTestSuiteExtendingForDir('latestDepForkedTest', 'latestDepTest', 'test') dependencies { compileOnly group: 'software.amazon.awssdk', name: 'dynamodb', version: '2.30.22' + testContainerImage(image('localstack/localstack:4.2.0', 'test.localstack.image')) // Include httpclient instrumentation for testing because it is a dependency for aws-sdk. testImplementation project(':dd-java-agent:instrumentation:apache-httpclient:apache-httpclient-4.0') diff --git a/dd-java-agent/instrumentation/aws-java/aws-java-dynamodb-2.0/src/test/groovy/DynamoDbClientTest.groovy b/dd-java-agent/instrumentation/aws-java/aws-java-dynamodb-2.0/src/test/groovy/DynamoDbClientTest.groovy index 02acd67be37..60ee72c6e0f 100644 --- a/dd-java-agent/instrumentation/aws-java/aws-java-dynamodb-2.0/src/test/groovy/DynamoDbClientTest.groovy +++ b/dd-java-agent/instrumentation/aws-java/aws-java-dynamodb-2.0/src/test/groovy/DynamoDbClientTest.groovy @@ -29,7 +29,7 @@ import spock.lang.Shared import java.time.Duration class DynamoDbClientTest extends InstrumentationSpecification { - static final LOCALSTACK = new GenericContainer(DockerImageName.parse("localstack/localstack:4.2.0")) + static final LOCALSTACK = new GenericContainer(DockerImageName.parse(System.getProperty("test.localstack.image"))) .withExposedPorts(4566) .withEnv("SERVICES", "dynamodb") .withReuse(true) diff --git a/dd-java-agent/instrumentation/aws-java/aws-java-eventbridge-2.0/build.gradle b/dd-java-agent/instrumentation/aws-java/aws-java-eventbridge-2.0/build.gradle index f21a227f1df..217dd6dc439 100644 --- a/dd-java-agent/instrumentation/aws-java/aws-java-eventbridge-2.0/build.gradle +++ b/dd-java-agent/instrumentation/aws-java/aws-java-eventbridge-2.0/build.gradle @@ -1,5 +1,6 @@ plugins { id 'dd-trace-java.module.instrumentation' + id 'dd-trace-java.testcontainers' } muzzle { @@ -16,6 +17,7 @@ addTestSuiteExtendingForDir('latestDepForkedTest', 'latestDepTest', 'test') dependencies { compileOnly group: 'software.amazon.awssdk', name: 'eventbridge', version: '2.27.19' + testContainerImage(image('localstack/localstack:4.2.0', 'test.localstack.image')) // Include httpclient instrumentation for testing because it is a dependency for aws-sdk. testImplementation project(':dd-java-agent:instrumentation:apache-httpclient:apache-httpclient-4.0') diff --git a/dd-java-agent/instrumentation/aws-java/aws-java-eventbridge-2.0/src/test/groovy/EventBridgeClientTest.groovy b/dd-java-agent/instrumentation/aws-java/aws-java-eventbridge-2.0/src/test/groovy/EventBridgeClientTest.groovy index 3362d3f4b97..19d158d3adf 100644 --- a/dd-java-agent/instrumentation/aws-java/aws-java-eventbridge-2.0/src/test/groovy/EventBridgeClientTest.groovy +++ b/dd-java-agent/instrumentation/aws-java/aws-java-eventbridge-2.0/src/test/groovy/EventBridgeClientTest.groovy @@ -20,7 +20,7 @@ import software.amazon.awssdk.services.sqs.model.QueueAttributeName import spock.lang.Shared class EventBridgeClientTest extends InstrumentationSpecification { - static final LOCALSTACK = new GenericContainer(DockerImageName.parse("localstack/localstack:4.2.0")) + static final LOCALSTACK = new GenericContainer(DockerImageName.parse(System.getProperty("test.localstack.image"))) .withExposedPorts(4566) .withEnv("SERVICES", "sns,sqs,events") .withReuse(true) diff --git a/dd-java-agent/instrumentation/aws-java/aws-java-s3-2.0/build.gradle b/dd-java-agent/instrumentation/aws-java/aws-java-s3-2.0/build.gradle index de35825b2e5..bfe17e9d31b 100644 --- a/dd-java-agent/instrumentation/aws-java/aws-java-s3-2.0/build.gradle +++ b/dd-java-agent/instrumentation/aws-java/aws-java-s3-2.0/build.gradle @@ -1,5 +1,6 @@ plugins { id 'dd-trace-java.module.instrumentation' + id 'dd-trace-java.testcontainers' } muzzle { @@ -16,6 +17,7 @@ addTestSuiteExtendingForDir('latestDepForkedTest', 'latestDepTest', 'test') dependencies { compileOnly group: 'software.amazon.awssdk', name: 's3', version: '2.29.26' + testContainerImage(image('localstack/localstack:4.2.0', 'test.localstack.image')) // Include httpclient instrumentation for testing because it is a dependency for aws-sdk. testRuntimeOnly project(':dd-java-agent:instrumentation:apache-httpclient:apache-httpclient-4.0') diff --git a/dd-java-agent/instrumentation/aws-java/aws-java-s3-2.0/src/test/groovy/S3ClientTest.groovy b/dd-java-agent/instrumentation/aws-java/aws-java-s3-2.0/src/test/groovy/S3ClientTest.groovy index c2071fb7858..76dc369ed01 100644 --- a/dd-java-agent/instrumentation/aws-java/aws-java-s3-2.0/src/test/groovy/S3ClientTest.groovy +++ b/dd-java-agent/instrumentation/aws-java/aws-java-s3-2.0/src/test/groovy/S3ClientTest.groovy @@ -18,7 +18,7 @@ import spock.lang.Shared import java.time.Duration class S3ClientTest extends InstrumentationSpecification { - static final LOCALSTACK = new GenericContainer(DockerImageName.parse("localstack/localstack:4.2.0")) + static final LOCALSTACK = new GenericContainer(DockerImageName.parse(System.getProperty("test.localstack.image"))) .withExposedPorts(4566) .withEnv("SERVICES", "s3") .withReuse(true) diff --git a/dd-java-agent/instrumentation/aws-java/aws-java-sdk-2.2/build.gradle b/dd-java-agent/instrumentation/aws-java/aws-java-sdk-2.2/build.gradle index 7945dca62b9..cfdc266d8ec 100644 --- a/dd-java-agent/instrumentation/aws-java/aws-java-sdk-2.2/build.gradle +++ b/dd-java-agent/instrumentation/aws-java/aws-java-sdk-2.2/build.gradle @@ -1,5 +1,6 @@ plugins { id 'dd-trace-java.module.instrumentation' + id 'dd-trace-java.testcontainers' } muzzle { @@ -26,10 +27,15 @@ addForkedTestTask('payloadTaggingTest') addTestSuiteForDir('latestPayloadTaggingTest', 'payloadTaggingTest') addTestSuiteExtendingForDir('latestPayloadTaggingForkedTest', 'latestPayloadTaggingTest', 'payloadTaggingTest') +configurations.named('latestPayloadTaggingTestContainerImage') { + extendsFrom(configurations.named('payloadTaggingTestContainerImage').get()) +} + def fixedSdkVersion = '2.20.33' // 2.20.34 is missing and breaks IDEA import dependencies { compileOnly group: 'software.amazon.awssdk', name: 'aws-core', version: '2.2.0' + payloadTaggingTestContainerImage(image('localstack/localstack:4.2.0', 'test.localstack.image')) testImplementation project(':dd-java-agent:instrumentation:aws-java:aws-java-common') // Include httpclient instrumentation for testing because it is a dependency for aws-sdk. diff --git a/dd-java-agent/instrumentation/aws-java/aws-java-sdk-2.2/src/payloadTaggingTest/groovy/PayloadTaggingTest.groovy b/dd-java-agent/instrumentation/aws-java/aws-java-sdk-2.2/src/payloadTaggingTest/groovy/PayloadTaggingTest.groovy index 2f8a4ee613e..ea4719db72d 100644 --- a/dd-java-agent/instrumentation/aws-java/aws-java-sdk-2.2/src/payloadTaggingTest/groovy/PayloadTaggingTest.groovy +++ b/dd-java-agent/instrumentation/aws-java/aws-java-sdk-2.2/src/payloadTaggingTest/groovy/PayloadTaggingTest.groovy @@ -26,7 +26,7 @@ abstract class AbstractPayloadTaggingTest extends InstrumentationSpecification { static final Object NA = {} static final int DEFAULT_PORT = 4566 - static final LOCALSTACK = new GenericContainer(DockerImageName.parse("localstack/localstack:4.2.0")) + static final LOCALSTACK = new GenericContainer(DockerImageName.parse(System.getProperty("test.localstack.image"))) .withExposedPorts(DEFAULT_PORT) .withEnv("SERVICES", "apigateway,events,s3,sns,sqs,kinesis") .withReuse(true) @@ -364,4 +364,3 @@ class PayloadTaggingMaxTagsForkedTest extends AbstractPayloadTaggingTest { ] } } - diff --git a/dd-java-agent/instrumentation/aws-java/aws-java-sfn-2.0/build.gradle b/dd-java-agent/instrumentation/aws-java/aws-java-sfn-2.0/build.gradle index b9307d3a0a9..1d2dd96574e 100644 --- a/dd-java-agent/instrumentation/aws-java/aws-java-sfn-2.0/build.gradle +++ b/dd-java-agent/instrumentation/aws-java/aws-java-sfn-2.0/build.gradle @@ -1,5 +1,6 @@ plugins { id 'dd-trace-java.module.instrumentation' + id 'dd-trace-java.testcontainers' } muzzle { @@ -17,6 +18,7 @@ addTestSuiteExtendingForDir('latestDepForkedTest', 'latestDepTest', 'test') dependencies { compileOnly group: 'software.amazon.awssdk', name: 'sfn', version: '2.15.35' + testContainerImage(image('localstack/localstack:4.2.0', 'test.localstack.image')) // Include httpclient instrumentation for testing because it is a dependency for aws-sdk. testImplementation project(':dd-java-agent:instrumentation:apache-httpclient:apache-httpclient-4.0') diff --git a/dd-java-agent/instrumentation/aws-java/aws-java-sfn-2.0/src/test/groovy/SfnClientTest.groovy b/dd-java-agent/instrumentation/aws-java/aws-java-sfn-2.0/src/test/groovy/SfnClientTest.groovy index 18b2c36ee06..74f9c4e183c 100644 --- a/dd-java-agent/instrumentation/aws-java/aws-java-sfn-2.0/src/test/groovy/SfnClientTest.groovy +++ b/dd-java-agent/instrumentation/aws-java/aws-java-sfn-2.0/src/test/groovy/SfnClientTest.groovy @@ -24,7 +24,7 @@ abstract class SfnClientTest extends VersionedNamingTestBase { @Shared Object endPoint def setupSpec() { - localStack = new GenericContainer(DockerImageName.parse("localstack/localstack:4.2.0")) + localStack = new GenericContainer(DockerImageName.parse(System.getProperty("test.localstack.image"))) .withExposedPorts(4566) .withEnv("SERVICES", "stepfunctions") .withReuse(true) diff --git a/dd-java-agent/instrumentation/aws-java/aws-java-sns-1.0/build.gradle b/dd-java-agent/instrumentation/aws-java/aws-java-sns-1.0/build.gradle index 93f5d7abd56..d39b71702d8 100644 --- a/dd-java-agent/instrumentation/aws-java/aws-java-sns-1.0/build.gradle +++ b/dd-java-agent/instrumentation/aws-java/aws-java-sns-1.0/build.gradle @@ -1,5 +1,6 @@ plugins { id 'dd-trace-java.module.instrumentation' + id 'dd-trace-java.testcontainers' } muzzle { @@ -17,6 +18,7 @@ addTestSuiteExtendingForDir('latestDepForkedTest', 'latestDepTest', 'test') dependencies { compileOnly group: 'com.amazonaws', name: 'aws-java-sdk-sns', version: '1.12.710' + testContainerImage(image('localstack/localstack:4.2.0', 'test.localstack.image')) testImplementation project(':dd-java-agent:instrumentation:aws-java:aws-java-common') // Include httpclient instrumentation for testing because it is a dependency for aws-sdk. diff --git a/dd-java-agent/instrumentation/aws-java/aws-java-sns-1.0/src/test/groovy/SnsClientTest.groovy b/dd-java-agent/instrumentation/aws-java/aws-java-sns-1.0/src/test/groovy/SnsClientTest.groovy index 4aa1e6e2ddc..a4154bc543d 100644 --- a/dd-java-agent/instrumentation/aws-java/aws-java-sns-1.0/src/test/groovy/SnsClientTest.groovy +++ b/dd-java-agent/instrumentation/aws-java/aws-java-sns-1.0/src/test/groovy/SnsClientTest.groovy @@ -25,7 +25,7 @@ import spock.lang.Shared abstract class SnsClientTest extends VersionedNamingTestBase { - static final LOCALSTACK = new GenericContainer(DockerImageName.parse("localstack/localstack:4.2.0")) + static final LOCALSTACK = new GenericContainer(DockerImageName.parse(System.getProperty("test.localstack.image"))) .withExposedPorts(4566) // Default LocalStack port .withEnv("SERVICES", "sns,sqs") // Enable SNS and SQS service .withReuse(true) diff --git a/dd-java-agent/instrumentation/aws-java/aws-java-sns-2.0/build.gradle b/dd-java-agent/instrumentation/aws-java/aws-java-sns-2.0/build.gradle index 63610c31c8c..ad24d8fbefe 100644 --- a/dd-java-agent/instrumentation/aws-java/aws-java-sns-2.0/build.gradle +++ b/dd-java-agent/instrumentation/aws-java/aws-java-sns-2.0/build.gradle @@ -1,5 +1,6 @@ plugins { id 'dd-trace-java.module.instrumentation' + id 'dd-trace-java.testcontainers' } muzzle { @@ -16,6 +17,7 @@ addTestSuiteExtendingForDir('latestDepForkedTest', 'latestDepTest', 'test') dependencies { compileOnly group: 'software.amazon.awssdk', name: 'sns', version: '2.25.40' + testContainerImage(image('localstack/localstack:4.2.0', 'test.localstack.image')) testImplementation project(':dd-java-agent:instrumentation:aws-java:aws-java-common') // Include httpclient instrumentation for testing because it is a dependency for aws-sdk. diff --git a/dd-java-agent/instrumentation/aws-java/aws-java-sns-2.0/src/test/groovy/SnsClientTest.groovy b/dd-java-agent/instrumentation/aws-java/aws-java-sns-2.0/src/test/groovy/SnsClientTest.groovy index d340515d7a1..e6c6930f681 100644 --- a/dd-java-agent/instrumentation/aws-java/aws-java-sns-2.0/src/test/groovy/SnsClientTest.groovy +++ b/dd-java-agent/instrumentation/aws-java/aws-java-sns-2.0/src/test/groovy/SnsClientTest.groovy @@ -23,7 +23,7 @@ import java.time.Duration import static datadog.trace.agent.test.utils.TraceUtils.basicSpan abstract class SnsClientTest extends VersionedNamingTestBase { - static final LOCALSTACK = new GenericContainer(DockerImageName.parse("localstack/localstack:4.2.0")) + static final LOCALSTACK = new GenericContainer(DockerImageName.parse(System.getProperty("test.localstack.image"))) .withExposedPorts(4566) // Default LocalStack port .withEnv("SERVICES", "sns,sqs") // Enable SNS and SQS service .withReuse(true) diff --git a/dd-java-agent/instrumentation/couchbase/couchbase-3.1/build.gradle b/dd-java-agent/instrumentation/couchbase/couchbase-3.1/build.gradle index 9be16049b71..1ac8cf0bbe7 100644 --- a/dd-java-agent/instrumentation/couchbase/couchbase-3.1/build.gradle +++ b/dd-java-agent/instrumentation/couchbase/couchbase-3.1/build.gradle @@ -1,5 +1,8 @@ +import org.gradle.nativeplatform.platform.internal.DefaultNativePlatform + plugins { id 'dd-trace-java.module.instrumentation' + id 'dd-trace-java.testcontainers' } muzzle { @@ -17,6 +20,11 @@ addTestSuiteForDir('latestDepTest', 'test') dependencies { compileOnly group: 'com.couchbase.client', name: 'java-client', version: '3.1.0' + testContainerImage(image( + DefaultNativePlatform.getCurrentArchitecture().isArm64() + ? 'couchbase/server:7.1.0-aarch64' + : 'couchbase/server:7.1.0', + 'test.couchbase.image')) testImplementation group: 'com.couchbase.client', name: 'java-client', version: '3.1.0' testImplementation group: 'org.testcontainers', name: 'couchbase', version: libs.versions.testcontainers.get() diff --git a/dd-java-agent/instrumentation/couchbase/couchbase-3.1/src/test/groovy/CouchbaseClient31Test.groovy b/dd-java-agent/instrumentation/couchbase/couchbase-3.1/src/test/groovy/CouchbaseClient31Test.groovy index e37cda84896..e55dc3c8a32 100644 --- a/dd-java-agent/instrumentation/couchbase/couchbase-3.1/src/test/groovy/CouchbaseClient31Test.groovy +++ b/dd-java-agent/instrumentation/couchbase/couchbase-3.1/src/test/groovy/CouchbaseClient31Test.groovy @@ -21,6 +21,7 @@ import org.slf4j.LoggerFactory import java.time.Duration import org.testcontainers.couchbase.BucketDefinition import org.testcontainers.couchbase.CouchbaseContainer +import org.testcontainers.utility.DockerImageName import spock.lang.Shared import static datadog.trace.agent.test.utils.TraceUtils.basicSpan @@ -39,8 +40,9 @@ abstract class CouchbaseClient31Test extends VersionedNamingTestBase { Bucket bucket def setupSpec() { - def arch = System.getProperty("os.arch") == "aarch64" ? "-aarch64" : "" - couchbase = new CouchbaseContainer("couchbase/server:7.1.0${arch}") + couchbase = new CouchbaseContainer( + DockerImageName.parse(System.getProperty("test.couchbase.image")) + .asCompatibleSubstituteFor("couchbase/server")) .withBucket(new BucketDefinition(BUCKET).withPrimaryIndex(true)) .withStartupTimeout(Duration.ofSeconds(240)) .withStartupAttempts(3) diff --git a/dd-java-agent/instrumentation/couchbase/couchbase-3.2/build.gradle b/dd-java-agent/instrumentation/couchbase/couchbase-3.2/build.gradle index 60459d3f2b2..18acd377bca 100644 --- a/dd-java-agent/instrumentation/couchbase/couchbase-3.2/build.gradle +++ b/dd-java-agent/instrumentation/couchbase/couchbase-3.2/build.gradle @@ -1,5 +1,8 @@ +import org.gradle.nativeplatform.platform.internal.DefaultNativePlatform + plugins { id 'dd-trace-java.module.instrumentation' + id 'dd-trace-java.testcontainers' } muzzle { @@ -17,6 +20,11 @@ addTestSuiteForDir('latestDepTest', 'test') dependencies { compileOnly group: 'com.couchbase.client', name: 'java-client', version: '3.2.0' + testContainerImage(image( + DefaultNativePlatform.getCurrentArchitecture().isArm64() + ? 'couchbase/server:7.1.0-aarch64' + : 'couchbase/server:7.1.0', + 'test.couchbase.image')) testImplementation group: 'com.couchbase.client', name: 'java-client', version: '3.2.0' testImplementation group: 'org.testcontainers', name: 'couchbase', version: libs.versions.testcontainers.get() diff --git a/dd-java-agent/instrumentation/couchbase/couchbase-3.2/src/test/groovy/CouchbaseClient32Test.groovy b/dd-java-agent/instrumentation/couchbase/couchbase-3.2/src/test/groovy/CouchbaseClient32Test.groovy index ff7e9968cc8..b55d08a9efa 100644 --- a/dd-java-agent/instrumentation/couchbase/couchbase-3.2/src/test/groovy/CouchbaseClient32Test.groovy +++ b/dd-java-agent/instrumentation/couchbase/couchbase-3.2/src/test/groovy/CouchbaseClient32Test.groovy @@ -28,6 +28,7 @@ import org.slf4j.Logger import org.slf4j.LoggerFactory import org.testcontainers.couchbase.BucketDefinition import org.testcontainers.couchbase.CouchbaseContainer +import org.testcontainers.utility.DockerImageName import reactor.core.publisher.Mono import spock.lang.Shared @@ -45,8 +46,9 @@ abstract class CouchbaseClient32Test extends VersionedNamingTestBase { Bucket bucket def setupSpec() { - def arch = System.getProperty("os.arch") == "aarch64" ? "-aarch64" : "" - couchbase = new CouchbaseContainer("couchbase/server:7.1.0${arch}") + couchbase = new CouchbaseContainer( + DockerImageName.parse(System.getProperty("test.couchbase.image")) + .asCompatibleSubstituteFor("couchbase/server")) .withBucket(new BucketDefinition(BUCKET).withPrimaryIndex(true)) .withStartupTimeout(Duration.ofSeconds(240)) .withStartupAttempts(3) diff --git a/dd-java-agent/instrumentation/jdbc/src/test/groovy/RemoteJDBCInstrumentationTest.groovy b/dd-java-agent/instrumentation/jdbc/src/test/groovy/RemoteJDBCInstrumentationTest.groovy index d8608095149..2870bca13d3 100644 --- a/dd-java-agent/instrumentation/jdbc/src/test/groovy/RemoteJDBCInstrumentationTest.groovy +++ b/dd-java-agent/instrumentation/jdbc/src/test/groovy/RemoteJDBCInstrumentationTest.groovy @@ -204,7 +204,7 @@ abstract class RemoteJDBCInstrumentationTest extends VersionedNamingTestBase { def image = DockerImageName.parse(System.getProperty("test.sqlserver.image")) .asCompatibleSubstituteFor(MSSQLServerContainer.IMAGE) - MSSQLServerContainer server = new MSSQLServerContainer(image) + def server = new MSSQLServerContainer(image) .acceptLicense() .withPassword(jdbcPasswords.get(SQLSERVER)) // SQL Server can occasionally abort while booting on virtualized CI hosts. diff --git a/dd-java-agent/instrumentation/lettuce/lettuce-5.0/build.gradle b/dd-java-agent/instrumentation/lettuce/lettuce-5.0/build.gradle index 14f2768a709..8c75a2232d6 100644 --- a/dd-java-agent/instrumentation/lettuce/lettuce-5.0/build.gradle +++ b/dd-java-agent/instrumentation/lettuce/lettuce-5.0/build.gradle @@ -1,6 +1,7 @@ plugins { id 'dd-trace-java.module.instrumentation' id 'dd-trace-java.jmh-conventions' + id 'dd-trace-java.testcontainers' } muzzle { @@ -22,6 +23,7 @@ addTestSuiteForDir('lettuce62Test', 'test') dependencies { compileOnly group: 'io.lettuce', name: 'lettuce-core', version: '5.0.0.RELEASE' jmh group: 'io.lettuce', name: 'lettuce-core', version: '5.0.0.RELEASE' + testContainerImage(image('redis:6.2.6', 'test.redis.image')) testImplementation group: 'com.redis.testcontainers', name: 'testcontainers-redis', version: '1.6.2' testImplementation libs.testcontainers diff --git a/dd-java-agent/instrumentation/lettuce/lettuce-5.0/src/test/groovy/Lettuce5ClientTestBase.groovy b/dd-java-agent/instrumentation/lettuce/lettuce-5.0/src/test/groovy/Lettuce5ClientTestBase.groovy index 9749dbdfb0f..ecff8ad6984 100644 --- a/dd-java-agent/instrumentation/lettuce/lettuce-5.0/src/test/groovy/Lettuce5ClientTestBase.groovy +++ b/dd-java-agent/instrumentation/lettuce/lettuce-5.0/src/test/groovy/Lettuce5ClientTestBase.groovy @@ -1,3 +1,5 @@ +import static datadog.trace.agent.test.utils.TraceUtils.runUnderTrace + import com.redis.testcontainers.RedisContainer import datadog.trace.agent.test.naming.VersionedNamingTestBase import datadog.trace.agent.test.utils.PortUtils @@ -12,8 +14,6 @@ import org.testcontainers.utility.DockerImageName import spock.lang.Shared import spock.util.concurrent.PollingConditions -import static datadog.trace.agent.test.utils.TraceUtils.runUnderTrace - abstract class Lettuce5ClientTestBase extends VersionedNamingTestBase { public static final int DB_INDEX = 0 // Disable autoreconnect so we do not get stray traces popping up on server shutdown @@ -21,9 +21,9 @@ abstract class Lettuce5ClientTestBase extends VersionedNamingTestBase { @Shared Map testHashMap = [ - firstname: "John", - lastname : "Doe", - age : "53" + firstname: "John", + lastname : "Doe", + age : "53" ] int port @@ -33,8 +33,10 @@ abstract class Lettuce5ClientTestBase extends VersionedNamingTestBase { String dbUriNonExistent String embeddedDbUri - RedisContainer redisServer = new RedisContainer(DockerImageName.parse("redis:6.2.6")) - .waitingFor(Wait.forListeningPort()) + RedisContainer redisServer = new RedisContainer( + DockerImageName.parse(System.getProperty("test.redis.image")) + .asCompatibleSubstituteFor("redis")) + .waitingFor(Wait.forListeningPort()) RedisClient redisClient StatefulRedisConnection connection diff --git a/dd-java-agent/instrumentation/lettuce/lettuce-5.0/src/test/java/Lettuce5ClusterTest.java b/dd-java-agent/instrumentation/lettuce/lettuce-5.0/src/test/java/Lettuce5ClusterTest.java index ccefb34e0b7..70f56efaf5f 100644 --- a/dd-java-agent/instrumentation/lettuce/lettuce-5.0/src/test/java/Lettuce5ClusterTest.java +++ b/dd-java-agent/instrumentation/lettuce/lettuce-5.0/src/test/java/Lettuce5ClusterTest.java @@ -26,6 +26,7 @@ import org.junit.jupiter.api.Test; import org.testcontainers.containers.GenericContainer; import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.utility.DockerImageName; class Lettuce5ClusterTest extends AbstractInstrumentationTest { private static final String TEST_SET_KEY = "TESTSETKEY"; @@ -53,7 +54,8 @@ void setUpRedis() throws Exception { // Redis cluster discovery returns the announced node port, so the host-side port must be // stable. Use the same random ports inside the container so cluster nodes can also reach each // other at their announced addresses. - redisCluster = new GenericContainer<>("redis:6.2.6"); + redisCluster = + new GenericContainer<>(DockerImageName.parse(System.getProperty("test.redis.image"))); redisCluster.setPortBindings( Arrays.asList( redisClusterMasterPort + ":" + redisClusterMasterPort, diff --git a/dd-java-agent/instrumentation/lettuce/lettuce-5.0/src/test/java/Lettuce5MasterReplicaTest.java b/dd-java-agent/instrumentation/lettuce/lettuce-5.0/src/test/java/Lettuce5MasterReplicaTest.java index 0b1e0a1c3f3..c7f01bbd0f4 100644 --- a/dd-java-agent/instrumentation/lettuce/lettuce-5.0/src/test/java/Lettuce5MasterReplicaTest.java +++ b/dd-java-agent/instrumentation/lettuce/lettuce-5.0/src/test/java/Lettuce5MasterReplicaTest.java @@ -33,7 +33,9 @@ class Lettuce5MasterReplicaTest extends AbstractInstrumentationTest { @BeforeEach void setUpRedis() throws Exception { redisServer = - new RedisContainer(DockerImageName.parse("redis:6.2.6")) + new RedisContainer( + DockerImageName.parse(System.getProperty("test.redis.image")) + .asCompatibleSubstituteFor("redis")) .waitingFor(Wait.forListeningPort()); redisServer.start(); diff --git a/dd-java-agent/instrumentation/mongo/mongo-common/src/testFixtures/groovy/MongoBaseTest.groovy b/dd-java-agent/instrumentation/mongo/mongo-common/src/testFixtures/groovy/MongoBaseTest.groovy index 55f686f712b..e45286d6438 100644 --- a/dd-java-agent/instrumentation/mongo/mongo-common/src/testFixtures/groovy/MongoBaseTest.groovy +++ b/dd-java-agent/instrumentation/mongo/mongo-common/src/testFixtures/groovy/MongoBaseTest.groovy @@ -7,6 +7,7 @@ import datadog.trace.bootstrap.instrumentation.api.Tags import datadog.trace.core.DDSpan import org.slf4j.LoggerFactory import org.testcontainers.containers.MongoDBContainer +import org.testcontainers.utility.DockerImageName import spock.lang.Shared abstract class MongoBaseTest extends VersionedNamingTestBase { @@ -32,12 +33,10 @@ abstract class MongoBaseTest extends VersionedNamingTestBase { abstract String dbType() - def mongodbImageName() { - return "mongo:4.4.29" - } - def setupSpec() throws Exception { - mongoDbContainer = new MongoDBContainer(mongodbImageName()) + mongoDbContainer = new MongoDBContainer( + DockerImageName.parse(System.getProperty("test.mongo.image")) + .asCompatibleSubstituteFor("mongo")) mongoDbContainer.start() port = mongoDbContainer.getMappedPort(27017) logger.info("MongoDB started on port {}", port) diff --git a/dd-java-agent/instrumentation/mongo/mongo-driver/mongo-driver-3/mongo-driver-3.1/build.gradle b/dd-java-agent/instrumentation/mongo/mongo-driver/mongo-driver-3/mongo-driver-3.1/build.gradle index 29a894a4d1d..54fd338b83c 100644 --- a/dd-java-agent/instrumentation/mongo/mongo-driver/mongo-driver-3/mongo-driver-3.1/build.gradle +++ b/dd-java-agent/instrumentation/mongo/mongo-driver/mongo-driver-3/mongo-driver-3.1/build.gradle @@ -1,5 +1,6 @@ plugins { id 'dd-trace-java.module.instrumentation' + id 'dd-trace-java.testcontainers' } muzzle { @@ -29,6 +30,7 @@ dependencies { testImplementation libs.bundles.junit5 compileOnly group: 'org.mongodb', name: 'mongo-java-driver', version: '3.1.0' + testContainerImage(image('mongo:4.4.29', 'test.mongo.image')) implementation(project(':dd-java-agent:instrumentation:mongo:mongo-common')) { transitive = false diff --git a/dd-java-agent/instrumentation/mongo/mongo-driver/mongo-driver-3/mongo-driver-3.4/build.gradle b/dd-java-agent/instrumentation/mongo/mongo-driver/mongo-driver-3/mongo-driver-3.4/build.gradle index ac693dbfd96..67d1debc317 100644 --- a/dd-java-agent/instrumentation/mongo/mongo-driver/mongo-driver-3/mongo-driver-3.4/build.gradle +++ b/dd-java-agent/instrumentation/mongo/mongo-driver/mongo-driver-3/mongo-driver-3.4/build.gradle @@ -1,5 +1,6 @@ plugins { id 'dd-trace-java.module.instrumentation' + id 'dd-trace-java.testcontainers' } muzzle { @@ -36,6 +37,7 @@ dependencies { compileOnly group: 'org.mongodb', name: 'mongo-java-driver', version: '3.4.0' compileOnly group: 'org.mongodb', name: 'mongodb-driver-core', version: '3.4.0' + testContainerImage(image('mongo:4.4.29', 'test.mongo.image')) implementation(project(':dd-java-agent:instrumentation:mongo:mongo-common')) { transitive = false diff --git a/dd-java-agent/instrumentation/mongo/mongo-driver/mongo-driver-3/mongo-driver-3.6/build.gradle b/dd-java-agent/instrumentation/mongo/mongo-driver/mongo-driver-3/mongo-driver-3.6/build.gradle index bbc11a5a13c..88c0800beeb 100644 --- a/dd-java-agent/instrumentation/mongo/mongo-driver/mongo-driver-3/mongo-driver-3.6/build.gradle +++ b/dd-java-agent/instrumentation/mongo/mongo-driver/mongo-driver-3/mongo-driver-3.6/build.gradle @@ -1,5 +1,6 @@ plugins { id 'dd-trace-java.module.instrumentation' + id 'dd-trace-java.testcontainers' } muzzle { @@ -34,6 +35,7 @@ addTestSuiteForDir('latestDepTest', 'test') dependencies { compileOnly group: 'org.mongodb', name: 'mongo-java-driver', version: '3.6.0' compileOnly group: 'org.mongodb', name: 'mongodb-driver-core', version: '3.6.0' + testContainerImage(image('mongo:4.4.29', 'test.mongo.image')) implementation(project(':dd-java-agent:instrumentation:mongo:mongo-common')) { transitive = false diff --git a/dd-java-agent/instrumentation/mongo/mongo-driver/mongo-driver-3/mongo-driver-3.8/build.gradle b/dd-java-agent/instrumentation/mongo/mongo-driver/mongo-driver-3/mongo-driver-3.8/build.gradle index 71a55e33118..174c7c45621 100644 --- a/dd-java-agent/instrumentation/mongo/mongo-driver/mongo-driver-3/mongo-driver-3.8/build.gradle +++ b/dd-java-agent/instrumentation/mongo/mongo-driver/mongo-driver-3/mongo-driver-3.8/build.gradle @@ -1,5 +1,6 @@ plugins { id 'dd-trace-java.module.instrumentation' + id 'dd-trace-java.testcontainers' } muzzle { @@ -34,6 +35,7 @@ addTestSuiteForDir('latestDepTest', 'test') dependencies { compileOnly group: 'org.mongodb', name: 'mongo-java-driver', version: '3.8.0' compileOnly group: 'org.mongodb', name: 'mongodb-driver-core', version: '3.8.0' + testContainerImage(image('mongo:4.4.29', 'test.mongo.image')) implementation(project(':dd-java-agent:instrumentation:mongo:mongo-common')) { transitive = false diff --git a/dd-java-agent/instrumentation/mongo/mongo-driver/mongo-driver-4.0/build.gradle b/dd-java-agent/instrumentation/mongo/mongo-driver/mongo-driver-4.0/build.gradle index b4ee761702c..5e47265167a 100644 --- a/dd-java-agent/instrumentation/mongo/mongo-driver/mongo-driver-4.0/build.gradle +++ b/dd-java-agent/instrumentation/mongo/mongo-driver/mongo-driver-4.0/build.gradle @@ -1,5 +1,6 @@ plugins { id 'dd-trace-java.module.instrumentation' + id 'dd-trace-java.testcontainers' } muzzle { @@ -27,6 +28,7 @@ addTestSuiteForDir('mongo410ForkedTest', 'test') dependencies { compileOnly group: 'org.mongodb', name: 'mongodb-driver-sync', version: '4.0.0' compileOnly group: 'org.mongodb', name: 'mongodb-driver-reactivestreams', version: '4.0.0' + testContainerImage(image('mongo:4.4.29', 'test.mongo.image')) implementation(project(':dd-java-agent:instrumentation:mongo:mongo-common')) { transitive = false diff --git a/dd-java-agent/instrumentation/mongo/mongo-test/mongo-test-async-3.3/build.gradle b/dd-java-agent/instrumentation/mongo/mongo-test/mongo-test-async-3.3/build.gradle index 9c87d0f1187..8f6f239b6b1 100644 --- a/dd-java-agent/instrumentation/mongo/mongo-test/mongo-test-async-3.3/build.gradle +++ b/dd-java-agent/instrumentation/mongo/mongo-test/mongo-test-async-3.3/build.gradle @@ -1,10 +1,13 @@ plugins { id 'dd-trace-java.module.instrumentation' + id 'dd-trace-java.testcontainers' } addTestSuiteForDir('latestDepTest', 'test') dependencies { + testContainerImage(image('mongo:4.4.29', 'test.mongo.image')) + testImplementation(project(':dd-java-agent:instrumentation:mongo:mongo-common')) { transitive = false } diff --git a/dd-java-agent/instrumentation/mongo/mongo-test/mongo-test-core-3.1/build.gradle b/dd-java-agent/instrumentation/mongo/mongo-test/mongo-test-core-3.1/build.gradle index ccf44f9650e..48d6f90e7a0 100644 --- a/dd-java-agent/instrumentation/mongo/mongo-test/mongo-test-core-3.1/build.gradle +++ b/dd-java-agent/instrumentation/mongo/mongo-test/mongo-test-core-3.1/build.gradle @@ -1,10 +1,13 @@ plugins { id 'dd-trace-java.module.instrumentation' + id 'dd-trace-java.testcontainers' } addTestSuiteForDir('latestDepTest', 'test') dependencies { + testContainerImage(image('mongo:4.4.29', 'test.mongo.image')) + testImplementation group: 'org.testcontainers', name: 'mongodb', version: libs.versions.testcontainers.get() // We need to pull in this dependency to get the 'suspend span' instrumentation for spock tests @@ -22,4 +25,3 @@ dependencies { testImplementation group: 'org.mongodb', name: 'mongodb-driver', version: '3.1.0' latestDepTestImplementation group: 'org.mongodb', name: 'mongodb-driver', version: '3.6+' } - diff --git a/dd-java-agent/instrumentation/mongo/mongo-test/mongo-test-core-3.7/build.gradle b/dd-java-agent/instrumentation/mongo/mongo-test/mongo-test-core-3.7/build.gradle index f436e6a8905..4bbfb9a8b96 100644 --- a/dd-java-agent/instrumentation/mongo/mongo-test/mongo-test-core-3.7/build.gradle +++ b/dd-java-agent/instrumentation/mongo/mongo-test/mongo-test-core-3.7/build.gradle @@ -1,10 +1,13 @@ plugins { id 'dd-trace-java.module.instrumentation' + id 'dd-trace-java.testcontainers' } addTestSuiteForDir('latestDepTest', 'test') dependencies { + testContainerImage(image('mongo:4.4.29', 'test.mongo.image')) + testImplementation group: 'org.testcontainers', name: 'mongodb', version: libs.versions.testcontainers.get() // We need to pull in this dependency to get the 'suspend span' instrumentation for spock tests diff --git a/dd-java-agent/instrumentation/mongo/mongo-test/mongo-test-sync-3.10/build.gradle b/dd-java-agent/instrumentation/mongo/mongo-test/mongo-test-sync-3.10/build.gradle index 5a5b016f198..e6878f999b8 100644 --- a/dd-java-agent/instrumentation/mongo/mongo-test/mongo-test-sync-3.10/build.gradle +++ b/dd-java-agent/instrumentation/mongo/mongo-test/mongo-test-sync-3.10/build.gradle @@ -1,10 +1,13 @@ plugins { id 'dd-trace-java.module.instrumentation' + id 'dd-trace-java.testcontainers' } addTestSuiteForDir('latestDepTest', 'test') dependencies { + testContainerImage(image('mongo:4.4.29', 'test.mongo.image')) + testImplementation(project(':dd-java-agent:instrumentation:mongo:mongo-common')) { transitive = false } diff --git a/dd-java-agent/instrumentation/rabbitmq-amqp-2.7/build.gradle b/dd-java-agent/instrumentation/rabbitmq-amqp-2.7/build.gradle index c7b17611d5b..01fd1960aff 100644 --- a/dd-java-agent/instrumentation/rabbitmq-amqp-2.7/build.gradle +++ b/dd-java-agent/instrumentation/rabbitmq-amqp-2.7/build.gradle @@ -1,5 +1,6 @@ plugins { id 'dd-trace-java.module.instrumentation' + id 'dd-trace-java.testcontainers' } muzzle { @@ -18,6 +19,7 @@ addTestSuite('reactorTest') dependencies { compileOnly group: 'com.rabbitmq', name: 'amqp-client', version: '2.7.0' + testContainerImage(image('rabbitmq:3.9.20-alpine', 'test.rabbitmq.image')) testImplementation group: 'com.rabbitmq', name: 'amqp-client', version: '2.7.0' testImplementation group: 'org.springframework.amqp', name: 'spring-rabbit', version: '1.1.0.RELEASE' diff --git a/dd-java-agent/instrumentation/rabbitmq-amqp-2.7/src/reactorTest/groovy/ReactorRabbitMQTest.groovy b/dd-java-agent/instrumentation/rabbitmq-amqp-2.7/src/reactorTest/groovy/ReactorRabbitMQTest.groovy index da056550c8f..804f7ccc6df 100644 --- a/dd-java-agent/instrumentation/rabbitmq-amqp-2.7/src/reactorTest/groovy/ReactorRabbitMQTest.groovy +++ b/dd-java-agent/instrumentation/rabbitmq-amqp-2.7/src/reactorTest/groovy/ReactorRabbitMQTest.groovy @@ -4,6 +4,7 @@ import com.rabbitmq.client.ConnectionFactory import datadog.trace.agent.test.InstrumentationSpecification import datadog.trace.agent.test.utils.PortUtils import org.testcontainers.containers.RabbitMQContainer +import org.testcontainers.utility.DockerImageName import reactor.core.publisher.Mono import reactor.core.scheduler.Schedulers import reactor.rabbitmq.RabbitFlux @@ -29,7 +30,9 @@ class ReactorRabbitMQTest extends InstrumentationSpecification { } def setupSpec() { - rabbitMQContainer = new RabbitMQContainer('rabbitmq:3.9.20-alpine') + rabbitMQContainer = new RabbitMQContainer( + DockerImageName.parse(System.getProperty("test.rabbitmq.image")) + .asCompatibleSubstituteFor("rabbitmq")) .withExposedPorts(defaultRabbitMQPort) .withStartupTimeout(Duration.ofSeconds(120)) rabbitMQContainer.start() diff --git a/dd-java-agent/instrumentation/rabbitmq-amqp-2.7/src/test/groovy/RabbitMQTest.groovy b/dd-java-agent/instrumentation/rabbitmq-amqp-2.7/src/test/groovy/RabbitMQTest.groovy index 966ab9d4b80..1308127dbf0 100644 --- a/dd-java-agent/instrumentation/rabbitmq-amqp-2.7/src/test/groovy/RabbitMQTest.groovy +++ b/dd-java-agent/instrumentation/rabbitmq-amqp-2.7/src/test/groovy/RabbitMQTest.groovy @@ -26,6 +26,7 @@ import org.springframework.amqp.rabbit.connection.CachingConnectionFactory import org.springframework.amqp.rabbit.core.RabbitAdmin import org.springframework.amqp.rabbit.core.RabbitTemplate import org.testcontainers.containers.RabbitMQContainer +import org.testcontainers.utility.DockerImageName import spock.lang.Shared import spock.util.concurrent.PollingConditions @@ -70,7 +71,9 @@ abstract class RabbitMQTestBase extends VersionedNamingTestBase { } def setupSpec() { - rabbitMQContainer = new RabbitMQContainer('rabbitmq:3.9.20-alpine') + rabbitMQContainer = new RabbitMQContainer( + DockerImageName.parse(System.getProperty("test.rabbitmq.image")) + .asCompatibleSubstituteFor("rabbitmq")) .withExposedPorts(defaultRabbitMQPort) .withStartupTimeout(Duration.ofSeconds(120)) rabbitMQContainer.start() diff --git a/dd-java-agent/instrumentation/redisson/redisson-2.0.0/build.gradle b/dd-java-agent/instrumentation/redisson/redisson-2.0.0/build.gradle index 3b79ecba0f6..e977ce3a118 100644 --- a/dd-java-agent/instrumentation/redisson/redisson-2.0.0/build.gradle +++ b/dd-java-agent/instrumentation/redisson/redisson-2.0.0/build.gradle @@ -1,5 +1,6 @@ plugins { id 'dd-trace-java.module.instrumentation' + id 'dd-trace-java.testcontainers' } muzzle { @@ -17,6 +18,7 @@ addTestSuiteExtendingForDir('latestDepForkedTest', 'latestDepTest', 'test') dependencies { compileOnly group: 'org.redisson', name: 'redisson', version: '2.0.0' + testContainerImage(image('redis:6.2.6', 'test.redis.image')) testImplementation group: 'com.redis.testcontainers', name: 'testcontainers-redis', version: '1.6.2' testImplementation libs.testcontainers diff --git a/dd-java-agent/instrumentation/redisson/redisson-2.0.0/src/test/groovy/RedissonClientTest.groovy b/dd-java-agent/instrumentation/redisson/redisson-2.0.0/src/test/groovy/RedissonClientTest.groovy index 3201a5f6969..bf3ce441cef 100644 --- a/dd-java-agent/instrumentation/redisson/redisson-2.0.0/src/test/groovy/RedissonClientTest.groovy +++ b/dd-java-agent/instrumentation/redisson/redisson-2.0.0/src/test/groovy/RedissonClientTest.groovy @@ -19,7 +19,9 @@ import spock.lang.Shared abstract class RedissonClientTest extends VersionedNamingTestBase { @Shared - RedisServer redisServer = new RedisContainer(DockerImageName.parse("redis:6.2.6")).waitingFor(Wait.forListeningPort()) + RedisServer redisServer = new RedisContainer( + DockerImageName.parse(System.getProperty("test.redis.image"))) + .waitingFor(Wait.forListeningPort()) @Shared Config config = new Config() diff --git a/dd-java-agent/instrumentation/redisson/redisson-2.3.0/build.gradle b/dd-java-agent/instrumentation/redisson/redisson-2.3.0/build.gradle index 8ac63db2bdb..ae779c8397c 100644 --- a/dd-java-agent/instrumentation/redisson/redisson-2.3.0/build.gradle +++ b/dd-java-agent/instrumentation/redisson/redisson-2.3.0/build.gradle @@ -1,5 +1,6 @@ plugins { id 'dd-trace-java.module.instrumentation' + id 'dd-trace-java.testcontainers' } muzzle { @@ -17,6 +18,7 @@ addTestSuiteExtendingForDir('latestDepForkedTest', 'latestDepTest', 'test') dependencies { compileOnly group: 'org.redisson', name: 'redisson', version: '2.3.0' + testContainerImage(image('redis:6.2.6', 'test.redis.image')) testImplementation group: 'com.redis.testcontainers', name: 'testcontainers-redis', version: '1.6.2' testImplementation libs.testcontainers diff --git a/dd-java-agent/instrumentation/redisson/redisson-2.3.0/src/test/groovy/RedissonClientTest.groovy b/dd-java-agent/instrumentation/redisson/redisson-2.3.0/src/test/groovy/RedissonClientTest.groovy index d8e0c5bbc78..286a63ad83d 100644 --- a/dd-java-agent/instrumentation/redisson/redisson-2.3.0/src/test/groovy/RedissonClientTest.groovy +++ b/dd-java-agent/instrumentation/redisson/redisson-2.3.0/src/test/groovy/RedissonClientTest.groovy @@ -16,7 +16,9 @@ import static datadog.trace.api.config.TraceInstrumentationConfig.DB_CLIENT_HOST abstract class RedissonClientTest extends VersionedNamingTestBase { @Shared - RedisServer redisServer = new RedisContainer(DockerImageName.parse("redis:6.2.6")).waitingFor(Wait.forListeningPort()) + RedisServer redisServer = new RedisContainer( + DockerImageName.parse(System.getProperty("test.redis.image"))) + .waitingFor(Wait.forListeningPort()) @Shared Config config = new Config() diff --git a/dd-java-agent/instrumentation/redisson/redisson-3.10.3/build.gradle b/dd-java-agent/instrumentation/redisson/redisson-3.10.3/build.gradle index 159330befa4..00894b36155 100644 --- a/dd-java-agent/instrumentation/redisson/redisson-3.10.3/build.gradle +++ b/dd-java-agent/instrumentation/redisson/redisson-3.10.3/build.gradle @@ -1,5 +1,6 @@ plugins { id 'dd-trace-java.module.instrumentation' + id 'dd-trace-java.testcontainers' } muzzle { @@ -19,6 +20,7 @@ dependencies { compileOnly group: 'org.redisson', name: 'redisson', version: '3.10.3', { exclude group: 'org.slf4j', module: 'slf4j-api' } + testContainerImage(image('redis:6.2.6', 'test.redis.image')) testImplementation group: 'com.redis.testcontainers', name: 'testcontainers-redis', version: '1.6.2' testImplementation libs.testcontainers diff --git a/dd-java-agent/instrumentation/redisson/redisson-3.10.3/src/test/groovy/RedissonClientTest.groovy b/dd-java-agent/instrumentation/redisson/redisson-3.10.3/src/test/groovy/RedissonClientTest.groovy index d1292ab48fd..43562d2b55f 100644 --- a/dd-java-agent/instrumentation/redisson/redisson-3.10.3/src/test/groovy/RedissonClientTest.groovy +++ b/dd-java-agent/instrumentation/redisson/redisson-3.10.3/src/test/groovy/RedissonClientTest.groovy @@ -16,7 +16,9 @@ import static datadog.trace.api.config.TraceInstrumentationConfig.DB_CLIENT_HOST abstract class RedissonClientTest extends VersionedNamingTestBase { @Shared - RedisServer redisServer = new RedisContainer(DockerImageName.parse("redis:6.2.6")).waitingFor(Wait.forListeningPort()) + RedisServer redisServer = new RedisContainer( + DockerImageName.parse(System.getProperty("test.redis.image"))) + .waitingFor(Wait.forListeningPort()) @Shared Config config = new Config() diff --git a/dd-java-agent/instrumentation/spring/spring-rabbit-1.5/build.gradle b/dd-java-agent/instrumentation/spring/spring-rabbit-1.5/build.gradle index 78cfe257bbc..b7270924bee 100644 --- a/dd-java-agent/instrumentation/spring/spring-rabbit-1.5/build.gradle +++ b/dd-java-agent/instrumentation/spring/spring-rabbit-1.5/build.gradle @@ -1,5 +1,6 @@ plugins { id 'dd-trace-java.module.instrumentation' + id 'dd-trace-java.testcontainers' } muzzle { @@ -17,6 +18,7 @@ addTestSuiteForDir('latestDepTest', 'test') dependencies { compileOnly group: 'org.springframework.amqp', name: 'spring-rabbit', version: '2.0.0.RELEASE' compileOnly group: 'org.springframework.amqp', name: 'spring-amqp', version: '2.0.0.RELEASE' + testContainerImage(image('rabbitmq:3.9.20-alpine', 'test.rabbitmq.image')) testImplementation project(':dd-java-agent:instrumentation:datadog:tracing:trace-annotation') testImplementation project(':dd-java-agent:instrumentation:rabbitmq-amqp-2.7') diff --git a/dd-java-agent/instrumentation/spring/spring-rabbit-1.5/src/test/groovy/SpringAmqpTest.groovy b/dd-java-agent/instrumentation/spring/spring-rabbit-1.5/src/test/groovy/SpringAmqpTest.groovy index c76dda008c3..af075e759e8 100644 --- a/dd-java-agent/instrumentation/spring/spring-rabbit-1.5/src/test/groovy/SpringAmqpTest.groovy +++ b/dd-java-agent/instrumentation/spring/spring-rabbit-1.5/src/test/groovy/SpringAmqpTest.groovy @@ -1,6 +1,7 @@ import datadog.trace.agent.test.InstrumentationSpecification import datadog.trace.agent.test.utils.PortUtils import org.testcontainers.containers.RabbitMQContainer +import org.testcontainers.utility.DockerImageName import rabbit.MessagingRabbitMQApplication import rabbit.Receiver import rabbit.Sender @@ -17,7 +18,9 @@ class SpringAmqpTest extends InstrumentationSpecification { @Override def setupSpec() { - rabbit = new RabbitMQContainer("rabbitmq:3.9.20-alpine") + rabbit = new RabbitMQContainer( + DockerImageName.parse(System.getProperty("test.rabbitmq.image")) + .asCompatibleSubstituteFor("rabbitmq")) rabbit.start() def hostName = rabbit.getHost() def port = rabbit.getMappedPort(MessagingRabbitMQApplication.port) diff --git a/dd-java-agent/instrumentation/spymemcached-2.10/build.gradle b/dd-java-agent/instrumentation/spymemcached-2.10/build.gradle index 6563b7c276d..01eec3adc73 100644 --- a/dd-java-agent/instrumentation/spymemcached-2.10/build.gradle +++ b/dd-java-agent/instrumentation/spymemcached-2.10/build.gradle @@ -1,5 +1,6 @@ plugins { id 'dd-trace-java.module.instrumentation' + id 'dd-trace-java.testcontainers' } muzzle { @@ -20,6 +21,7 @@ tasks.named("latestDepTest", Test) { dependencies { compileOnly group: 'net.spy', name: 'spymemcached', version: '2.10.4' + testContainerImage(image('library/memcached:1.6.14-alpine', 'test.memcached.image')) testImplementation group: 'net.spy', name: 'spymemcached', version: '2.10.4' testImplementation libs.testcontainers diff --git a/dd-java-agent/instrumentation/spymemcached-2.10/src/test/groovy/datadog/trace/instrumentation/spymemcached/SpymemcachedTest.groovy b/dd-java-agent/instrumentation/spymemcached-2.10/src/test/groovy/datadog/trace/instrumentation/spymemcached/SpymemcachedTest.groovy index f8d8d3d593d..9c16f96d8e8 100644 --- a/dd-java-agent/instrumentation/spymemcached-2.10/src/test/groovy/datadog/trace/instrumentation/spymemcached/SpymemcachedTest.groovy +++ b/dd-java-agent/instrumentation/spymemcached-2.10/src/test/groovy/datadog/trace/instrumentation/spymemcached/SpymemcachedTest.groovy @@ -15,6 +15,7 @@ import net.spy.memcached.internal.CheckedOperationTimeoutException import net.spy.memcached.ops.Operation import net.spy.memcached.ops.OperationQueueFactory import org.testcontainers.containers.GenericContainer +import org.testcontainers.utility.DockerImageName import spock.lang.Shared import java.time.Duration @@ -56,7 +57,7 @@ abstract class SpymemcachedTest extends VersionedNamingTestBase { } def setupSpec() { - memcachedContainer = new GenericContainer('library/memcached:1.6.14-alpine') + memcachedContainer = new GenericContainer(DockerImageName.parse(System.getProperty('test.memcached.image'))) .withExposedPorts(defaultMemcachedPort) .withStartupTimeout(Duration.ofSeconds(120)) memcachedContainer.start() diff --git a/dd-java-agent/instrumentation/vertx/vertx-mysql-client/vertx-mysql-client-3.9/src/test/java/TestDatabases.java b/dd-java-agent/instrumentation/vertx/vertx-mysql-client/vertx-mysql-client-3.9/src/test/java/TestDatabases.java index 5d7111ae69a..27fecadc279 100644 --- a/dd-java-agent/instrumentation/vertx/vertx-mysql-client/vertx-mysql-client-3.9/src/test/java/TestDatabases.java +++ b/dd-java-agent/instrumentation/vertx/vertx-mysql-client/vertx-mysql-client-3.9/src/test/java/TestDatabases.java @@ -1,6 +1,5 @@ import datadog.trace.agent.test.utils.PortUtils; import java.io.Closeable; -import java.io.IOException; import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -14,14 +13,15 @@ public static TestDatabases initialise(String dbName) { return new TestDatabases(dbName); } - private final MySQLContainer mysql; + private final MySQLContainer mysql; private final Map dbInfos; + @SuppressWarnings("resource") private TestDatabases(String dbName) { Map infos = new HashMap<>(); mysql = - new MySQLContainer( + new MySQLContainer<>( DockerImageName.parse(System.getProperty("test.mysql.image")) .asCompatibleSubstituteFor("mysql")) .withDatabaseName(dbName) @@ -44,7 +44,7 @@ private TestDatabases(String dbName) { } @Override - public void close() throws IOException { + public void close() { if (null != mysql) { mysql.close(); } diff --git a/dd-java-agent/instrumentation/vertx/vertx-mysql-client/vertx-mysql-client-4.0/src/test/java/TestDatabases.java b/dd-java-agent/instrumentation/vertx/vertx-mysql-client/vertx-mysql-client-4.0/src/test/java/TestDatabases.java index e6ed288255b..78ad9a82b20 100644 --- a/dd-java-agent/instrumentation/vertx/vertx-mysql-client/vertx-mysql-client-4.0/src/test/java/TestDatabases.java +++ b/dd-java-agent/instrumentation/vertx/vertx-mysql-client/vertx-mysql-client-4.0/src/test/java/TestDatabases.java @@ -1,6 +1,5 @@ import datadog.trace.agent.test.utils.PortUtils; import java.io.Closeable; -import java.io.IOException; import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -10,13 +9,14 @@ public class TestDatabases implements Closeable { - private final MySQLContainer mysql; + private final MySQLContainer mysql; private final Map dbInfos; + @SuppressWarnings("resource") private TestDatabases(String dbName) { Map infos = new HashMap<>(); mysql = - new MySQLContainer( + new MySQLContainer<>( DockerImageName.parse(System.getProperty("test.mysql.image")) .asCompatibleSubstituteFor("mysql")) .withDatabaseName(dbName) @@ -43,7 +43,7 @@ public static TestDatabases initialise(String dbName) { } @Override - public void close() throws IOException { + public void close() { if (null != mysql) { mysql.close(); } diff --git a/dd-java-agent/instrumentation/vertx/vertx-pg-client/vertx-pg-client-4.0/src/test/java/TestDatabases.java b/dd-java-agent/instrumentation/vertx/vertx-pg-client/vertx-pg-client-4.0/src/test/java/TestDatabases.java index 12f06ccaac3..3e1f72e8d41 100644 --- a/dd-java-agent/instrumentation/vertx/vertx-pg-client/vertx-pg-client-4.0/src/test/java/TestDatabases.java +++ b/dd-java-agent/instrumentation/vertx/vertx-pg-client/vertx-pg-client-4.0/src/test/java/TestDatabases.java @@ -1,6 +1,5 @@ import datadog.trace.agent.test.utils.PortUtils; import java.io.Closeable; -import java.io.IOException; import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -10,13 +9,14 @@ public class TestDatabases implements Closeable { - private final PostgreSQLContainer pgsql; + private final PostgreSQLContainer pgsql; private final Map dbInfos; + @SuppressWarnings("resource") private TestDatabases(String dbName) { Map infos = new HashMap<>(); pgsql = - new PostgreSQLContainer( + new PostgreSQLContainer<>( DockerImageName.parse(System.getProperty("test.postgres.image")) .asCompatibleSubstituteFor("postgres")) .withDatabaseName(dbName) @@ -41,7 +41,7 @@ public static TestDatabases initialise(String dbName) { } @Override - public void close() throws IOException { + public void close() { if (null != pgsql) { pgsql.close(); } diff --git a/dd-java-agent/instrumentation/vertx/vertx-redis-client/vertx-redis-client-3.9/build.gradle b/dd-java-agent/instrumentation/vertx/vertx-redis-client/vertx-redis-client-3.9/build.gradle index 77488806a6b..c5a9cb45db7 100644 --- a/dd-java-agent/instrumentation/vertx/vertx-redis-client/vertx-redis-client-3.9/build.gradle +++ b/dd-java-agent/instrumentation/vertx/vertx-redis-client/vertx-redis-client-3.9/build.gradle @@ -1,5 +1,6 @@ plugins { id 'dd-trace-java.module.instrumentation' + id 'dd-trace-java.testcontainers' } muzzle { @@ -21,6 +22,7 @@ addTestSuiteExtendingForDir('redis4xForkedTest', 'redis4xTest', 'test') dependencies { compileOnly project(':dd-java-agent:instrumentation:vertx:vertx-redis-client:vertx-redis-client-stubs') compileOnly group: 'io.vertx', name: 'vertx-redis-client', version: '3.9.0' + testContainerImage(image('redis:6.2.6', 'test.redis.image')) // only needed for the rx tests testImplementation project(':dd-java-agent:instrumentation:vertx:vertx-rx-3.5') diff --git a/dd-java-agent/instrumentation/vertx/vertx-redis-client/vertx-redis-client-3.9/src/test/groovy/VertxRedisTestBase.groovy b/dd-java-agent/instrumentation/vertx/vertx-redis-client/vertx-redis-client-3.9/src/test/groovy/VertxRedisTestBase.groovy index 8c26b18680a..0c7b5f4abd5 100644 --- a/dd-java-agent/instrumentation/vertx/vertx-redis-client/vertx-redis-client-3.9/src/test/groovy/VertxRedisTestBase.groovy +++ b/dd-java-agent/instrumentation/vertx/vertx-redis-client/vertx-redis-client-3.9/src/test/groovy/VertxRedisTestBase.groovy @@ -36,8 +36,9 @@ abstract class VertxRedisTestBase extends VersionedNamingTestBase { @AutoCleanup(value = "stop") @Shared - def redisServer = new RedisContainer(DockerImageName.parse("redis:6.2.6")) - .waitingFor(Wait.forListeningPort()) + def redisServer = new RedisContainer( + DockerImageName.parse(System.getProperty("test.redis.image"))) + .waitingFor(Wait.forListeningPort()) @Shared @AutoCleanup(quiet = true) diff --git a/dd-smoke-tests/spring-boot-rabbit/build.gradle b/dd-smoke-tests/spring-boot-rabbit/build.gradle index 0c3f554d546..1fa4b6e5d7c 100644 --- a/dd-smoke-tests/spring-boot-rabbit/build.gradle +++ b/dd-smoke-tests/spring-boot-rabbit/build.gradle @@ -3,6 +3,7 @@ import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar plugins { id 'com.gradleup.shadow' id 'dd-trace-java.module.smoke-test' + id 'dd-trace-java.testcontainers' } description = 'SpringBoot RabbitMQ Smoke Tests.' @@ -20,6 +21,8 @@ tasks.named("shadowJar", ShadowJar) { } dependencies { + testContainerImage(image('rabbitmq:3.9.20-alpine', 'test.rabbitmq.image')) + implementation project(':dd-trace-api') implementation group: 'org.springframework.boot', name: 'spring-boot-starter-web', version: '2.5.4' implementation group: 'org.springframework.boot', name: 'spring-boot-starter-amqp', version: '2.5.4' diff --git a/dd-smoke-tests/spring-boot-rabbit/src/test/java/datadog/smoketest/SpringBootRabbitSmokeTest.java b/dd-smoke-tests/spring-boot-rabbit/src/test/java/datadog/smoketest/SpringBootRabbitSmokeTest.java index ed8d42c7c2b..b1213e7469d 100644 --- a/dd-smoke-tests/spring-boot-rabbit/src/test/java/datadog/smoketest/SpringBootRabbitSmokeTest.java +++ b/dd-smoke-tests/spring-boot-rabbit/src/test/java/datadog/smoketest/SpringBootRabbitSmokeTest.java @@ -85,7 +85,9 @@ class SpringBootRabbitSmokeTest { @Container private static final RabbitMQContainer RABBIT = - new RabbitMQContainer(DockerImageName.parse("rabbitmq:3.9.20-alpine")); + new RabbitMQContainer( + DockerImageName.parse(System.getProperty("test.rabbitmq.image")) + .asCompatibleSubstituteFor("rabbitmq")); @Order(1) @RegisterExtension diff --git a/dd-smoke-tests/springboot-mongo/build.gradle b/dd-smoke-tests/springboot-mongo/build.gradle index 4778f981281..48e2eb21aa4 100644 --- a/dd-smoke-tests/springboot-mongo/build.gradle +++ b/dd-smoke-tests/springboot-mongo/build.gradle @@ -3,6 +3,7 @@ import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar plugins { id 'com.gradleup.shadow' id 'dd-trace-java.module.smoke-test' + id 'dd-trace-java.testcontainers' } description = 'SpringBoot Spring-Data-Mongo Smoke Tests.' @@ -20,6 +21,8 @@ tasks.named("shadowJar", ShadowJar) { } dependencies { + testContainerImage(image('mongo:4.4.29', 'test.mongo.image')) + implementation group: 'org.springframework.boot', name: 'spring-boot-starter-web', version: '2.4.1' implementation group: 'org.springframework.boot', name: 'spring-boot-starter-data-mongodb', version: '2.4.1' diff --git a/dd-smoke-tests/springboot-mongo/src/test/groovy/datadog/smoketest/SpringBootMongoIntegrationTest.groovy b/dd-smoke-tests/springboot-mongo/src/test/groovy/datadog/smoketest/SpringBootMongoIntegrationTest.groovy index 76e8743fe00..38a3d62de15 100644 --- a/dd-smoke-tests/springboot-mongo/src/test/groovy/datadog/smoketest/SpringBootMongoIntegrationTest.groovy +++ b/dd-smoke-tests/springboot-mongo/src/test/groovy/datadog/smoketest/SpringBootMongoIntegrationTest.groovy @@ -23,7 +23,9 @@ class SpringBootMongoIntegrationTest extends AbstractServerSmokeTest { @Override void beforeProcessBuilders() { - mongoDbContainer = new MongoDBContainer(DockerImageName.parse("mongo:4.4.29")) + mongoDbContainer = new MongoDBContainer( + DockerImageName.parse(System.getProperty("test.mongo.image")) + .asCompatibleSubstituteFor("mongo")) mongoDbContainer.start() mongoDbUri = mongoDbContainer.replicaSetUrl } diff --git a/dd-trace-core/build.gradle b/dd-trace-core/build.gradle index 5424650d961..46695a38059 100644 --- a/dd-trace-core/build.gradle +++ b/dd-trace-core/build.gradle @@ -2,6 +2,7 @@ plugins { id 'dd-trace-java.module.product-subsystem' id 'dd-trace-java.jmh-conventions' id 'dd-trace-java.version-file' + id 'dd-trace-java.testcontainers' } description = 'dd-trace-core' @@ -74,6 +75,9 @@ tasks.named('forkedTest', Test) { } dependencies { + testContainerImage(image('registry.ddbuild.io/images/mirror/dd-apm-test-agent/ddapm-test-agent:v1.64.1', 'test.ddapm.agent.image')) + traceAgentTestContainerImage(image('datadog/agent:7.40.1', 'test.datadog.agent.image')) + api project(':dd-trace-api') api project(':communication') api project(':internal-api') diff --git a/dd-trace-core/src/test/java/datadog/trace/TracerConnectionReliabilityTest.java b/dd-trace-core/src/test/java/datadog/trace/TracerConnectionReliabilityTest.java index cfda2f730c7..490df7f06a4 100644 --- a/dd-trace-core/src/test/java/datadog/trace/TracerConnectionReliabilityTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/TracerConnectionReliabilityTest.java @@ -131,8 +131,7 @@ GenericContainer startTestAgentContainer() { //noinspection GrDeprecatedAPIUsage Use FixedHostPortGenericContainer against deprecation // because we need to know the exposed to configure the tracer at start GenericContainer agentContainer = - new FixedHostPortGenericContainer( - "registry.ddbuild.io/images/mirror/dd-apm-test-agent/ddapm-test-agent:v1.64.1") + new FixedHostPortGenericContainer(System.getProperty("test.ddapm.agent.image")) .withFixedExposedPort(agentContainerPort, DEFAULT_TRACE_AGENT_PORT) .withEnv( "ENABLED_CHECKS", diff --git a/dd-trace-core/src/traceAgentTest/java/AbstractTraceAgentTest.java b/dd-trace-core/src/traceAgentTest/java/AbstractTraceAgentTest.java index 3c30bbe8cfa..d7885d6b0dd 100644 --- a/dd-trace-core/src/traceAgentTest/java/AbstractTraceAgentTest.java +++ b/dd-trace-core/src/traceAgentTest/java/AbstractTraceAgentTest.java @@ -12,6 +12,7 @@ import org.junit.jupiter.api.BeforeEach; import org.testcontainers.containers.GenericContainer; import org.testcontainers.containers.startupcheck.MinimumDurationRunningStartupCheckStrategy; +import org.testcontainers.utility.DockerImageName; abstract class AbstractTraceAgentTest extends DDJavaSpecification { @@ -30,7 +31,8 @@ static void setupSpec() { env.put("DD_HOSTNAME", "doesnotexist"); env.put("DD_LOGS_STDOUT", "yes"); agentContainer = - new GenericContainer<>("datadog/agent:7.40.1") + new GenericContainer<>( + DockerImageName.parse(System.getProperty("test.datadog.agent.image"))) .withEnv(env) .withExposedPorts(DEFAULT_TRACE_AGENT_PORT) .withStartupTimeout(Duration.ofSeconds(120)) diff --git a/dd-trace-core/src/traceAgentTest/java/datadog/trace/common/metrics/MetricsIntegrationTest.java b/dd-trace-core/src/traceAgentTest/java/datadog/trace/common/metrics/MetricsIntegrationTest.java index 607512427ae..537de40bfb7 100644 --- a/dd-trace-core/src/traceAgentTest/java/datadog/trace/common/metrics/MetricsIntegrationTest.java +++ b/dd-trace-core/src/traceAgentTest/java/datadog/trace/common/metrics/MetricsIntegrationTest.java @@ -32,6 +32,7 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.testcontainers.containers.GenericContainer; import org.testcontainers.containers.startupcheck.MinimumDurationRunningStartupCheckStrategy; +import org.testcontainers.utility.DockerImageName; @ExtendWith(WithConfigExtension.class) class MetricsIntegrationTest { @@ -55,7 +56,8 @@ static void setupSpec() { env.put("DD_HOSTNAME", "doesnotexist"); env.put("DD_LOGS_STDOUT", "yes"); agentContainer = - new GenericContainer<>("datadog/agent:7.40.1") + new GenericContainer<>( + DockerImageName.parse(System.getProperty("test.datadog.agent.image"))) .withEnv(env) .withExposedPorts(ConfigDefaults.DEFAULT_TRACE_AGENT_PORT) .withStartupTimeout(Duration.ofSeconds(120)) From 8ade5b12632e6d8a2c62df9f2076e3edb0ec5d12 Mon Sep 17 00:00:00 2001 From: Brice Dutheil Date: Thu, 24 Sep 2026 18:37:06 +0200 Subject: [PATCH 12/15] ci: restore full test coverage --- .gitlab-ci.yml | 1411 +++++++++++++++-- .../datastax-cassandra-3.0/build.gradle | 5 + .../datastax-cassandra-3.8/build.gradle | 5 + .../datastax-cassandra-4.0/build.gradle | 6 + 4 files changed, 1280 insertions(+), 147 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2f3a3c13a97..bf87d739e1d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,23 +1,145 @@ -# Temporary pipeline for the container-image fingerprinting work. -stages: [tests] +include: + - local: ".gitlab/one-pipeline.locked.yml" + - local: ".gitlab/benchmarks.yml" + - local: ".gitlab/exploration-tests.yml" + - local: ".gitlab/ci-visibility-tests.yml" + - project: 'DataDog/apm-reliability/apm-sdks-benchmarks' + file: '.gitlab/ci-java-spring-petclinic-parallel.yml' + ref: &apm_sdks_benchmarks_sha '5e416ea523f39bbdf9e44f39e6895a07d3715ff5' # pinned by .github/workflows/update-apm-sdks-benchmarks-reference.yaml + - project: 'DataDog/apm-reliability/apm-sdks-benchmarks' + file: '.gitlab/ci-java-load-parallel.yml' + ref: *apm_sdks_benchmarks_sha + - project: 'DataDog/apm-reliability/apm-sdks-benchmarks' + file: '.gitlab/ci-java-startup-parallel.yml' + ref: *apm_sdks_benchmarks_sha + - project: 'DataDog/apm-reliability/apm-sdks-benchmarks' + file: '.gitlab/ci-java-dacapo-parallel.yml' + ref: *apm_sdks_benchmarks_sha + - project: 'DataDog/apm-reliability/apm-sdks-benchmarks' + file: '.gitlab/ci-java-post-pr-comment.yml' + ref: *apm_sdks_benchmarks_sha + - local: ".gitlab/java-benchmark-configs.yml" + +stages: + - build + - publish + - java-spring-petclinic-parallel + - java-spring-petclinic-parallel-slo + - java-startup-parallel + - java-startup-parallel-slo + - java-load-parallel + - java-load-parallel-slo + - java-dacapo-parallel + - java-dacapo-parallel-slo + - java-post-pr-comment + - shared-pipeline-build + - shared-pipeline-test + - publish-release-artifacts + - shared-pipeline-publish + - benchmarks + - tests + - tests-arm64 + - test-summary + - exploration-tests + - ci-visibility-tests + - generate-signing-key variables: + APM_SDKS_BENCHMARKS_SHA: *apm_sdks_benchmarks_sha + # Test the OpenTelemetry Operator-compatible Java image jobs from one-pipeline. + CI_OTEL_OPERATOR_IMAGES_ENABLED: "true" + CI_OTEL_OPERATOR_LANGUAGE: java + # Gitlab runner features; see https://docs.gitlab.com/runner/configuration/feature-flags.html + # Fold and time all script sections + FF_SCRIPT_SECTIONS: 1 + REGISTRY: 486234852809.dkr.ecr.us-east-1.amazonaws.com + BUILD_JOB_NAME: "build" DEPENDENCY_CACHE_POLICY: pull BUILD_CACHE_POLICY: pull - GRADLE_VERSION: "9.7.1" + GRADLE_VERSION: "9.7.1" # must match gradle-wrapper.properties MASS_READ_URL: "https://mass-read.us1.ddbuild.io" MAVEN_REPOSITORY_PROXY: "https://depot-read-api-java.us1.ddbuild.io/magicmirror/magicmirror/@current/" GRADLE_PLUGIN_PROXY: "https://depot-read-api-java.us1.ddbuild.io/magicmirror/magicmirror/@current/" - BUILDER_IMAGE_REPO: "registry.ddbuild.io/images/mirror/dd-trace-java-docker-build" - BUILDER_IMAGE_VERSION_PREFIX: "ci-" + BUILDER_IMAGE_REPO: "registry.ddbuild.io/images/mirror/dd-trace-java-docker-build" # images are pinned in images/mirror.lock.yaml in the DataDog/images repo + BUILDER_IMAGE_VERSION_PREFIX: "ci-" # use either an empty string (e.g. "") for latest images or a version followed by a hyphen (e.g. "ci-" or "123_merge-") TEST_COUNTS_S3_BUCKET: "dd-trace-java-ci-test-reports" - PROFILE_TESTS: "false" - testJvm: "25" - CI_SPLIT: "1/1" + REPO_NOTIFICATION_CHANNEL: "#apm-java-escalations" + DEFAULT_TEST_JVMS: /^(8|11|17|21|25|tip)$/ # the latest "tip" version is 26 + PROFILE_TESTS: + description: "Enable profiling of tests" + value: "false" + NON_DEFAULT_JVMS: + description: "Enable tests on JVMs that are not the default" + value: "false" + RUN_FLAKY_TESTS: + description: "Enable flaky tests" + value: "false" + + # One pipeline injection package size ratchet + OCI_PACKAGE_MAX_SIZE_BYTES: 40_000_000 + LIB_INJECTION_IMAGE_MAX_SIZE_BYTES: 40_000_000 +# trigger new commit cancel workflow: auto_cancel: on_new_commit: interruptible + rules: + # skip gitlab pipeline for github merge queue - we are currently using the datadog merge queue + - if: '$CI_COMMIT_BRANCH =~ /^gh-readonly-queue\//' + when: never + - if: '$CI_COMMIT_BRANCH == "master"' + variables: + SYSTEM_TESTS_RUN_ALL_VMS: "true" + auto_cancel: + on_new_commit: none + - if: '$CI_COMMIT_BRANCH =~ /^v[0-9]+\.[0-9]+\.[0-9]+$/' + auto_cancel: + on_new_commit: none + - when: always + +.test_matrix: &test_matrix + - testJvm: &test_jvms + - "8" + - "11" + - "17" + - "21" + - "25" + - "27" # JDK 27 TODO: remove after GA (tip will move to 27) + - "semeru11" + - "oracle8" + - "zulu8" + - "semeru8" + - "ibm8" + - "zulu11" + - "semeru17" + - "tip" + CI_SPLIT: ["1/1"] + +# Gitlab doesn't support "parallel" and "parallel:matrix" at the same time +# These blocks emulate "parallel" by including it in the matrix +.test_matrix_2: &test_matrix_2 + - testJvm: *test_jvms + CI_SPLIT: ["1/2", "2/2"] + +.test_matrix_4: &test_matrix_4 + - testJvm: *test_jvms + CI_SPLIT: ["1/4", "2/4", "3/4", "4/4"] + +.test_matrix_6: &test_matrix_6 + - testJvm: *test_jvms + CI_SPLIT: ["1/6", "2/6", "3/6", "4/6", "5/6", "6/6"] + +.test_matrix_8: &test_matrix_8 + - testJvm: *test_jvms + CI_SPLIT: ["1/8", "2/8", "3/8", "4/8", "5/8", "6/8", "7/8", "8/8"] + +.test_matrix_12: &test_matrix_12 + - testJvm: *test_jvms + CI_SPLIT: [ "1/12", "2/12", "3/12", "4/12", "5/12", "6/12", "7/12", "8/12", "9/12", "10/12", "11/12", "12/12" ] + +.master_only: &master_only + - if: $CI_COMMIT_BRANCH == "master" + when: on_success default: tags: [ "arch:amd64" ] @@ -26,6 +148,7 @@ default: .set_datadog_api_keys: &set_datadog_api_keys - export DATADOG_API_KEY_PROD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.DATADOG_API_KEY_PROD --with-decryption --query "Parameter.Value" --out text) +# CI_NODE_INDEX and CI_NODE_TOTAL are 1-indexed and not always set. These steps normalize the numbers for jobs .normalize_node_index: &normalize_node_index - if [ "$CI_NO_SPLIT" == "true" ] ; then CI_NODE_INDEX=1; CI_NODE_TOTAL=1; fi # A job uses parallel but doesn't intend to split by index - if [ -n "$CI_SPLIT" ]; then CI_NODE_INDEX="${CI_SPLIT%%/*}"; CI_NODE_TOTAL="${CI_SPLIT##*/}"; fi @@ -76,6 +199,15 @@ default: KUBERNETES_MEMORY_REQUEST: 32Gi KUBERNETES_MEMORY_LIMIT: 32Gi +.gitlab_base_ref_params: &gitlab_base_ref_params + - | + export GIT_BASE_REF=$(.gitlab/find-gh-base-ref.sh) + if [[ -n "$GIT_BASE_REF" ]]; then + export GRADLE_PARAMS="$GRADLE_PARAMS -PgitBaseRef=origin/$GIT_BASE_REF" + else + echo "Failed to find base ref for PR" >&2 + fi + .gradle_build: &gradle_build image: ${BUILDER_IMAGE_REPO}:${BUILDER_IMAGE_VERSION_PREFIX}base stage: build @@ -155,6 +287,474 @@ default: - *cgroup_info - *container_info +# Check and fail early if maven central credentials are incorrect. When a new token is generated +# on the central publisher portal, it invalidates the old one. This check prevents going further. +# See https://datadoghq.atlassian.net/wiki/x/Oog5OgE +maven-central-pre-release-check: + image: ${BUILDER_IMAGE_REPO}:${BUILDER_IMAGE_VERSION_PREFIX}base + stage: .pre + rules: + - if: '$CI_COMMIT_BRANCH == "master"' + when: on_success + allow_failure: false + - if: '$CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+$/' + when: on_success + allow_failure: false + script: + - | + MAVEN_CENTRAL_USERNAME=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.central_username --with-decryption --query "Parameter.Value" --out text) + MAVEN_CENTRAL_PASSWORD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.central_password --with-decryption --query "Parameter.Value" --out text) + # See https://central.sonatype.org/publish/publish-portal-ossrh-staging-api/ + # Use the staging API search endpoint to validate the tokens without relying on a specific deployment + AUTHORIZATION_HEADER="Authorization: Bearer $(printf '%s:%s' "$MAVEN_CENTRAL_USERNAME" "$MAVEN_CENTRAL_PASSWORD" | base64)" + if ! curl --silent --show-error --fail \ + "https://ossrh-staging-api.central.sonatype.com/manual/search/repositories?ip=any" \ + --header "$AUTHORIZATION_HEADER" \ + > /dev/null; then + echo "Failed to authenticate tokens against maven central staging API. Check credentials and see https://datadoghq.atlassian.net/wiki/x/Oog5OgE" + exit 1 + fi + +dd-octo-sts-pre-release-check: + image: registry.ddbuild.io/images/dd-octo-sts-ci-base:2025.06-1 + stage: .pre + tags: [ "arch:amd64" ] + id_tokens: + DDOCTOSTS_ID_TOKEN: + aud: dd-octo-sts + rules: + - if: '$POPULATE_CACHE' + when: never + - if: '$CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+$/' + when: on_success + allow_failure: false + before_script: + - dd-octo-sts version + - dd-octo-sts debug --scope DataDog/dd-trace-java --policy self.gitlab.release + - dd-octo-sts token --scope DataDog/dd-trace-java --policy self.gitlab.release > test-github-token.txt + script: + - gh auth login --with-token < test-github-token.txt + - gh auth status + after_script: + - dd-octo-sts revoke -t $(cat test-github-token.txt) + retry: + max: 2 + when: always + +build: + needs: + - job: maven-central-pre-release-check + optional: true + - job: dd-octo-sts-pre-release-check + optional: true + extends: .gradle_build + variables: + BUILD_CACHE_POLICY: push + CACHE_TYPE: "lib" + DEPENDENCY_CACHE_POLICY: pull + script: + - if [ $CI_PIPELINE_SOURCE == "schedule" ] ; then ./gradlew resolveAndLockAll --write-locks $GRADLE_ARGS; fi + - ./gradlew --version + - ./gradlew clean :dd-java-agent:shadowJar :dd-java-agent:check :dd-trace-api:jar :dd-trace-ot:shadowJar :products:feature-flagging:feature-flagging-api:jar -PskipTests -x spotlessCheck $GRADLE_ARGS + - echo UPSTREAM_TRACER_VERSION=$(java -jar workspace/dd-java-agent/build/libs/*.jar) >> upstream.env + - echo "BUILD_JOB_NAME=$CI_JOB_NAME" >> build.env + - echo "BUILD_JOB_ID=$CI_JOB_ID" >> build.env + artifacts: + when: always + paths: + - 'workspace/dd-java-agent/build/libs/*.jar' + - 'workspace/dd-trace-api/build/libs/*.jar' + - 'workspace/dd-trace-ot/build/libs/*.jar' + - 'workspace/products/feature-flagging/feature-flagging-api/build/libs/*.jar' + - 'upstream.env' + - '.gradle/daemon/*/*.out.log' + reports: + dotenv: build.env + +build_tests: + extends: .gradle_build + variables: + <<: *tier_xl_variables + BUILD_CACHE_POLICY: push + DEPENDENCY_CACHE_POLICY: pull + parallel: + matrix: + - GRADLE_TARGET: ":baseTest" + CACHE_TYPE: "base" + - GRADLE_TARGET: ":profilingTest" + CACHE_TYPE: "profiling" + - GRADLE_TARGET: ":instrumentationTest" + CACHE_TYPE: "inst" + - GRADLE_TARGET: ":instrumentationLatestDepTest" + CACHE_TYPE: "latestdep" + - GRADLE_TARGET: ":smokeTest" + CACHE_TYPE: "smoke" + MAVEN_OPTS: "-Xms256M -Xmx1024M" + script: + - *gitlab_base_ref_params + - ./gradlew --version + - ./gradlew clean $GRADLE_TARGET $GRADLE_PARAMS -PskipTests $GRADLE_ARGS + +populate_plugin_cache: + extends: .gradle_build + cache: + - &plugin_cache + key: dependency-plugins-v1 + paths: + - .gradle-plugin-cache/caches/modules-2 + policy: pull-push + when: always + unprotect: true + rules: + - if: '$POPULATE_CACHE' + when: on_success + script: + - mkdir -p .gradle-plugin-cache + - sudo chown -R 1001:1001 .gradle-plugin-cache + - export GRADLE_USER_HOME=$(pwd)/.gradle-plugin-cache + - | + gradle_status=0 + ./gradlew help -PskipTests $GRADLE_ARGS || gradle_status=$? + if ! find .gradle-plugin-cache/caches/modules-2/files-2.1 -type f -print -quit 2>/dev/null | grep -q .; then + echo "WARNING: No plugin artifacts were cached; skipping the cache upload." + rm -rf .gradle-plugin-cache/caches/modules-2 + fi + exit "$gradle_status" + # This cache warmer is an optimization; population jobs can continue with a cold plugin cache. + allow_failure: true + +populate_dep_cache: + extends: build_tests + # Wait for build-cache producers so their caches are uploaded before this job restores them. + needs: + - job: populate_plugin_cache + - job: build + artifacts: false + - job: build_tests + artifacts: false + variables: + BUILD_CACHE_POLICY: pull + DEPENDENCY_CACHE_POLICY: push + cache: + - <<: *plugin_cache + policy: pull + - *dependency_cache + - *build_cache + rules: + - if: '$POPULATE_CACHE' + when: on_success + before_script: + - !reference [.gradle_build, before_script] + # Seed the otherwise cold writable cache, keeping the cache uploaded below self-contained. + - mkdir -p .gradle/caches + - | + if [ -d .gradle-plugin-cache/caches/modules-2 ]; then + sudo chown -R 1001:1001 .gradle-plugin-cache + cp -a .gradle-plugin-cache/caches/modules-2 .gradle/caches/ + else + echo "Plugin cache is unavailable; continuing with a cold dependency cache." + fi + parallel: + matrix: + - GRADLE_TARGET: ":dd-java-agent:shadowJar :dd-trace-api:jar :dd-trace-ot:shadowJar" + CACHE_TYPE: "lib" + - GRADLE_TARGET: ":baseTest" + CACHE_TYPE: "base" + - GRADLE_TARGET: ":profilingTest" + CACHE_TYPE: "profiling" + - GRADLE_TARGET: ":instrumentationTest" + CACHE_TYPE: "inst" + - GRADLE_TARGET: ":instrumentationLatestDepTest" + CACHE_TYPE: "latestdep" + - GRADLE_TARGET: ":smokeTest" + CACHE_TYPE: "smoke" + - GRADLE_TARGET: "spotlessCheck" + CACHE_TYPE: "spotless" + GRADLE_MEMORY_MAX: "6G" + +publish-artifacts-to-s3: + image: registry.ddbuild.io/images/mirror/amazon/aws-cli:2.4.29 + stage: publish + needs: [ build ] + script: + - source upstream.env + - export VERSION="${UPSTREAM_TRACER_VERSION%~*}" # remove ~githash from the end of version + - aws s3 cp workspace/dd-java-agent/build/libs/dd-java-agent-${VERSION}.jar s3://dd-trace-java-builds/${CI_COMMIT_REF_NAME}/dd-java-agent.jar + - aws s3 cp workspace/dd-trace-api/build/libs/dd-trace-api-${VERSION}.jar s3://dd-trace-java-builds/${CI_COMMIT_REF_NAME}/dd-trace-api.jar + - aws s3 cp workspace/dd-trace-ot/build/libs/dd-trace-ot-${VERSION}.jar s3://dd-trace-java-builds/${CI_COMMIT_REF_NAME}/dd-trace-ot.jar + - aws s3 cp workspace/products/feature-flagging/feature-flagging-api/build/libs/dd-openfeature-${VERSION}.jar s3://dd-trace-java-builds/${CI_COMMIT_REF_NAME}/dd-openfeature.jar + - aws s3 cp workspace/dd-java-agent/build/libs/dd-java-agent-${VERSION}.jar s3://dd-trace-java-builds/${CI_PIPELINE_ID}/dd-java-agent.jar + - aws s3 cp workspace/dd-trace-api/build/libs/dd-trace-api-${VERSION}.jar s3://dd-trace-java-builds/${CI_PIPELINE_ID}/dd-trace-api.jar + - aws s3 cp workspace/dd-trace-ot/build/libs/dd-trace-ot-${VERSION}.jar s3://dd-trace-java-builds/${CI_PIPELINE_ID}/dd-trace-ot.jar + - aws s3 cp workspace/products/feature-flagging/feature-flagging-api/build/libs/dd-openfeature-${VERSION}.jar s3://dd-trace-java-builds/${CI_PIPELINE_ID}/dd-openfeature.jar + - | + cat << EOF > links.json + { + "S3 Links": [ + { + "external_link": { + "label": "Public Link to dd-java-agent.jar", + "url": "https://s3.us-east-1.amazonaws.com/dd-trace-java-builds/${CI_PIPELINE_ID}/dd-java-agent.jar" + } + }, + { + "external_link": { + "label": "Public Link to dd-openfeature.jar", + "url": "https://s3.us-east-1.amazonaws.com/dd-trace-java-builds/${CI_PIPELINE_ID}/dd-openfeature.jar" + } + } + ] + } + EOF + artifacts: + reports: + annotations: + - links.json + + +spotless: + extends: .gradle_build + stage: tests + needs: [] + variables: + GRADLE_MEMORY_MAX: 6G + CACHE_TYPE: "spotless" + script: + - ./gradlew --version + # test-published-dependencies's build file needs main version file + - ./gradlew spotlessCheck writeMainVersionFile $GRADLE_ARGS + - cd test-published-dependencies && ./gradlew spotlessCheck $GRADLE_ARGS + +check-instrumentation-naming: + extends: .gradle_build + stage: tests + needs: [ ] + script: + - ./gradlew --version + - ./gradlew checkInstrumentationNaming + +config-inversion-linter: + extends: .gradle_build + stage: tests + needs: [] + script: + - ./gradlew --version + - ./gradlew checkConfigurations + +test_published_artifacts: + extends: .gradle_build + image: ${BUILDER_IMAGE_REPO}:${BUILDER_IMAGE_VERSION_PREFIX}7 # Needs Java7 for some tests + stage: tests + needs: [ build ] + variables: + CACHE_TYPE: "lib" + script: + - mvn_local_repo=$(./mvnw help:evaluate -Dexpression=settings.localRepository -q -DforceStdout) + - rm -rf "${mvn_local_repo}/com/datadoghq" + - export GPG_PRIVATE_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.signing.gpg_private_key --with-decryption --query "Parameter.Value" --out text) + - export GPG_PASSWORD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.signing.gpg_passphrase --with-decryption --query "Parameter.Value" --out text) + - export GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xms2G -Xmx2G -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp'" + - ./gradlew publishToMavenLocal $GRADLE_ARGS + - cd test-published-dependencies + - printf '\norg.gradle.console=colored\n' >> gradle.properties + - export GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xms1G -Xmx1G -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp'" + - ./gradlew --version + - ./gradlew check --info $GRADLE_ARGS + after_script: + - *cgroup_info + - source .gitlab/gitlab-utils.sh + - gitlab_section_start "collect-reports" "Collecting reports" + - .gitlab/collect_reports.sh + - gitlab_section_end "collect-reports" + artifacts: + when: always + paths: + - ./check_reports + +validate_build: + extends: .gradle_build + stage: tests + needs: [ build ] + variables: + CACHE_TYPE: "lib" + script: + # Preserve the `build` job artifacts before Gradle rebuilds and overwrites them under + # workspace/**/build/libs, so jardiff can compare the rebuilt jars against them. + - mkdir -p reference-artifacts + - cp workspace/dd-java-agent/build/libs/*.jar reference-artifacts/ + - cp workspace/dd-trace-api/build/libs/*.jar reference-artifacts/ + - cp workspace/dd-trace-ot/build/libs/*.jar reference-artifacts/ + # Scheduled builds refresh dependency locks before creating the reference artifacts. + # Refresh them here as well so the candidate uses the same dependency resolution. + - if [ $CI_PIPELINE_SOURCE == "schedule" ] ; then ./gradlew resolveAndLockAll --write-locks $GRADLE_ARGS; fi + - ./gradlew --version + # This will run the shadowJar task and exercise the build cache, allowing to identify build-cache issues + # Keep both JAR sets for direct inspection when the comparison fails. + - | + if ! ./gradlew compareToReferenceJar -PjardiffReferenceDir="$CI_PROJECT_DIR/reference-artifacts" -PskipTests $GRADLE_ARGS; then + mkdir -p check_reports/jar-comparison-artifacts/reference check_reports/jar-comparison-artifacts/candidate + cp reference-artifacts/*.jar check_reports/jar-comparison-artifacts/reference/ + cp workspace/dd-java-agent/build/libs/*.jar check_reports/jar-comparison-artifacts/candidate/ + cp workspace/dd-trace-api/build/libs/*.jar check_reports/jar-comparison-artifacts/candidate/ + cp workspace/dd-trace-ot/build/libs/*.jar check_reports/jar-comparison-artifacts/candidate/ + exit 1 + fi + after_script: + - source .gitlab/gitlab-utils.sh + - gitlab_section_start "collect-reports" "Collecting reports" + - .gitlab/collect_reports.sh --destination ./check_reports + - gitlab_section_end "collect-reports" + artifacts: + when: always + paths: + - ./check_reports + - '.gradle/daemon/*/*.out.log' + +.check_job: + extends: .gradle_build + needs: [ build ] + stage: tests + variables: + CACHE_TYPE: "lib" + script: + - *gitlab_base_ref_params + - ./gradlew --version + - ./gradlew $GRADLE_TARGET -x spotlessCheck $GRADLE_PARAMS -PskipTests -PrunBuildSrcTests -Pslot=$CI_NODE_INDEX/$CI_NODE_TOTAL $GRADLE_ARGS + after_script: + - *container_info + - *cgroup_info + - source .gitlab/gitlab-utils.sh + - gitlab_section_start "collect-reports" "Collecting reports" + - .gitlab/collect_reports.sh --destination ./check_reports --move + - .gitlab/collect_results.sh + - gitlab_section_end "collect-reports" + artifacts: + when: always + paths: + - ./check_reports + - ./results + - '.gradle/daemon/*/*.out.log' + reports: + junit: results/*.xml + retry: + max: 2 + when: + - unknown_failure + - stuck_or_timeout_failure + - runner_system_failure + - unmet_prerequisites + - scheduler_failure + - data_integrity_failure + +check_build_src: + extends: .check_job + needs: [] + variables: + GRADLE_TARGET: ":buildSrc:build" + +check_base: + extends: .check_job + variables: + GRADLE_TARGET: ":baseCheck" + +check_inst: + extends: .check_job + parallel: 4 + variables: + GRADLE_TARGET: ":instrumentationCheck" + CACHE_TYPE: "inst" + +check_smoke: + extends: .check_job + parallel: 4 + variables: + GRADLE_TARGET: ":smokeCheck" + CACHE_TYPE: "smoke" + +check_profiling: + extends: .check_job + variables: + GRADLE_TARGET: ":profilingCheck" + +check_debugger: + extends: .check_job + variables: + GRADLE_TARGET: ":debuggerCheck" + +muzzle: + extends: .gradle_build + # needs:parallel:matrix limits this job to a specific build_tests combination. + # Keep matrix vars exact and in build_tests declaration order: + # https://docs.gitlab.com/ci/yaml/#needsparallelmatrix + needs: &needs_build_tests_inst + - job: build_tests + parallel: + matrix: + - GRADLE_TARGET: ":instrumentationTest" + CACHE_TYPE: "inst" + stage: tests + rules: + - if: '$CI_COMMIT_BRANCH =~ /^mq-working-branch-/' + when: never + - if: '$CI_COMMIT_BRANCH =~ /^gh-readonly-queue/' + when: never + - when: on_success + parallel: + matrix: + - CI_SPLIT: ["1/8", "2/8", "3/8", "4/8", "5/8", "6/8", "7/8", "8/8"] + variables: + CACHE_TYPE: "inst" + script: + - export SKIP_BUILDSCAN="true" + - ./gradlew --version + - ./gradlew :runMuzzle -Pslot=$CI_NODE_INDEX/$CI_NODE_TOTAL $GRADLE_ARGS + after_script: + - *container_info + - *cgroup_info + - *set_datadog_api_keys + - source .gitlab/gitlab-utils.sh + - gitlab_section_start "collect-reports" "Collecting reports" + - .gitlab/collect_reports.sh + - .gitlab/collect_results.sh + - .gitlab/upload_ciapp.sh $CACHE_TYPE + - gitlab_section_end "collect-reports" + artifacts: + when: always + paths: + - ./reports + - ./results + - '.gradle/daemon/*/*.out.log' + reports: + junit: results/*.xml + +muzzle-dep-report: + extends: .gradle_build + needs: *needs_build_tests_inst + stage: tests + rules: + - if: '$CI_COMMIT_BRANCH =~ /^mq-working-branch-/' + when: never + - if: '$CI_COMMIT_BRANCH =~ /^gh-readonly-queue/' + when: never + - when: on_success + variables: + CACHE_TYPE: "inst" + script: + - export SKIP_BUILDSCAN="true" + - ./gradlew --version + - ./gradlew generateMuzzleReport muzzleInstrumentationReport $GRADLE_ARGS + after_script: + - *container_info + - *cgroup_info + - .gitlab/collect_muzzle_deps.sh + artifacts: + when: always + paths: + - ./reports + - '.gradle/daemon/*/*.out.log' + +# In Gitlab, DD_* variables are set because the build runner is instrumented with Datadog telemetry +# To have a pristine environment for the tests, these variables are saved before the test run and restored afterwards .prepare_test_env: &prepare_test_env - export gitlabVariables=("DD_SERVICE" "DD_ENTITY_ID" "DD_SITE" "DD_ENV" "DD_DATACENTER" "DD_PARTITION" "DD_CLOUDPROVIDER") - '[ ! -e pretest.env ] || rm pretest.env' @@ -181,6 +781,7 @@ default: TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX: "registry.ddbuild.io/images/mirror/" JETTY_AVAILABLE_PROCESSORS: 4 # Jetty incorrectly calculates processor count in containers script: + - *gitlab_base_ref_params - > if [ "$PROFILE_TESTS" == "true" ] && [ "$testJvm" != "ibm8" ] && [ "$testJvm" != "oracle8" ]; then @@ -232,6 +833,82 @@ default: - scheduler_failure - data_integrity_failure +.test_job_amd64: + extends: .test_job_common + tags: [ "docker-in-docker:amd64" ] # use docker-in-docker runner for testcontainers + needs: [ build_tests ] + stage: tests + rules: + # Protected branches (master/mq/gh-readonly): all JVMs run unconditionally + - if: '$CI_COMMIT_BRANCH == "master"' + when: on_success + - if: '$CI_COMMIT_BRANCH =~ /^mq-working-branch-/' + when: on_success + - if: '$CI_COMMIT_BRANCH =~ /^gh-readonly-queue/' + when: on_success + # Enable for default test JVMs or for NON_DEFAULT_JVMS + - if: '$NON_DEFAULT_JVMS == "true"' + when: on_success + - if: '$CI_COMMIT_MESSAGE =~ /\[ci: NON_DEFAULT_JVMS\]/' + when: on_success + - if: '$testJvm =~ $DEFAULT_TEST_JVMS' + when: on_success + +.test_job_arm64: + extends: .test_job_common + tags: [ "docker-in-docker:arm64" ] + stage: tests-arm64 + # Use the amd64 build only as a compilation gate. Do not download its platform-specific artifacts. + needs: + - job: build + artifacts: false + variables: + DEFAULT_TEST_JVMS: /^(8|11|17|21|25|27|tip)$/ # Java 27 TODO: remove 27 after GA (tip will move to 27) + TEST_JVM_ARGS: "-Xshare:off" + rules: + # IBM 8 has no arm64 image published upstream. + - if: '$testJvm == "ibm8"' + when: never + # Oracle 8 is available on arm64 but too flaky to run in CI. + - if: '$testJvm == "oracle8"' + when: never + # arm64 tests are newly introduced to the merge queue and master. Keep them + # non-blocking (allow_failure) for now so we can collect stability stats and + # fix flaky/failing jobs without blocking the whole team. Remove allow_failure + # once the arm64 suite is proven stable. + - if: '$CI_COMMIT_BRANCH =~ /^mq-working-branch-/' + when: on_success + allow_failure: true + - if: '$CI_COMMIT_BRANCH == "master"' + when: on_success + allow_failure: true + # Enable non-default JVMs on demand. + - if: '$NON_DEFAULT_JVMS == "true"' + when: on_success + allow_failure: true + - if: '$CI_COMMIT_MESSAGE =~ /\[ci: NON_DEFAULT_JVMS\]/' + when: on_success + allow_failure: true + # Keep the default JVM subset available for manual runs on feature branches/PRs. + - if: '$testJvm =~ $DEFAULT_TEST_JVMS' + when: manual + allow_failure: true + cache: + - key: dependency-$CACHE_TYPE + paths: + - .gradle/wrapper + - .gradle/caches + - .gradle/notifications + - .mvn/caches + policy: pull + fallback_keys: + - dependency-base + - dependency-lib + unprotect: true + before_script: + - git config --global --add safe.directory "$CI_PROJECT_DIR" + - !reference [.gradle_build, before_script] + .test_job_with_test_agent_common: variables: CI_USE_TEST_AGENT: "true" @@ -250,150 +927,590 @@ default: - !reference [.test_job_common, script] - .gitlab/check_test_agent_results.sh -test_image_plugin: - extends: .gradle_build - stage: tests - script: - - JAVA_HOME=$JAVA_25_HOME ./gradlew -p build-logic :testcontainers:test :testcontainers:validatePlugins $GRADLE_ARGS - artifacts: - when: always - paths: - - build-logic/testcontainers/build/reports/tests/ - reports: - junit: build-logic/testcontainers/build/test-results/test/*.xml +.test_job_amd64_with_test_agent: + extends: + - .test_job_amd64 + - .test_job_with_test_agent_common -test_container_images: +.test_job_arm64_with_test_agent: extends: - - .test_job_common + - .test_job_arm64 - .test_job_with_test_agent_common - stage: tests - tags: ["docker-in-docker:amd64"] + +agent_integration_tests: + extends: .test_job_amd64 + tags: [ "arch:amd64" ] variables: + testJvm: "8" + CI_AGENT_HOST: local-agent + GRADLE_TARGET: "traceAgentTest" + CACHE_TYPE: "base" + services: + - name: registry.ddbuild.io/images/mirror/datadog/agent:7.40.1 + alias: local-agent + variables: + DD_APM_ENABLED: "true" + DD_BIND_HOST: "0.0.0.0" + DD_HOSTNAME: "local-agent" + DD_API_KEY: "invalid_key_but_this_is_fine" + +test_base: + extends: .test_job_amd64 + # needs:parallel:matrix limits this job to a specific build_tests combination. + # Keep matrix vars exact and in build_tests declaration order: + # https://docs.gitlab.com/ci/yaml/#needsparallelmatrix + needs: + - job: build_tests + parallel: + matrix: + - GRADLE_TARGET: ":baseTest" + CACHE_TYPE: "base" + variables: + GRADLE_TARGET: ":baseTest" + CACHE_TYPE: "base" + parallel: + matrix: *test_matrix_4 + script: + - if [ "$testJvm" == "8" ]; then export GRADLE_PARAMS="-PskipFlakyTests -PcheckCoverage"; fi + - !reference [.test_job_common, script] + +test_base_arm64: + extends: .test_job_arm64 + variables: + GRADLE_TARGET: ":baseTest" + CACHE_TYPE: "base" + parallel: + matrix: *test_matrix_4 + # run coverage only on JVM 8, mirroring the amd64 test_base job + script: + - if [ "$testJvm" == "8" ]; then export GRADLE_PARAMS="$GRADLE_PARAMS -PcheckCoverage"; fi + - !reference [.test_job_common, script] + +test_inst: + extends: .test_job_amd64_with_test_agent + needs: *needs_build_tests_inst + variables: + <<: *tier_l_variables + GRADLE_TARGET: ":instrumentationTest" CACHE_TYPE: "inst" + parallel: + matrix: *test_matrix_8 + +test_inst_arm64: + extends: .test_job_arm64_with_test_agent + variables: + <<: *tier_l_variables + GRADLE_TARGET: ":instrumentationTest" + CACHE_TYPE: "inst" + parallel: + matrix: *test_matrix_8 + +test_inst_latest: + extends: .test_job_amd64_with_test_agent + # needs:parallel:matrix limits this job to a specific build_tests combination. + # Keep matrix vars exact and in build_tests declaration order: + # https://docs.gitlab.com/ci/yaml/#needsparallelmatrix + needs: + - job: build_tests + parallel: + matrix: + - GRADLE_TARGET: ":instrumentationLatestDepTest" + CACHE_TYPE: "latestdep" + variables: + <<: *tier_l_variables + GRADLE_TARGET: ":instrumentationLatestDepTest" + CACHE_TYPE: "latestdep" + parallel: + matrix: + - testJvm: ["8", "17", "21", "25", "27", "tip"] # Java 27 TODO: remove 27 after GA (tip will move to 27) + # Gitlab doesn't support "parallel" and "parallel:matrix" at the same time + # This emulates "parallel" by including it in the matrix + CI_SPLIT: [ "1/6", "2/6", "3/6", "4/6", "5/6", "6/6"] + +test_inst_latest_arm64: + extends: .test_job_arm64_with_test_agent + variables: + <<: *tier_l_variables + GRADLE_TARGET: ":instrumentationLatestDepTest" + CACHE_TYPE: "latestdep" + parallel: + matrix: + - testJvm: ["8", "17", "21", "25", "27", "tip"] # Java 27 TODO: remove 27 after GA (tip will move to 27) + # Gitlab doesn't support "parallel" and "parallel:matrix" at the same time + # This emulates "parallel" by including it in the matrix + CI_SPLIT: [ "1/6", "2/6", "3/6", "4/6", "5/6", "6/6"] + +test_flaky: + extends: .test_job_amd64_with_test_agent + variables: + GRADLE_PARAMS: "-PrunFlakyTests" + CACHE_TYPE: "smoke" + testJvm: "8" + CONTINUE_ON_FAILURE: "true" + rules: + - *master_only + - if: $RUN_FLAKY_TESTS == "true" + when: on_success + parallel: + matrix: + - GRADLE_TARGET: [":baseTest", ":smokeTest", ":debuggerTest"] + # Gitlab doesn't support "parallel" and "parallel:matrix" at the same time + # This emulates "parallel" by including it in the matrix + CI_SPLIT: [ "1/4", "2/4", "3/4", "4/4" ] + +test_flaky_inst: + extends: .test_job_amd64 + needs: *needs_build_tests_inst + variables: + GRADLE_TARGET: ":instrumentationTest" + GRADLE_PARAMS: "-PrunFlakyTests" + CACHE_TYPE: "inst" + testJvm: "8" + CONTINUE_ON_FAILURE: "true" + rules: + - *master_only + - if: $RUN_FLAKY_TESTS == "true" + when: on_success + parallel: 6 + +test_profiling: + extends: .test_job_amd64 + # needs:parallel:matrix limits this job to a specific build_tests combination. + # Keep matrix vars exact and in build_tests declaration order: + # https://docs.gitlab.com/ci/yaml/#needsparallelmatrix + needs: + - job: build_tests + parallel: + matrix: + - GRADLE_TARGET: ":profilingTest" + CACHE_TYPE: "profiling" + variables: + GRADLE_TARGET: ":profilingTest" + CACHE_TYPE: "profiling" + parallel: + matrix: *test_matrix + +test_profiling_arm64: + extends: .test_job_arm64 + variables: + GRADLE_TARGET: ":profilingTest" + CACHE_TYPE: "profiling" + parallel: + matrix: *test_matrix + +# specific jvms list for debugger project because J9-based JVMs have issues with local vars +# so need to test at least against one J9-based JVM +test_debugger: + extends: .test_job_amd64 + variables: + GRADLE_TARGET: ":debuggerTest" + CACHE_TYPE: "base" + DEFAULT_TEST_JVMS: /^(8|11|17|21|25|27|tip|semeru8)$/ # Java 27 TODO: remove 27 after GA (tip will move to 27) + parallel: + matrix: *test_matrix + # avoid running coverage for semeru8, semeru11, semeru17 and ibm8 as some tests are disabled and therefore cannot reach the + # exepected coverage + script: + - if [[ "$testJvm" != "semeru8" && "$testJvm" != "semeru11" && "$testJvm" != "semeru17" && "$testJvm" != "ibm8" ]]; then export GRADLE_PARAMS="$GRADLE_PARAMS -PcheckCoverage"; fi + - !reference [.test_job_common, script] + +test_debugger_arm64: + extends: .test_job_arm64 + variables: + GRADLE_TARGET: ":debuggerTest" + CACHE_TYPE: "base" + parallel: + matrix: *test_matrix + # avoid running coverage for semeru8, semeru11 and semeru17 as some tests are disabled and therefore cannot reach the + # expected coverage (ibm8/oracle8 never run on arm64) + script: + - if [[ "$testJvm" != "semeru8" && "$testJvm" != "semeru11" && "$testJvm" != "semeru17" ]]; then export GRADLE_PARAMS="$GRADLE_PARAMS -PcheckCoverage"; fi + - !reference [.test_job_common, script] + +test_smoke: + extends: .test_job_amd64_with_test_agent + # needs:parallel:matrix limits this job to a specific build_tests combination. + # Keep matrix vars exact and in build_tests declaration order: + # https://docs.gitlab.com/ci/yaml/#needsparallelmatrix + needs: &needs_build_tests_smoke + - job: build_tests + parallel: + matrix: + - GRADLE_TARGET: ":smokeTest" + CACHE_TYPE: "smoke" + MAVEN_OPTS: "-Xms256M -Xmx1024M" + variables: + <<: *tier_l_variables + GRADLE_TARGET: "stageMainDist :smokeTest" + GRADLE_PARAMS: "-PskipFlakyTests" + CACHE_TYPE: "smoke" + parallel: + matrix: *test_matrix_8 + +test_smoke_arm64: + extends: .test_job_arm64_with_test_agent + variables: + <<: *tier_l_variables + GRADLE_TARGET: "stageMainDist :smokeTest" + GRADLE_PARAMS: "-PskipFlakyTests" + CACHE_TYPE: "smoke" + parallel: + matrix: *test_matrix_8 + +test_ssi_smoke: + extends: .test_job_amd64 + needs: *needs_build_tests_smoke + rules: + - if: $CI_COMMIT_BRANCH == "master" + when: on_success + - if: '$CI_COMMIT_BRANCH =~ /^mq-working-branch-/' + when: on_success + - if: '$CI_COMMIT_BRANCH =~ /^gh-readonly-queue/' + when: on_success + variables: + <<: *tier_l_variables + GRADLE_TARGET: "stageMainDist :smokeTest" + CACHE_TYPE: "smoke" + DD_INJECT_FORCE: "true" + DD_INJECTION_ENABLED: "tracer" + parallel: + matrix: *test_matrix_8 + +test_ssi_smoke_arm64: + extends: .test_job_arm64 + variables: + <<: *tier_l_variables + GRADLE_TARGET: "stageMainDist :smokeTest" + CACHE_TYPE: "smoke" + DD_INJECT_FORCE: "true" + DD_INJECTION_ENABLED: "tracer" + parallel: + matrix: *test_matrix_8 + +test_smoke_graalvm: + extends: .test_job_amd64 + needs: *needs_build_tests_smoke + tags: [ "arch:amd64" ] + variables: + <<: *tier_l_variables + GRADLE_TARGET: "stageMainDist :dd-smoke-test:spring-boot-3.0-native:test :dd-smoke-test:quarkus-native:test" + CACHE_TYPE: "smoke" + CI_NO_SPLIT: "true" + NON_DEFAULT_JVMS: "true" + parallel: + matrix: + - testJvm: ["graalvm17", "graalvm21", "graalvm25"] + +test_smoke_graalvm_arm64: + extends: .test_job_arm64 + tags: [ "arch:arm64" ] + variables: + <<: *tier_l_variables + GRADLE_TARGET: "stageMainDist :dd-smoke-test:spring-boot-3.0-native:test :dd-smoke-test:quarkus-native:test" + CACHE_TYPE: "smoke" CI_NO_SPLIT: "true" + NON_DEFAULT_JVMS: "true" parallel: matrix: - - GRADLE_TARGET: - - ":dd-java-agent:instrumentation:datastax-cassandra:datastax-cassandra-3.0:forkedTest" - - ":dd-java-agent:instrumentation:datastax-cassandra:datastax-cassandra-3.0:latestDepTest" - - ":dd-java-agent:instrumentation:datastax-cassandra:datastax-cassandra-3.0:test" - - ":dd-java-agent:instrumentation:datastax-cassandra:datastax-cassandra-3.8:forkedTest" - - ":dd-java-agent:instrumentation:datastax-cassandra:datastax-cassandra-3.8:latestDepTest" - - ":dd-java-agent:instrumentation:datastax-cassandra:datastax-cassandra-3.8:test" - - ":dd-java-agent:instrumentation:datastax-cassandra:datastax-cassandra-4.0:forkedTest" - - ":dd-java-agent:instrumentation:datastax-cassandra:datastax-cassandra-4.0:latestDepTest" - - ":dd-java-agent:instrumentation:datastax-cassandra:datastax-cassandra-4.0:test" - - ":dd-java-agent:instrumentation:google-pubsub-1.116:forkedTest" - - ":dd-java-agent:instrumentation:google-pubsub-1.116:latestDepForkedTest" - - ":dd-java-agent:instrumentation:google-pubsub-1.116:latestDepTest" - - ":dd-java-agent:instrumentation:google-pubsub-1.116:test" - - ":dd-java-agent:instrumentation:jdbc:forkedTest" - - ":dd-java-agent:instrumentation:jdbc:latestDepJava11Test" - - ":dd-java-agent:instrumentation:jdbc:latestDepTest" - - ":dd-java-agent:instrumentation:jdbc:oldH2Test" - - ":dd-java-agent:instrumentation:jdbc:oldPostgresTest" - - ":dd-java-agent:instrumentation:jdbc:test" - - ":dd-java-agent:instrumentation:vertx:vertx-mysql-client:vertx-mysql-client-3.9:forkedTest" - - ":dd-java-agent:instrumentation:vertx:vertx-mysql-client:vertx-mysql-client-3.9:latestDepForkedTest" - - ":dd-java-agent:instrumentation:vertx:vertx-mysql-client:vertx-mysql-client-4.0:forkedTest" - - ":dd-java-agent:instrumentation:vertx:vertx-mysql-client:vertx-mysql-client-4.0:latestDepForkedTest" - - ":dd-java-agent:instrumentation:vertx:vertx-pg-client:vertx-pg-client-4.0:forkedTest" - - ":dd-java-agent:instrumentation:vertx:vertx-pg-client:vertx-pg-client-4.0:latestDepForkedTest" - - ":dd-java-agent:instrumentation:aerospike-4.0:forkedTest" - - ":dd-java-agent:instrumentation:aerospike-4.0:latest7DepForkedTest" - - ":dd-java-agent:instrumentation:aerospike-4.0:latestDepForkedTest" - - ":dd-java-agent:instrumentation:aws-java:aws-java-dynamodb-2.0:latestDepTest" - - ":dd-java-agent:instrumentation:aws-java:aws-java-dynamodb-2.0:test" - - ":dd-java-agent:instrumentation:aws-java:aws-java-eventbridge-2.0:latestDepTest" - - ":dd-java-agent:instrumentation:aws-java:aws-java-eventbridge-2.0:test" - - ":dd-java-agent:instrumentation:aws-java:aws-java-s3-2.0:latestDepTest" - - ":dd-java-agent:instrumentation:aws-java:aws-java-s3-2.0:test" - - ":dd-java-agent:instrumentation:aws-java:aws-java-sdk-2.2:latestPayloadTaggingForkedTest" - - ":dd-java-agent:instrumentation:aws-java:aws-java-sdk-2.2:payloadTaggingTestForkedTest" - - ":dd-java-agent:instrumentation:aws-java:aws-java-sfn-2.0:latestDepTest" - - ":dd-java-agent:instrumentation:aws-java:aws-java-sfn-2.0:test" - - ":dd-java-agent:instrumentation:aws-java:aws-java-sns-1.0:forkedTest" - - ":dd-java-agent:instrumentation:aws-java:aws-java-sns-1.0:latestDepForkedTest" - - ":dd-java-agent:instrumentation:aws-java:aws-java-sns-1.0:latestDepTest" - - ":dd-java-agent:instrumentation:aws-java:aws-java-sns-1.0:test" - - ":dd-java-agent:instrumentation:aws-java:aws-java-sns-2.0:forkedTest" - - ":dd-java-agent:instrumentation:aws-java:aws-java-sns-2.0:latestDepForkedTest" - - ":dd-java-agent:instrumentation:aws-java:aws-java-sns-2.0:latestDepTest" - - ":dd-java-agent:instrumentation:aws-java:aws-java-sns-2.0:test" - - ":dd-java-agent:instrumentation:lettuce:lettuce-5.0:forkedTest" - - ":dd-java-agent:instrumentation:lettuce:lettuce-5.0:latestDepForkedTest" - - ":dd-java-agent:instrumentation:lettuce:lettuce-5.0:latestDepTest" - - ":dd-java-agent:instrumentation:lettuce:lettuce-5.0:lettuce51Test" - - ":dd-java-agent:instrumentation:lettuce:lettuce-5.0:lettuce60Test" - - ":dd-java-agent:instrumentation:lettuce:lettuce-5.0:lettuce61Test" - - ":dd-java-agent:instrumentation:lettuce:lettuce-5.0:lettuce62Test" - - ":dd-java-agent:instrumentation:lettuce:lettuce-5.0:test" - - ":dd-java-agent:instrumentation:spymemcached-2.10:forkedTest" - - ":dd-java-agent:instrumentation:spymemcached-2.10:latestDepForkedTest" - - ":dd-java-agent:instrumentation:spymemcached-2.10:latestDepTest" - - ":dd-java-agent:instrumentation:spymemcached-2.10:test" - - ":dd-java-agent:instrumentation:couchbase:couchbase-3.1:forkedTest" - - ":dd-java-agent:instrumentation:couchbase:couchbase-3.1:latestDepTest" - - ":dd-java-agent:instrumentation:couchbase:couchbase-3.1:test" - - ":dd-java-agent:instrumentation:couchbase:couchbase-3.2:forkedTest" - - ":dd-java-agent:instrumentation:couchbase:couchbase-3.2:latestDepTest" - - ":dd-java-agent:instrumentation:couchbase:couchbase-3.2:test" - - ":dd-java-agent:instrumentation:mongo:mongo-driver:mongo-driver-3:mongo-driver-3.1:forkedTest" - - ":dd-java-agent:instrumentation:mongo:mongo-driver:mongo-driver-3:mongo-driver-3.1:latestDepTest" - - ":dd-java-agent:instrumentation:mongo:mongo-driver:mongo-driver-3:mongo-driver-3.1:test" - - ":dd-java-agent:instrumentation:mongo:mongo-driver:mongo-driver-3:mongo-driver-3.4:forkedTest" - - ":dd-java-agent:instrumentation:mongo:mongo-driver:mongo-driver-3:mongo-driver-3.4:latestDepTest" - - ":dd-java-agent:instrumentation:mongo:mongo-driver:mongo-driver-3:mongo-driver-3.4:test" - - ":dd-java-agent:instrumentation:mongo:mongo-driver:mongo-driver-3:mongo-driver-3.6:forkedTest" - - ":dd-java-agent:instrumentation:mongo:mongo-driver:mongo-driver-3:mongo-driver-3.6:latestDepTest" - - ":dd-java-agent:instrumentation:mongo:mongo-driver:mongo-driver-3:mongo-driver-3.6:test" - - ":dd-java-agent:instrumentation:mongo:mongo-driver:mongo-driver-3:mongo-driver-3.8:forkedTest" - - ":dd-java-agent:instrumentation:mongo:mongo-driver:mongo-driver-3:mongo-driver-3.8:latestDepTest" - - ":dd-java-agent:instrumentation:mongo:mongo-driver:mongo-driver-3:mongo-driver-3.8:test" - - ":dd-java-agent:instrumentation:mongo:mongo-driver:mongo-driver-4.0:forkedTest" - - ":dd-java-agent:instrumentation:mongo:mongo-driver:mongo-driver-4.0:latestDepForkedTest" - - ":dd-java-agent:instrumentation:mongo:mongo-driver:mongo-driver-4.0:latestDepTest" - - ":dd-java-agent:instrumentation:mongo:mongo-driver:mongo-driver-4.0:mongo410ForkedTest" - - ":dd-java-agent:instrumentation:mongo:mongo-driver:mongo-driver-4.0:mongo410Test" - - ":dd-java-agent:instrumentation:mongo:mongo-driver:mongo-driver-4.0:mongo43ForkedTest" - - ":dd-java-agent:instrumentation:mongo:mongo-driver:mongo-driver-4.0:mongo43Test" - - ":dd-java-agent:instrumentation:mongo:mongo-driver:mongo-driver-4.0:test" - - ":dd-java-agent:instrumentation:mongo:mongo-test:mongo-test-async-3.3:forkedTest" - - ":dd-java-agent:instrumentation:mongo:mongo-test:mongo-test-async-3.3:latestDepTest" - - ":dd-java-agent:instrumentation:mongo:mongo-test:mongo-test-async-3.3:test" - - ":dd-java-agent:instrumentation:mongo:mongo-test:mongo-test-core-3.1:forkedTest" - - ":dd-java-agent:instrumentation:mongo:mongo-test:mongo-test-core-3.1:latestDepTest" - - ":dd-java-agent:instrumentation:mongo:mongo-test:mongo-test-core-3.1:test" - - ":dd-java-agent:instrumentation:mongo:mongo-test:mongo-test-core-3.7:forkedTest" - - ":dd-java-agent:instrumentation:mongo:mongo-test:mongo-test-core-3.7:latestDepTest" - - ":dd-java-agent:instrumentation:mongo:mongo-test:mongo-test-core-3.7:test" - - ":dd-java-agent:instrumentation:mongo:mongo-test:mongo-test-sync-3.10:forkedTest" - - ":dd-java-agent:instrumentation:mongo:mongo-test:mongo-test-sync-3.10:latestDepTest" - - ":dd-java-agent:instrumentation:mongo:mongo-test:mongo-test-sync-3.10:test" - - ":dd-java-agent:instrumentation:rabbitmq-amqp-2.7:forkedTest" - - ":dd-java-agent:instrumentation:rabbitmq-amqp-2.7:latestDepTest" - - ":dd-java-agent:instrumentation:rabbitmq-amqp-2.7:latestReactorTest" - - ":dd-java-agent:instrumentation:rabbitmq-amqp-2.7:reactorTest" - - ":dd-java-agent:instrumentation:rabbitmq-amqp-2.7:test" - - ":dd-java-agent:instrumentation:redisson:redisson-2.0.0:forkedTest" - - ":dd-java-agent:instrumentation:redisson:redisson-2.0.0:latestDepForkedTest" - - ":dd-java-agent:instrumentation:redisson:redisson-2.0.0:latestDepTest" - - ":dd-java-agent:instrumentation:redisson:redisson-2.0.0:test" - - ":dd-java-agent:instrumentation:redisson:redisson-2.3.0:forkedTest" - - ":dd-java-agent:instrumentation:redisson:redisson-2.3.0:latestDepForkedTest" - - ":dd-java-agent:instrumentation:redisson:redisson-2.3.0:latestDepTest" - - ":dd-java-agent:instrumentation:redisson:redisson-2.3.0:test" - - ":dd-java-agent:instrumentation:redisson:redisson-3.10.3:forkedTest" - - ":dd-java-agent:instrumentation:redisson:redisson-3.10.3:latestDepForkedTest" - - ":dd-java-agent:instrumentation:redisson:redisson-3.10.3:latestDepTest" - - ":dd-java-agent:instrumentation:redisson:redisson-3.10.3:test" - - ":dd-java-agent:instrumentation:spring:spring-rabbit-1.5:latestDepTest" - - ":dd-java-agent:instrumentation:spring:spring-rabbit-1.5:test" - - ":dd-java-agent:instrumentation:vertx:vertx-redis-client:vertx-redis-client-3.9:forkedTest" - - ":dd-java-agent:instrumentation:vertx:vertx-redis-client:vertx-redis-client-3.9:latestDepForkedTest" - - ":dd-java-agent:instrumentation:vertx:vertx-redis-client:vertx-redis-client-3.9:redis4xForkedTest" - - ":dd-trace-core:test" - - ":dd-trace-core:traceAgentTest" - - ":dd-smoke-tests:spring-boot-rabbit:test" - - ":dd-smoke-tests:springboot-mongo:test" - - ":dd-smoke-tests:websphere-jmx:test" + - testJvm: ["graalvm17", "graalvm21", "graalvm25"] + +test_smoke_semeru8_debugger: + extends: .test_job_amd64 + needs: *needs_build_tests_smoke + tags: [ "arch:amd64" ] + variables: + GRADLE_TARGET: "stageMainDist dd-smoke-tests:debugger-integration-tests:test" + CACHE_TYPE: "smoke" + NON_DEFAULT_JVMS: "true" + testJvm: "semeru8" + +aggregate_test_counts: + image: ${BUILDER_IMAGE_REPO}:${BUILDER_IMAGE_VERSION_PREFIX}base + stage: test-summary + # Keep stage ordering, but prevent GitLab from downloading all previous-stage artifacts. + dependencies: [] + rules: + - if: '$POPULATE_CACHE' + when: never + - if: '$CI_COMMIT_BRANCH == "master"' + when: on_success + - if: '$CI_COMMIT_BRANCH =~ /^mq-working-branch-/' + when: on_success + - if: '$CI_COMMIT_BRANCH =~ /^gh-readonly-queue/' + when: on_success + - if: '$CI_COMMIT_BRANCH' + when: on_success + script: + - *set_datadog_api_keys + - export TEST_COUNTS_S3_PREFIX="test-counts/${CI_PIPELINE_ID}" + - mkdir -p ./test_counts_aggregate + - echo "Downloading test count files from s3://${TEST_COUNTS_S3_BUCKET}/${TEST_COUNTS_S3_PREFIX}/" + - aws s3 cp "s3://${TEST_COUNTS_S3_BUCKET}/${TEST_COUNTS_S3_PREFIX}/" ./test_counts_aggregate/ --recursive --exclude "*" --include "test_counts_*.json" --only-show-errors + - find ./test_counts_aggregate -name 'test_counts_*.json' -type f -print | sort + - .gitlab/aggregate_test_counts.sh ./test_counts_aggregate + artifacts: + when: always + paths: + - test_counts_aggregate/test_counts_*.json + - test_counts_summary.json + - test_counts_report.md + +deploy_to_profiling_backend: + stage: publish + needs: [ build ] + rules: + - if: '$POPULATE_CACHE' + when: never + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' + when: on_success + - if: '$CI_COMMIT_TAG =~ /^v.*/' + when: on_success + - when: manual + allow_failure: true + trigger: + project: DataDog/profiling-backend + branch: dogfooding + variables: + UPSTREAM_PACKAGE_JOB: $BUILD_JOB_NAME + UPSTREAM_PACKAGE_JOB_ID: $BUILD_JOB_ID + UPSTREAM_PROJECT_ID: $CI_PROJECT_ID + UPSTREAM_PROJECT_NAME: $CI_PROJECT_NAME + UPSTREAM_PIPELINE_ID: $CI_PIPELINE_ID + UPSTREAM_BRANCH: $CI_COMMIT_BRANCH + UPSTREAM_TAG: $CI_COMMIT_TAG + +deploy_to_di_backend:manual: + stage: publish + needs: [ build ] + rules: + - if: '$POPULATE_CACHE' + when: never + - when: manual + allow_failure: true + trigger: + project: DataDog/debugger-demos + branch: main + variables: + UPSTREAM_PACKAGE_JOB: build + UPSTREAM_PROJECT_ID: $CI_PROJECT_ID + UPSTREAM_PROJECT_NAME: $CI_PROJECT_NAME + UPSTREAM_PIPELINE_ID: $CI_PIPELINE_ID + UPSTREAM_BRANCH: $CI_COMMIT_BRANCH + UPSTREAM_TAG: $CI_COMMIT_TAG + UPSTREAM_COMMIT_AUTHOR: $CI_COMMIT_AUTHOR + UPSTREAM_COMMIT_SHORT_SHA: $CI_COMMIT_SHORT_SHA + +deploy_to_reliability_env: + needs: [ build ] + +# If the deploy_to_maven_central job is re-run, re-trigger the deploy_artifacts_to_github job as well so that the artifacts match. +deploy_to_maven_central: + extends: .gradle_build + stage: publish-release-artifacts + needs: + - job: build + - job: system_tests + optional: true + variables: + CACHE_TYPE: "lib" + rules: + - if: '$POPULATE_CACHE' + when: never + - if: '$CI_COMMIT_BRANCH == "master"' + when: on_success + # Do not deploy release candidate versions + - if: '$CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+$/' + when: on_success + allow_failure: true + - when: manual + allow_failure: true + script: + - export MAVEN_CENTRAL_USERNAME=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.central_username --with-decryption --query "Parameter.Value" --out text) + - export MAVEN_CENTRAL_PASSWORD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.central_password --with-decryption --query "Parameter.Value" --out text) + - export GPG_PRIVATE_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.signing.gpg_private_key --with-decryption --query "Parameter.Value" --out text) + - export GPG_PASSWORD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.signing.gpg_passphrase --with-decryption --query "Parameter.Value" --out text) + - ./gradlew publishToSonatype closeSonatypeStagingRepository -PskipTests $GRADLE_ARGS + artifacts: + paths: + - 'workspace/dd-java-agent/build/libs/*.jar' + - 'workspace/dd-trace-api/build/libs/*.jar' + - 'workspace/dd-trace-ot/build/libs/*.jar' + +deploy_snapshot_with_ddprof_snapshot: + extends: .gradle_build + stage: publish + needs: [ build ] + variables: + CACHE_TYPE: "lib" + rules: + - if: '$POPULATE_CACHE' + when: never + # Manual trigger only - for testing with ddprof snapshot versions + - when: manual + allow_failure: true + script: + - export MAVEN_CENTRAL_USERNAME=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.central_username --with-decryption --query "Parameter.Value" --out text) + - export MAVEN_CENTRAL_PASSWORD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.central_password --with-decryption --query "Parameter.Value" --out text) + - export GPG_PRIVATE_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.signing.gpg_private_key --with-decryption --query "Parameter.Value" --out text) + - export GPG_PASSWORD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.signing.gpg_passphrase --with-decryption --query "Parameter.Value" --out text) + - echo "Publishing dd-trace-java snapshot with ddprof snapshot dependency" + - ./gradlew -PbuildInfo.build.number=$CI_JOB_ID -PddprofUseSnapshot publishToSonatype -PskipTests $GRADLE_ARGS + artifacts: + paths: + - 'workspace/dd-java-agent/build/libs/*.jar' + - 'workspace/dd-trace-api/build/libs/*.jar' + - 'workspace/dd-trace-ot/build/libs/*.jar' + +deploy_artifacts_to_github: + stage: publish-release-artifacts + image: registry.ddbuild.io/images/dd-octo-sts-ci-base:2025.06-1 + tags: [ "arch:amd64" ] + id_tokens: + DDOCTOSTS_ID_TOKEN: + aud: dd-octo-sts + rules: + - if: '$POPULATE_CACHE' + when: never + - if: '$CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+$/' + when: on_success + # Requires the deploy_to_maven_central job to have run first (the UP-TO-DATE gradle check across jobs is broken) + # This will deploy the artifacts built from the publishToSonatype task to the GitHub release + needs: + - job: deploy_to_maven_central + # The deploy_to_maven_central job is not run for release candidate versions + optional: true + before_script: + - dd-octo-sts version + - dd-octo-sts debug --scope DataDog/dd-trace-java --policy self.gitlab.release + - dd-octo-sts token --scope DataDog/dd-trace-java --policy self.gitlab.release > github-token.txt + script: + - gh auth login --with-token < github-token.txt + - gh auth status + - export VERSION=${CI_COMMIT_TAG##v} # remove "v" from front of tag to get version + - cp workspace/dd-java-agent/build/libs/dd-java-agent-${VERSION}.jar workspace/dd-java-agent/build/libs/dd-java-agent.jar # upload two filenames + - gh release upload --clobber --repo DataDog/dd-trace-java $CI_COMMIT_TAG workspace/dd-java-agent/build/libs/dd-java-agent.jar + - gh release upload --clobber --repo DataDog/dd-trace-java $CI_COMMIT_TAG workspace/dd-java-agent/build/libs/dd-java-agent-${VERSION}.jar + - gh release upload --clobber --repo DataDog/dd-trace-java $CI_COMMIT_TAG workspace/dd-trace-api/build/libs/dd-trace-api-${VERSION}.jar + - gh release upload --clobber --repo DataDog/dd-trace-java $CI_COMMIT_TAG workspace/dd-trace-ot/build/libs/dd-trace-ot-${VERSION}.jar + after_script: + - dd-octo-sts revoke -t $(cat github-token.txt) + retry: + max: 2 + when: always + +requirements_json_test: + rules: + - when: on_success + variables: + REQUIREMENTS_BLOCK_JSON_PATH: "metadata/requirements-block.json" + REQUIREMENTS_ALLOW_JSON_PATH: "metadata/requirements-allow.json" + +package-oci: + needs: [ build ] + +override_verify_maven_central: + image: registry.ddbuild.io/images/base/gbi-ubuntu_2204:release + stage: publish + needs: [ ] + rules: + - if: '$POPULATE_CACHE' + when: never + - when: manual + allow_failure: true + script: + - touch OVERRIDE_MAVEN_VERIFY + cache: # Cache is used to signal between the override_verify_maven_central and verify_maven_central_deployment jobs + - key: $CI_PIPELINE_ID-OVERRIDE_SIGNAL + paths: + - OVERRIDE_MAVEN_VERIFY + policy: push + unprotect: true + +# Verify Maven Central deployment is publicly available before publishing OCI images +verify_maven_central_deployment: + image: registry.ddbuild.io/images/base/gbi-ubuntu_2204:release + stage: publish-release-artifacts + needs: [ deploy_to_maven_central ] + rules: + - if: '$POPULATE_CACHE' + when: never + - if: '$CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+$/' + when: on_success + cache: # Cache is used to signal between the override_verify_maven_central and verify_maven_central_deployment jobs + - key: $CI_PIPELINE_ID-OVERRIDE_SIGNAL + paths: + - OVERRIDE_MAVEN_VERIFY + policy: pull + unprotect: true + script: + - if [ -f OVERRIDE_MAVEN_VERIFY ]; then echo "SKIPPING MAVEN VERIFICATION"; exit 0; fi + - | + export VERSION=${CI_COMMIT_TAG##v} + ARTIFACT_URLS=( + "https://repo1.maven.org/maven2/com/datadoghq/dd-java-agent/${VERSION}/dd-java-agent-${VERSION}.jar" + "https://repo1.maven.org/maven2/com/datadoghq/dd-trace-api/${VERSION}/dd-trace-api-${VERSION}.jar" + "https://repo1.maven.org/maven2/com/datadoghq/dd-trace-ot/${VERSION}/dd-trace-ot-${VERSION}.jar" + ) + # Try once immediately (fast path on job retry), then every 5 mins for 45 mins. + TRY=0 + MAX_TRIES=10 + RETRY_DELAY=300 + while [ $TRY -lt $MAX_TRIES ]; do + ARTIFACTS_AVAILABLE=true + for URL in "${ARTIFACT_URLS[@]}"; do + if ! curl --location --fail --silent --show-error -I "$URL"; then + ARTIFACTS_AVAILABLE=false + break + fi + done + if [ "$ARTIFACTS_AVAILABLE" = true ]; then + break + fi + TRY=$((TRY + 1)) + if [ $TRY -eq $MAX_TRIES ]; then + echo "The release was not available after 45 mins. Manually re-run the job to try again." + exit 1 + fi + sleep $RETRY_DELAY + done + +publishing-gate: + stage: publish-release-artifacts + needs: + - job: verify_maven_central_deployment + optional: true # Required for releases only + +configure_system_tests: + variables: + SYSTEM_TESTS_REF: "main" # system tests are pinned on release branches only + SYSTEM_TESTS_SCENARIOS_GROUPS: "simple_onboarding,simple_onboarding_profiling,simple_onboarding_appsec,docker-ssi,lib-injection" + +create_key: + stage: generate-signing-key + when: manual + needs: [ ] + variables: + PROJECT_NAME: "dd-trace-java" + EXPORT_TO_KEYSERVER: "true" + image: $REGISTRY/ci/agent-key-management-tools/gpg:1 + script: + - /create.sh + artifacts: + expire_in: 13 mos + paths: + - pubkeys + +validate_supported_configurations_v2_local_file: + extends: .validate_supported_configurations_v2_local_file + variables: + LOCAL_JSON_PATH: "metadata/supported-configurations.json" + BACKFILLED: "false" + +update_central_configurations_version_range_v2: + extends: .update_central_configurations_version_range_v2 + variables: + LOCAL_REPO_NAME: "dd-trace-java" + LOCAL_JSON_PATH: "metadata/supported-configurations.json" + LANGUAGE_NAME: "java" + MULTIPLE_RELEASE_LINES: "false" + SEND_ALIASES: "true" diff --git a/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-3.0/build.gradle b/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-3.0/build.gradle index 72ff7d5d7e8..9718e557981 100644 --- a/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-3.0/build.gradle +++ b/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-3.0/build.gradle @@ -43,6 +43,11 @@ muzzle { } } +testJvmConstraints { + // Test use Cassandra 3 which requires Java 8. (Currently incompatible with Java 9.) + maxJavaVersion = JavaVersion.VERSION_1_8 +} + addTestSuiteForDir('latestDepTest', 'test') dependencies { diff --git a/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-3.8/build.gradle b/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-3.8/build.gradle index 2f34b57f836..5e74728b352 100644 --- a/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-3.8/build.gradle +++ b/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-3.8/build.gradle @@ -19,6 +19,11 @@ muzzle { } } +testJvmConstraints { + // Test use Cassandra 3 which requires Java 8. (Currently incompatible with Java 9.) + maxJavaVersion = JavaVersion.VERSION_1_8 +} + addTestSuiteForDir('latestDepTest', 'test') dependencies { diff --git a/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-4.0/build.gradle b/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-4.0/build.gradle index 681c2ebb304..5a87a224cd0 100644 --- a/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-4.0/build.gradle +++ b/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-4.0/build.gradle @@ -12,6 +12,12 @@ muzzle { } } +testJvmConstraints { + // TODO Java 17: The embedded cassandra deadlocks on start every time on Java 17 + // This can be changed to use test-containers + maxJavaVersion = JavaVersion.VERSION_15 +} + addTestSuiteForDir('latestDepTest', 'test') dependencies { From 76fee257b0a21cdf11596bc7208ffd80de66ab3c Mon Sep 17 00:00:00 2001 From: Brice Dutheil Date: Thu, 24 Sep 2026 18:50:48 +0200 Subject: [PATCH 13/15] style: format container fixtures --- .../couchbase/couchbase-3.1/build.gradle | 4 ++-- .../src/test/groovy/CouchbaseClient31Test.groovy | 2 +- .../couchbase/couchbase-3.2/build.gradle | 4 ++-- .../src/test/groovy/CouchbaseClient32Test.groovy | 2 +- .../src/test/groovy/Lettuce5ClientTestBase.groovy | 12 ++++++------ .../reactorTest/groovy/ReactorRabbitMQTest.groovy | 2 +- .../src/test/groovy/RabbitMQTest.groovy | 2 +- .../src/test/groovy/RedissonClientTest.groovy | 4 ++-- .../src/test/groovy/RedissonClientTest.groovy | 4 ++-- .../src/test/groovy/RedissonClientTest.groovy | 4 ++-- .../src/test/groovy/VertxRedisTestBase.groovy | 4 ++-- .../smoketest/SpringBootMongoIntegrationTest.groovy | 2 +- 12 files changed, 23 insertions(+), 23 deletions(-) diff --git a/dd-java-agent/instrumentation/couchbase/couchbase-3.1/build.gradle b/dd-java-agent/instrumentation/couchbase/couchbase-3.1/build.gradle index 1ac8cf0bbe7..36543313dfe 100644 --- a/dd-java-agent/instrumentation/couchbase/couchbase-3.1/build.gradle +++ b/dd-java-agent/instrumentation/couchbase/couchbase-3.1/build.gradle @@ -22,8 +22,8 @@ dependencies { compileOnly group: 'com.couchbase.client', name: 'java-client', version: '3.1.0' testContainerImage(image( DefaultNativePlatform.getCurrentArchitecture().isArm64() - ? 'couchbase/server:7.1.0-aarch64' - : 'couchbase/server:7.1.0', + ? 'couchbase/server:7.1.0-aarch64' + : 'couchbase/server:7.1.0', 'test.couchbase.image')) testImplementation group: 'com.couchbase.client', name: 'java-client', version: '3.1.0' diff --git a/dd-java-agent/instrumentation/couchbase/couchbase-3.1/src/test/groovy/CouchbaseClient31Test.groovy b/dd-java-agent/instrumentation/couchbase/couchbase-3.1/src/test/groovy/CouchbaseClient31Test.groovy index e55dc3c8a32..fc531250e73 100644 --- a/dd-java-agent/instrumentation/couchbase/couchbase-3.1/src/test/groovy/CouchbaseClient31Test.groovy +++ b/dd-java-agent/instrumentation/couchbase/couchbase-3.1/src/test/groovy/CouchbaseClient31Test.groovy @@ -42,7 +42,7 @@ abstract class CouchbaseClient31Test extends VersionedNamingTestBase { def setupSpec() { couchbase = new CouchbaseContainer( DockerImageName.parse(System.getProperty("test.couchbase.image")) - .asCompatibleSubstituteFor("couchbase/server")) + .asCompatibleSubstituteFor("couchbase/server")) .withBucket(new BucketDefinition(BUCKET).withPrimaryIndex(true)) .withStartupTimeout(Duration.ofSeconds(240)) .withStartupAttempts(3) diff --git a/dd-java-agent/instrumentation/couchbase/couchbase-3.2/build.gradle b/dd-java-agent/instrumentation/couchbase/couchbase-3.2/build.gradle index 18acd377bca..0a20c86d311 100644 --- a/dd-java-agent/instrumentation/couchbase/couchbase-3.2/build.gradle +++ b/dd-java-agent/instrumentation/couchbase/couchbase-3.2/build.gradle @@ -22,8 +22,8 @@ dependencies { compileOnly group: 'com.couchbase.client', name: 'java-client', version: '3.2.0' testContainerImage(image( DefaultNativePlatform.getCurrentArchitecture().isArm64() - ? 'couchbase/server:7.1.0-aarch64' - : 'couchbase/server:7.1.0', + ? 'couchbase/server:7.1.0-aarch64' + : 'couchbase/server:7.1.0', 'test.couchbase.image')) testImplementation group: 'com.couchbase.client', name: 'java-client', version: '3.2.0' diff --git a/dd-java-agent/instrumentation/couchbase/couchbase-3.2/src/test/groovy/CouchbaseClient32Test.groovy b/dd-java-agent/instrumentation/couchbase/couchbase-3.2/src/test/groovy/CouchbaseClient32Test.groovy index b55d08a9efa..fe11f51f2ff 100644 --- a/dd-java-agent/instrumentation/couchbase/couchbase-3.2/src/test/groovy/CouchbaseClient32Test.groovy +++ b/dd-java-agent/instrumentation/couchbase/couchbase-3.2/src/test/groovy/CouchbaseClient32Test.groovy @@ -48,7 +48,7 @@ abstract class CouchbaseClient32Test extends VersionedNamingTestBase { def setupSpec() { couchbase = new CouchbaseContainer( DockerImageName.parse(System.getProperty("test.couchbase.image")) - .asCompatibleSubstituteFor("couchbase/server")) + .asCompatibleSubstituteFor("couchbase/server")) .withBucket(new BucketDefinition(BUCKET).withPrimaryIndex(true)) .withStartupTimeout(Duration.ofSeconds(240)) .withStartupAttempts(3) diff --git a/dd-java-agent/instrumentation/lettuce/lettuce-5.0/src/test/groovy/Lettuce5ClientTestBase.groovy b/dd-java-agent/instrumentation/lettuce/lettuce-5.0/src/test/groovy/Lettuce5ClientTestBase.groovy index ecff8ad6984..50ccbdbb137 100644 --- a/dd-java-agent/instrumentation/lettuce/lettuce-5.0/src/test/groovy/Lettuce5ClientTestBase.groovy +++ b/dd-java-agent/instrumentation/lettuce/lettuce-5.0/src/test/groovy/Lettuce5ClientTestBase.groovy @@ -21,9 +21,9 @@ abstract class Lettuce5ClientTestBase extends VersionedNamingTestBase { @Shared Map testHashMap = [ - firstname: "John", - lastname : "Doe", - age : "53" + firstname: "John", + lastname : "Doe", + age : "53" ] int port @@ -34,9 +34,9 @@ abstract class Lettuce5ClientTestBase extends VersionedNamingTestBase { String embeddedDbUri RedisContainer redisServer = new RedisContainer( - DockerImageName.parse(System.getProperty("test.redis.image")) - .asCompatibleSubstituteFor("redis")) - .waitingFor(Wait.forListeningPort()) + DockerImageName.parse(System.getProperty("test.redis.image")) + .asCompatibleSubstituteFor("redis")) + .waitingFor(Wait.forListeningPort()) RedisClient redisClient StatefulRedisConnection connection diff --git a/dd-java-agent/instrumentation/rabbitmq-amqp-2.7/src/reactorTest/groovy/ReactorRabbitMQTest.groovy b/dd-java-agent/instrumentation/rabbitmq-amqp-2.7/src/reactorTest/groovy/ReactorRabbitMQTest.groovy index 804f7ccc6df..f0289cf5007 100644 --- a/dd-java-agent/instrumentation/rabbitmq-amqp-2.7/src/reactorTest/groovy/ReactorRabbitMQTest.groovy +++ b/dd-java-agent/instrumentation/rabbitmq-amqp-2.7/src/reactorTest/groovy/ReactorRabbitMQTest.groovy @@ -32,7 +32,7 @@ class ReactorRabbitMQTest extends InstrumentationSpecification { def setupSpec() { rabbitMQContainer = new RabbitMQContainer( DockerImageName.parse(System.getProperty("test.rabbitmq.image")) - .asCompatibleSubstituteFor("rabbitmq")) + .asCompatibleSubstituteFor("rabbitmq")) .withExposedPorts(defaultRabbitMQPort) .withStartupTimeout(Duration.ofSeconds(120)) rabbitMQContainer.start() diff --git a/dd-java-agent/instrumentation/rabbitmq-amqp-2.7/src/test/groovy/RabbitMQTest.groovy b/dd-java-agent/instrumentation/rabbitmq-amqp-2.7/src/test/groovy/RabbitMQTest.groovy index 1308127dbf0..7c638baac62 100644 --- a/dd-java-agent/instrumentation/rabbitmq-amqp-2.7/src/test/groovy/RabbitMQTest.groovy +++ b/dd-java-agent/instrumentation/rabbitmq-amqp-2.7/src/test/groovy/RabbitMQTest.groovy @@ -73,7 +73,7 @@ abstract class RabbitMQTestBase extends VersionedNamingTestBase { def setupSpec() { rabbitMQContainer = new RabbitMQContainer( DockerImageName.parse(System.getProperty("test.rabbitmq.image")) - .asCompatibleSubstituteFor("rabbitmq")) + .asCompatibleSubstituteFor("rabbitmq")) .withExposedPorts(defaultRabbitMQPort) .withStartupTimeout(Duration.ofSeconds(120)) rabbitMQContainer.start() diff --git a/dd-java-agent/instrumentation/redisson/redisson-2.0.0/src/test/groovy/RedissonClientTest.groovy b/dd-java-agent/instrumentation/redisson/redisson-2.0.0/src/test/groovy/RedissonClientTest.groovy index bf3ce441cef..6fae340e06e 100644 --- a/dd-java-agent/instrumentation/redisson/redisson-2.0.0/src/test/groovy/RedissonClientTest.groovy +++ b/dd-java-agent/instrumentation/redisson/redisson-2.0.0/src/test/groovy/RedissonClientTest.groovy @@ -20,8 +20,8 @@ import spock.lang.Shared abstract class RedissonClientTest extends VersionedNamingTestBase { @Shared RedisServer redisServer = new RedisContainer( - DockerImageName.parse(System.getProperty("test.redis.image"))) - .waitingFor(Wait.forListeningPort()) + DockerImageName.parse(System.getProperty("test.redis.image"))) + .waitingFor(Wait.forListeningPort()) @Shared Config config = new Config() diff --git a/dd-java-agent/instrumentation/redisson/redisson-2.3.0/src/test/groovy/RedissonClientTest.groovy b/dd-java-agent/instrumentation/redisson/redisson-2.3.0/src/test/groovy/RedissonClientTest.groovy index 286a63ad83d..dac92fa960e 100644 --- a/dd-java-agent/instrumentation/redisson/redisson-2.3.0/src/test/groovy/RedissonClientTest.groovy +++ b/dd-java-agent/instrumentation/redisson/redisson-2.3.0/src/test/groovy/RedissonClientTest.groovy @@ -17,8 +17,8 @@ abstract class RedissonClientTest extends VersionedNamingTestBase { @Shared RedisServer redisServer = new RedisContainer( - DockerImageName.parse(System.getProperty("test.redis.image"))) - .waitingFor(Wait.forListeningPort()) + DockerImageName.parse(System.getProperty("test.redis.image"))) + .waitingFor(Wait.forListeningPort()) @Shared Config config = new Config() diff --git a/dd-java-agent/instrumentation/redisson/redisson-3.10.3/src/test/groovy/RedissonClientTest.groovy b/dd-java-agent/instrumentation/redisson/redisson-3.10.3/src/test/groovy/RedissonClientTest.groovy index 43562d2b55f..b426c37d8ea 100644 --- a/dd-java-agent/instrumentation/redisson/redisson-3.10.3/src/test/groovy/RedissonClientTest.groovy +++ b/dd-java-agent/instrumentation/redisson/redisson-3.10.3/src/test/groovy/RedissonClientTest.groovy @@ -17,8 +17,8 @@ abstract class RedissonClientTest extends VersionedNamingTestBase { @Shared RedisServer redisServer = new RedisContainer( - DockerImageName.parse(System.getProperty("test.redis.image"))) - .waitingFor(Wait.forListeningPort()) + DockerImageName.parse(System.getProperty("test.redis.image"))) + .waitingFor(Wait.forListeningPort()) @Shared Config config = new Config() diff --git a/dd-java-agent/instrumentation/vertx/vertx-redis-client/vertx-redis-client-3.9/src/test/groovy/VertxRedisTestBase.groovy b/dd-java-agent/instrumentation/vertx/vertx-redis-client/vertx-redis-client-3.9/src/test/groovy/VertxRedisTestBase.groovy index 0c7b5f4abd5..dc8038613f8 100644 --- a/dd-java-agent/instrumentation/vertx/vertx-redis-client/vertx-redis-client-3.9/src/test/groovy/VertxRedisTestBase.groovy +++ b/dd-java-agent/instrumentation/vertx/vertx-redis-client/vertx-redis-client-3.9/src/test/groovy/VertxRedisTestBase.groovy @@ -37,8 +37,8 @@ abstract class VertxRedisTestBase extends VersionedNamingTestBase { @AutoCleanup(value = "stop") @Shared def redisServer = new RedisContainer( - DockerImageName.parse(System.getProperty("test.redis.image"))) - .waitingFor(Wait.forListeningPort()) + DockerImageName.parse(System.getProperty("test.redis.image"))) + .waitingFor(Wait.forListeningPort()) @Shared @AutoCleanup(quiet = true) diff --git a/dd-smoke-tests/springboot-mongo/src/test/groovy/datadog/smoketest/SpringBootMongoIntegrationTest.groovy b/dd-smoke-tests/springboot-mongo/src/test/groovy/datadog/smoketest/SpringBootMongoIntegrationTest.groovy index 38a3d62de15..032c583cc45 100644 --- a/dd-smoke-tests/springboot-mongo/src/test/groovy/datadog/smoketest/SpringBootMongoIntegrationTest.groovy +++ b/dd-smoke-tests/springboot-mongo/src/test/groovy/datadog/smoketest/SpringBootMongoIntegrationTest.groovy @@ -25,7 +25,7 @@ class SpringBootMongoIntegrationTest extends AbstractServerSmokeTest { void beforeProcessBuilders() { mongoDbContainer = new MongoDBContainer( DockerImageName.parse(System.getProperty("test.mongo.image")) - .asCompatibleSubstituteFor("mongo")) + .asCompatibleSubstituteFor("mongo")) mongoDbContainer.start() mongoDbUri = mongoDbContainer.replicaSetUrl } From 940c875f0b9f179e89f6d17ffd2bd29802ad7b6c Mon Sep 17 00:00:00 2001 From: Brice Dutheil Date: Thu, 24 Sep 2026 21:23:17 +0200 Subject: [PATCH 14/15] docs: Typo --- docs/how_to_test.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/how_to_test.md b/docs/how_to_test.md index 03bc8d7d487..e809fc38e7c 100644 --- a/docs/how_to_test.md +++ b/docs/how_to_test.md @@ -58,8 +58,8 @@ In order to identify such tests and avoid the continuous integration to fail, th ## Tests that use containers > [!IMPORTANT] -> Don't use image name in Test Container constructors like `new CassandraContainer("cassandra:4")` , -> or `new GenericContainer("icr.io/appcafe/websphere-traditional:latest")`. Tags can be moved. +> Don't use image name in Test Container constructors like `new CassandraContainer("cassandra:4")`, +> or `new GenericContainer("icr.io/appcafe/websphere-traditional:latest")`. Image tags can change. > Also, these are not properly tracked as _test_ task inputs and as such can't be fingerprinted. > Instead, use the `dd-trace-java.testcontainers` plugin to declare these as dependencies, > it will resolve the actual image digest before running the test. From 7db6d24e4f2bf42a437ab4f7123191b87d9e9ed5 Mon Sep 17 00:00:00 2001 From: Brice Dutheil Date: Thu, 24 Sep 2026 21:55:42 +0200 Subject: [PATCH 15/15] fix: preserve WebSphere Java 8 smoke coverage --- dd-smoke-tests/websphere-jmx/build.gradle | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dd-smoke-tests/websphere-jmx/build.gradle b/dd-smoke-tests/websphere-jmx/build.gradle index 4ba5b2d2c58..b3ee93ac840 100644 --- a/dd-smoke-tests/websphere-jmx/build.gradle +++ b/dd-smoke-tests/websphere-jmx/build.gradle @@ -12,8 +12,7 @@ dependencies { testJvmConstraints { // there is no need to run it multiple times since it runs on a container - minJavaVersion = JavaVersion.VERSION_25 - maxJavaVersion = JavaVersion.VERSION_25 + maxJavaVersion = JavaVersion.VERSION_1_8 } tasks.withType(Test).configureEach {