diff --git a/build-logic/settings.gradle.kts b/build-logic/settings.gradle.kts index 6d1ceec50bb..006d14e9c9a 100644 --- a/build-logic/settings.gradle.kts +++ b/build-logic/settings.gradle.kts @@ -55,3 +55,4 @@ rootProject.name = "build-logic" include(":conventions") include(":smoke-test") +include(":testcontainers") diff --git a/build-logic/smoke-test/src/main/resources/datadog/buildlogic/smoketest/proxy-repositories.init.gradle.kts b/build-logic/smoke-test/src/main/resources/datadog/buildlogic/smoketest/proxy-repositories.init.gradle.kts index 4d059fd9645..552577d09e1 100644 --- a/build-logic/smoke-test/src/main/resources/datadog/buildlogic/smoketest/proxy-repositories.init.gradle.kts +++ b/build-logic/smoke-test/src/main/resources/datadog/buildlogic/smoketest/proxy-repositories.init.gradle.kts @@ -1,12 +1,12 @@ import org.gradle.api.Action import org.gradle.api.Project +import org.gradle.kotlin.dsl.* import org.gradle.api.artifacts.dsl.RepositoryHandler import org.gradle.api.artifacts.repositories.MavenArtifactRepository import org.gradle.api.initialization.Settings import java.net.URI -// Requires Gradle 6.8+ and assumes project repositories are allowed by the target build. - +// Requires Gradle 6.8+ for Settings.providers and Settings.dependencyResolutionManagement. gradle.beforeSettings(Action { val gradlePluginProxy = providers.gradleProperty("gradlePluginProxy").orNull val mavenRepositoryProxy = providers.gradleProperty("mavenRepositoryProxy").orNull @@ -23,6 +23,8 @@ gradle.beforeSettings(Action { withType(MavenArtifactRepository::class.java).configureEach { // A repository declared without a URL has a null one until Gradle validates it; leave it be // so the nested build reports that itself instead of failing inside this init script. + // See https://github.com/gradle/gradle/issues/37612 + @Suppress("UNNECESSARY_SAFE_CALL") val repositoryUrl = url?.toString()?.trimEnd('/') if (repositoryUrl != null && repositoryUrl in mavenCentralUrls) { url = URI(proxy) @@ -33,6 +35,8 @@ gradle.beforeSettings(Action { fun RepositoryHandler.removeDuplicateMavenProxy() { val proxyUrl = mavenRepositoryProxy?.takeIf { it.isNotBlank() }?.trimEnd('/') ?: return + // see https://github.com/gradle/gradle/issues/37612 + @Suppress("UNNECESSARY_SAFE_CALL") val proxies = withType(MavenArtifactRepository::class.java) .filter { it.url?.toString()?.trimEnd('/') == proxyUrl } // Keep the injected repository: it is the only one known to be unrestricted, since a declared @@ -92,7 +96,7 @@ gradle.beforeSettings(Action { } }) - gradle.afterProject(Action { + gradle.afterProject(Action { repositories.removeDuplicateMavenProxy() }) }) diff --git a/build-logic/testcontainers/README.md b/build-logic/testcontainers/README.md new file mode 100644 index 00000000000..52786aa6ffb --- /dev/null +++ b/build-logic/testcontainers/README.md @@ -0,0 +1,156 @@ +# Container images as test inputs + +Apply this plugin only in modules that use containers: + +```kotlin +import datadog.buildlogic.testcontainers.image + +plugins { + id("dd-trace-java.testcontainers") +} + +dependencies { + testContainerImage(image("cassandra:4", "test.cassandra.image")) +} +``` + +> [!NOTE] +> `testContainerImage` is a configuration. +> Note the `image` function import. + +Use `GenericContainer` or the dedicated container type and pass the relevant +system property through its `DockerImageName` constructor. Note the compatibility +declaration that let it accept a mirrored image without changing the resolved +reference: + +```java +import org.testcontainers.cassandra.CassandraContainer; +import org.testcontainers.utility.DockerImageName; + +CassandraContainer container = new CassandraContainer( + DockerImageName.parse(System.getProperty("test.cassandra.image")) + .asCompatibleSubstituteFor("cassandra")); +``` + +`testContainerImage` is a dependency-scope configuration: it stores declarations +and is neither resolvable nor consumable. `image("image", "system property name")` +creates a Gradle module dependency. +Note that these declarations stay outside Java classpaths. Before test task runs, the plugin +resolves the images digest. + +The plugin identifies a test task's source sets from its `testClassesDirs` and +reads their image configurations. Forked or differently named tasks running the +same compiled tests therefore inherit the same images. Additional images can be +declared in `ContainerImage`. +In practice the plugin automatically creates an image configuration for each `*Implementation` +configuration, including those added by JVM Test Suites. For example: + +```kotlin +testing { + suites { + register("integrationTest") { + useJUnitJupiter() + project.dependencies { + add("integrationTestContainerImage", image("redis:7-alpine", "test.redis.image")) + } + } + } +} +``` + +With the repository's legacy Groovy `addTestSuite` helper, declare the image in +the project's `dependencies` block: + +```groovy +addTestSuite('integrationTest') + +dependencies { + integrationTestContainerImage(image('redis:7-alpine', 'test.redis.image')) +} +``` + +Automatic inheritance follows Java dependency configurations: if +`latestDepTestImplementation` extends `testImplementation`, its tests also read +`testContainerImage`. The plugin maps each configuration in that hierarchy to its +image counterpart. It also reads parents declared with `extendsFrom` on image +configurations: + +```kotlin +val databaseImages = configurations.dependencyScope("databaseImages") +configurations.testContainerImage { + extendsFrom(databaseImages.get()) +} +dependencies { + add(databaseImages.name, image("postgres:16-alpine", "test.postgres.image")) +} +``` + +The plugin ensures there’s only one image per system property for a given test task. +Declaring both `image("redis:7", "test.redis.image")` and `image("redis:8", "test.redis.image")` +for that task, directly or through inherited configurations, will fail. + +Groovy DSL is declared the same `dependencies { testContainerImage(image('cassandra:4', 'test.cassandra.image')) }`. + +## Resolution and test execution + +The plugin resolves declared tags to immutable reference like `registry/repository@sha256:...` +before Gradle checks the build-cache for reusable test results. These references are +both test inputs and system property values, so actual code in the tests use the exact +images. + +Like when java dependencies are declared with `+`, the tags are resolved on each invocation, +including when reusing the configuration cache. With a build cache this means unchanged +references allow to safely reuse test results, while Gradle is blind when the image is +resolving within test code. + +> [!NOTE] +> Only explicitly declared images can be tracked. Implicit Testcontainers helpers such as +> Alpine/Ryuk are not included. This can't be avoided. + +## Local registries and CI mirrors + +This plugin honors `TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX` environment variable. + +Locally, when asking for `cassandra:4` it is resolved from Docker Hub. +However, on [CI](../../.gitlab-ci.yml), the env var `TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX=...` is set, +so the plugin resolves and fingerprints from the prefix, instead, e.g.: + +```text +Local: registry-1.docker.io/library/cassandra@sha256:... +CI: registry.ddbuild.io/images/mirror/cassandra@sha256:... +``` + +More precisely: +- Prefix precedence: the test task's `TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX`, then + `hub.image.name.prefix` in `~/.testcontainers.properties`, then + `testcontainers.properties` in test classpath directories. +- Explicit registry hosts bypass the prefix. Other mappings belong in the image + declaration, as in [JDBC's SQL Server example](../../dd-java-agent/instrumentation/jdbc/build.gradle). +- Private registries use Docker's `config.json` and credential helpers, honoring + `DOCKER_CONFIG`. Remote registries require TLS; loopback registries also allow + local development certificates and HTTP. + +## Testcontainers configuration restrictions + +Testcontainers supports [configuration properties and environment variables](https://java.testcontainers.org/features/configuration/) +that change its runtime behavior. In particular, its [image-substitution settings](https://java.testcontainers.org/features/image_name_substitution/) +can replace the images used when containers start. + +> [!IMPORTANT] +> This plugin forbids custom image substitutors (`image.substitutor`) and all +> `*.container.image` overrides: **the test task fails when these settings are +> detected**. Runtime replacements could make the tested image differ from the one +> in Gradle's cache key. This restriction includes helper-image overrides such as +> `ryuk.container.image`. + +The check covers environment variables, `~/.testcontainers.properties`, and +`testcontainers.properties` in test classpath directories. The Docker Hub prefix +described above remains supported. + +For declared test images, choose the replacement directly in the declaration, +for example `image("registry.example/redis:7", "test.redis.image")`. This keeps the +image used by the test consistent with the image in Gradle's cache key. + +The plugin does not inspect `testcontainers.properties` inside dependency JARs. +Testcontainers can still load substitutions from those files; they escape this +check and can invalidate cache correctness. diff --git a/build-logic/testcontainers/build.gradle.kts b/build-logic/testcontainers/build.gradle.kts new file mode 100644 index 00000000000..597b606ed86 --- /dev/null +++ b/build-logic/testcontainers/build.gradle.kts @@ -0,0 +1,54 @@ +plugins { + `java-gradle-plugin` + `kotlin-dsl` + alias(libs.plugins.shadow) +} + +val jib = configurations.register("jib") { + dependencies.add(project.dependencies.create("com.google.cloud.tools:jib-core:0.28.2")) +} +configurations.compileOnly { extendsFrom(jib) } + +// buildSrc exports an older HttpClient through the parent classloader. Keep Jib's +// dependencies private, including when this plugin is consumed as an included build. +tasks.shadowJar { + configurations.add(jib) + enableAutoRelocation = true + relocationPrefix = "datadog.buildlogic.testcontainers.internal" + mergeServiceFiles() +} +configurations.apiElements { + outgoing.artifacts.clear() + outgoing.variants.clear() + outgoing.artifact(tasks.shadowJar) +} +configurations.runtimeElements { + outgoing.artifacts.clear() + outgoing.variants.clear() + outgoing.artifact(tasks.shadowJar) +} +tasks.pluginUnderTestMetadata { + pluginClasspath.setFrom(tasks.shadowJar) +} + +gradlePlugin { + plugins { + create("testcontainers") { + id = "dd-trace-java.testcontainers" + implementationClass = "datadog.buildlogic.testcontainers.TestcontainersPlugin" + } + } +} + +testing { + suites { + named("test") { + useJUnitJupiter(libs.versions.junit5) + dependencies { + implementation(libs.assertj.core) + implementation(libs.okhttp3.mockwebserver) + implementation("com.squareup.okhttp3:okhttp-tls:${libs.versions.okhttp3.testing.get()}") + } + } + } +} diff --git a/build-logic/testcontainers/src/main/kotlin/datadog/buildlogic/testcontainers/ContainerImageArguments.kt b/build-logic/testcontainers/src/main/kotlin/datadog/buildlogic/testcontainers/ContainerImageArguments.kt new file mode 100644 index 00000000000..44351ad7003 --- /dev/null +++ b/build-logic/testcontainers/src/main/kotlin/datadog/buildlogic/testcontainers/ContainerImageArguments.kt @@ -0,0 +1,33 @@ +package datadog.buildlogic.testcontainers + +import org.gradle.api.provider.Provider +import org.gradle.api.services.ServiceReference +import org.gradle.api.tasks.Input +import org.gradle.api.tasks.Internal +import org.gradle.api.tasks.Nested +import org.gradle.api.tasks.Optional +import org.gradle.process.CommandLineArgumentProvider +import java.io.File + +class ContainerImageArguments( + @get:Nested @get:Optional val containers: Provider, +) : CommandLineArgumentProvider { + override fun asArguments(): Iterable = + containers.orNull + ?.images + ?.map { (name, image) -> "-D$name=$image" } + .orEmpty() +} + +class ContainerImageInputs( + @get:Internal val declarations: Map, + @get:Internal val imageEnvironment: Map, + @get:Internal val configurationFiles: List, + @get:ServiceReference("testContainerImageResolver") val resolver: Provider, +) { + // Read during Test input snapshotting, after skip predicates and before cache lookup. + // Only the service reference is serialized by the configuration cache; its memo is per build. + @get:Input + val images: Map + get() = resolver.get().resolve(declarations, imageEnvironment, configurationFiles) +} diff --git a/build-logic/testcontainers/src/main/kotlin/datadog/buildlogic/testcontainers/ContainerImages.kt b/build-logic/testcontainers/src/main/kotlin/datadog/buildlogic/testcontainers/ContainerImages.kt new file mode 100644 index 00000000000..fed8c34dcb0 --- /dev/null +++ b/build-logic/testcontainers/src/main/kotlin/datadog/buildlogic/testcontainers/ContainerImages.kt @@ -0,0 +1,27 @@ +package datadog.buildlogic.testcontainers + +import org.gradle.api.InvalidUserDataException +import org.gradle.api.artifacts.ExternalModuleDependency +import org.gradle.api.artifacts.dsl.DependencyHandler +import org.gradle.api.attributes.Attribute + +internal const val IMAGE_DEPENDENCY_GROUP = "datadog.container-image" +internal val IMAGE_REFERENCE = Attribute.of("datadog.container-image.reference", String::class.java) + +/** Creates an image declaration for the test JVM property that consumes it. */ +fun DependencyHandler.image( + reference: String, + systemProperty: String, +): ExternalModuleDependency { + if (reference.isBlank()) { + throw InvalidUserDataException("Container image reference must not be empty") + } + if (!systemProperty.matches(Regex("[A-Za-z0-9_.-]+"))) { + throw InvalidUserDataException("Invalid container image system property: $systemProperty") + } + + // These logical dependencies are read from declaration-only configurations, never resolved by Gradle. + return (create("$IMAGE_DEPENDENCY_GROUP:$systemProperty") as ExternalModuleDependency).apply { + attributes { attribute(IMAGE_REFERENCE, reference) } + } +} diff --git a/build-logic/testcontainers/src/main/kotlin/datadog/buildlogic/testcontainers/ImageResolver.kt b/build-logic/testcontainers/src/main/kotlin/datadog/buildlogic/testcontainers/ImageResolver.kt new file mode 100644 index 00000000000..d348b4e6b85 --- /dev/null +++ b/build-logic/testcontainers/src/main/kotlin/datadog/buildlogic/testcontainers/ImageResolver.kt @@ -0,0 +1,132 @@ +package datadog.buildlogic.testcontainers + +import com.google.cloud.tools.jib.api.ImageReference +import com.google.cloud.tools.jib.api.RegistryUnauthorizedException +import com.google.cloud.tools.jib.event.EventHandlers +import com.google.cloud.tools.jib.frontend.CredentialRetrieverFactory +import com.google.cloud.tools.jib.http.FailoverHttpClient +import com.google.cloud.tools.jib.registry.RegistryClient +import org.gradle.api.GradleException +import org.gradle.api.InvalidUserDataException +import org.gradle.api.services.BuildService +import org.gradle.api.services.BuildServiceParameters +import java.io.File +import java.nio.file.Paths +import java.util.Properties +import java.util.concurrent.ConcurrentHashMap + +/** + * Resolves manifests without a Docker daemon or image-layer downloads. Memoized for this build only. + */ +abstract class ImageResolver : BuildService { + private val resolved = ConcurrentHashMap() + + fun resolve( + declarations: Map, + environment: Map, + configurationFiles: List, + ): Map { + val configuration = Properties() + configurationFiles.asReversed().filter { it.isFile }.forEach { file -> + file.inputStream().use { configuration.load(it) } + } + val customSubstitution = + configuration + .stringPropertyNames() + .filter { + it == "image.substitutor" || it.endsWith(".container.image") + }.any { configuration.getProperty(it).isNotBlank() } || + environment.any { (key, value) -> + (key == "TESTCONTAINERS_IMAGE_SUBSTITUTOR" || key.endsWith("_CONTAINER_IMAGE")) && value.isNotBlank() + } + if (customSubstitution) { + throw InvalidUserDataException( + "Custom Testcontainers image substitutions can replace a fingerprinted digest; move image overrides into testContainerImage declarations", + ) + } + + val prefix = + environment["TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX"]?.takeIf { it.isNotEmpty() } + ?: configuration.getProperty("hub.image.name.prefix", "") + return declarations.mapValues { (_, reference) -> resolve(reference, prefix) } + } + + private fun resolve( + reference: String, + hubPrefix: String, + ): String { + val firstComponent = reference.substringBefore('/') + val explicitRegistry = + reference.contains('/') && + (firstComponent.contains('.') || firstComponent.contains(':') || firstComponent == "localhost") + val effective = if (explicitRegistry) reference else hubPrefix + reference + return resolved.computeIfAbsent(effective) { resolveManifest(it) } + } + + /** + * Image digest resolution. + * + * **Why jib?** + * + * Jib provides a daemonless registry client that retrieves the manifest without downloading layers. + * However, it relies on Jib non-public API. But this is preferable to maintaining registry authentication + * code in this case. + * [Jib Core](https://github.com/GoogleContainerTools/jib/blob/master/jib-core/README.md) + * [RegistryClient](https://github.com/GoogleContainerTools/jib/blob/master/jib-core/src/main/java/com/google/cloud/tools/jib/registry/RegistryClient.java) + * + * Considered alternatives: + * - Testcontainers uses docker-java under the hood, it reuses the local image cache + * or pull the complete image, and returns the image name rather than the remote manifest digest. + * That is too late and too expensive for Gradle input fingerprinting. + * [RemoteDockerImage](https://github.com/testcontainers/testcontainers-java/blob/1.21.4/core/src/main/java/org/testcontainers/images/RemoteDockerImage.java) + * - Docker’s `/distribution/{name}/json` endpoint can obtain the registry digest without + * downloading layers. But it still requires a running Docker daemon, and docker-java 3.7.1 + * has no typed command for it. [Docker Engine API](https://docs.docker.com/reference/api/engine/version/v1.40/) + * - A direct HTTP implementation is possible, but it must implement registry naming, + * OCI/Docker media types, bearer-token challenges, basic authentication, Docker credential helpers, + * redirects, TLS, and digest validation. + * [Registry authentication](https://docs.docker.com/reference/api/registry/auth/) + * [Registry API specification](https://github.com/distribution/distribution/blob/main/docs/content/spec/api.md) + */ + private fun resolveManifest(reference: String): String { + try { + val image = ImageReference.parse(reference) + val repository = "${image.registry}/${image.repository}" + if (image.digest.isPresent) { + return "$repository@${image.digest.get()}" + } + // Local registries are useful for development and hermetic tests. Remote registries require TLS. + val loopback = image.registry.substringBefore(':') in setOf("localhost", "127.0.0.1") + val http = FailoverHttpClient(loopback, loopback) {} + try { + val factory = RegistryClient.factory(EventHandlers.NONE, image.registry, image.repository, http) + var client = factory.newRegistryClient() + val manifest = + try { + client.pullManifest(image.tag.orElse("latest")) + } catch (unauthorized: RegistryUnauthorizedException) { + val dockerConfig = + System.getenv("DOCKER_CONFIG") + ?: Paths.get(System.getProperty("user.home"), ".docker").toString() + val credential = + CredentialRetrieverFactory + .forImage(image) {} + .dockerConfig(Paths.get(dockerConfig, "config.json")) + .retrieve() + .orElse(null) + client = factory.setCredential(credential).newRegistryClient() + if (!client.doPullBearerAuth()) { + if (credential == null) throw unauthorized + client.configureBasicAuth() + } + client.pullManifest(image.tag.orElse("latest")) + } + return "$repository@${manifest.digest}" + } finally { + http.shutDown() + } + } catch (failure: Exception) { + throw GradleException("Cannot resolve test container image '$reference'; refusing to reuse test results without its digest", failure) + } + } +} diff --git a/build-logic/testcontainers/src/main/kotlin/datadog/buildlogic/testcontainers/TestcontainersPlugin.kt b/build-logic/testcontainers/src/main/kotlin/datadog/buildlogic/testcontainers/TestcontainersPlugin.kt new file mode 100644 index 00000000000..36099333799 --- /dev/null +++ b/build-logic/testcontainers/src/main/kotlin/datadog/buildlogic/testcontainers/TestcontainersPlugin.kt @@ -0,0 +1,111 @@ +package datadog.buildlogic.testcontainers + +import groovy.lang.Closure +import org.gradle.api.InvalidUserDataException +import org.gradle.api.Plugin +import org.gradle.api.Project +import org.gradle.api.artifacts.ExternalModuleDependency +import org.gradle.api.tasks.SourceSetContainer +import org.gradle.api.tasks.testing.Test +import org.gradle.kotlin.dsl.getByType +import org.gradle.kotlin.dsl.withType +import java.io.File + +/** Declares container images alongside the dependencies of the test suite that consumes them. */ +class TestcontainersPlugin : Plugin { + override fun apply(project: Project) { + val configurations = project.configurations + // dependencyScope express exactly what is needed it is equivalent to + // configurations.register("testContainerImage") { + // isCanBeResolved = false + // isCanBeConsumed = false + // } + // Additionally, it is lazy. + configurations.dependencyScope("testContainerImage") + configurations.named { it.endsWith("Implementation") }.all { + val imageConfigurationName = "${name.removeSuffix("Implementation")}ContainerImage" + if (imageConfigurationName !in configurations.names) { + configurations.dependencyScope(imageConfigurationName) + } + } + + // Groovy needs a bridge to call the Kotlin receiver extension with the same syntax. + project.dependencies.extensions.extraProperties.set( + "image", + object : Closure(null) { + fun doCall( + reference: String, + systemProperty: String, + ): ExternalModuleDependency = project.dependencies.image(reference, systemProperty) + }, + ) + + val resolver = + project.gradle.sharedServices.registerIfAbsent( + "testContainerImageResolver", + ImageResolver::class.java, + ) {} + + project.pluginManager.withPlugin("java") { + val sourceSets = project.extensions.getByType() + + project.tasks.withType().configureEach { + // ProviderFactory.provider snapshots project-local configuration for the configuration cache. + // Keep registry access in the nested input getter, so every build refreshes moving tags. + val containers = + project.providers.provider { + val testClasses = testClassesDirs.files + val implementationNames = + sourceSets + .filter { suite -> + suite.output.classesDirs.files + .any(testClasses::contains) + }.map { it.implementationConfigurationName } + .ifEmpty { listOf("${name}Implementation") } + val hierarchy = implementationNames.flatMap { configurations.findByName(it)?.hierarchy.orEmpty() } + val imageConfigurationNames = + setOf("${name}ContainerImage") + + hierarchy.map { "${it.name.removeSuffix("Implementation")}ContainerImage" } + + val images = linkedMapOf() + imageConfigurationNames.mapNotNull(configurations::findByName).forEach { configuration -> + configuration.allDependencies.forEach { dependency -> + val reference = (dependency as? ExternalModuleDependency)?.attributes?.getAttribute(IMAGE_REFERENCE) + if (dependency.group != IMAGE_DEPENDENCY_GROUP || reference == null) { + throw InvalidUserDataException("Use image(reference, systemProperty) for dependencies in ${configuration.name}") + } + + val previous = images.putIfAbsent(dependency.name, reference) + if (previous != null && previous != reference) { + throw InvalidUserDataException("Conflicting container images for '${dependency.name}' in $path") + } + } + } + + if (images.isNotEmpty()) { + val configurationFiles = + listOf(File(System.getProperty("user.home"), ".testcontainers.properties")) + + // Keep output directories that processResources has not created yet. + classpath.files.map { File(it, "testcontainers.properties") }.sortedBy { it.toURI().toString() } + + // Track inherited values for configuration-cache invalidation, but use the task's + // effective environment below so explicit overrides and removals are respected. + project.providers.environmentVariablesPrefixedBy("TESTCONTAINERS_").get() + val imageEnvironment = + environment + .filterKeys { + it == "TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX" || it == "TESTCONTAINERS_IMAGE_SUBSTITUTOR" || + (it.startsWith("TESTCONTAINERS_") && it.endsWith("_CONTAINER_IMAGE")) + }.mapValues { it.value.toString() } + + ContainerImageInputs(images, imageEnvironment, configurationFiles, resolver) + } else { + null + } + } + + jvmArgumentProviders.add(ContainerImageArguments(containers)) + } + } + } +} diff --git a/build-logic/testcontainers/src/test/kotlin/datadog/buildlogic/testcontainers/RegistryExtension.kt b/build-logic/testcontainers/src/test/kotlin/datadog/buildlogic/testcontainers/RegistryExtension.kt new file mode 100644 index 00000000000..26624a1a202 --- /dev/null +++ b/build-logic/testcontainers/src/test/kotlin/datadog/buildlogic/testcontainers/RegistryExtension.kt @@ -0,0 +1,79 @@ +package datadog.buildlogic.testcontainers + +import okhttp3.mockwebserver.Dispatcher +import okhttp3.mockwebserver.MockResponse +import okhttp3.mockwebserver.MockWebServer +import okhttp3.mockwebserver.RecordedRequest +import okhttp3.tls.HandshakeCertificates +import okhttp3.tls.HeldCertificate +import org.junit.jupiter.api.extension.AfterEachCallback +import org.junit.jupiter.api.extension.BeforeEachCallback +import org.junit.jupiter.api.extension.ExtensionContext +import java.net.InetAddress +import java.security.MessageDigest + +/** A local HTTPS registry with mutable manifests. */ +class RegistryExtension : + BeforeEachCallback, + AfterEachCallback { + private val server = MockWebServer() + + @Volatile var imageVersion = 1 + + @Volatile var unavailable = false + + val image: String + get() = "127.0.0.1:${server.port}/library/cassandra:4" + + val requestCount: Int + get() = server.requestCount + + val digest: String + get() = digest(manifest()) + + override fun beforeEach(context: ExtensionContext) { + val certificate = + HeldCertificate + .Builder() + .commonName("localhost") + .addSubjectAlternativeName("127.0.0.1") + .build() + val certificates = HandshakeCertificates.Builder().heldCertificate(certificate).build() + server.useHttps(certificates.sslSocketFactory(), false) + server.setDispatcher( + object : Dispatcher() { + override fun dispatch(request: RecordedRequest): MockResponse = + when { + unavailable -> { + MockResponse().setResponseCode(503) + } + + !request.path!!.startsWith("/v2/") -> { + MockResponse().setResponseCode(404) + } + + else -> { + val body = manifest() + MockResponse() + .setHeader("Content-Type", "application/vnd.oci.image.manifest.v1+json") + .setHeader("Docker-Content-Digest", digest(body)) + .setBody(body) + } + } + }, + ) + server.start(InetAddress.getByName("127.0.0.1"), 0) + } + + override fun afterEach(context: ExtensionContext) { + server.shutdown() + } + + private fun manifest() = + """{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json","config":{"mediaType":"application/vnd.oci.image.config.v1+json","digest":"${digest( + imageVersion.toString(), + )}","size":2},"layers":[]}""" + + private fun digest(body: String) = + "sha256:" + MessageDigest.getInstance("SHA-256").digest(body.toByteArray()).joinToString("") { "%02x".format(it) } +} diff --git a/build-logic/testcontainers/src/test/kotlin/datadog/buildlogic/testcontainers/TestcontainersPluginTest.kt b/build-logic/testcontainers/src/test/kotlin/datadog/buildlogic/testcontainers/TestcontainersPluginTest.kt new file mode 100644 index 00000000000..28c79cc62db --- /dev/null +++ b/build-logic/testcontainers/src/test/kotlin/datadog/buildlogic/testcontainers/TestcontainersPluginTest.kt @@ -0,0 +1,517 @@ +package datadog.buildlogic.testcontainers + +import org.assertj.core.api.Assertions.assertThat +import org.gradle.testkit.runner.GradleRunner +import org.gradle.testkit.runner.TaskOutcome +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.extension.RegisterExtension +import org.junit.jupiter.api.io.TempDir +import java.nio.file.Files +import java.nio.file.Path + +class TestcontainersPluginTest { + @TempDir + lateinit var directory: Path + + @RegisterExtension + @JvmField + val registry = RegistryExtension() + + @Test + fun `moving images are refreshed before cache lookup including configuration cache reuse`() { + fixture(registry.image) + + assertThat(run("help").task(":help")?.outcome).isEqualTo(TaskOutcome.SUCCESS) + assertThat(registry.requestCount).isZero() + + assertThat(run("test", "-PskipTests").task(":test")?.outcome).isEqualTo(TaskOutcome.SKIPPED) + assertThat(registry.requestCount).isZero() + + assertThat(run("test").task(":test")?.outcome).isEqualTo(TaskOutcome.SUCCESS) + assertThat(testReport()).content().contains("library/cassandra@${registry.digest}") + + val firstRequests = registry.requestCount + val warm = run("test") + + assertThat(warm.output).contains("Reusing configuration cache") + assertThat(warm.task(":test")?.outcome).isEqualTo(TaskOutcome.UP_TO_DATE) + assertThat(registry.requestCount).isGreaterThan(firstRequests) + + registry.imageVersion = 2 + val changed = run("test") + + assertThat(changed.output).contains("Reusing configuration cache") + assertThat(changed.task(":test")?.outcome).isEqualTo(TaskOutcome.SUCCESS) + assertThat(testReport()).content().contains("library/cassandra@${registry.digest}") + + run("clean") + val restored = run("test") + + assertThat(restored.output).contains("Reusing configuration cache") + assertThat(restored.task(":test")?.outcome).isEqualTo(TaskOutcome.FROM_CACHE) + assertThat(testReport()).content().contains("library/cassandra@${registry.digest}") + + val suites = run("forkedTest", "latestDepTest", "latestDepTestForkedTest", "latestDepForkedTest", "replayedTests") + + assertThat(suites.task(":replayedTests")?.outcome) + .isIn(TaskOutcome.SUCCESS, TaskOutcome.FROM_CACHE) + assertThat(testReport("forkedTest", "ImageForkedTest")) + .content() + .contains("library/cassandra@${registry.digest}") + assertThat(testReport("forkedTest", "ImageTest")).doesNotExist() + assertThat(testReport(testClass = "ImageForkedTest")).doesNotExist() + assertThat(testReport("latestDepTest")) + .content() + .contains("library/cassandra@${registry.digest}") + assertThat(testReport("latestDepTestForkedTest", "ImageForkedTest")) + .content() + .contains("library/cassandra@${registry.digest}") + assertThat(testReport("replayedTests")) + .content() + .contains("library/cassandra@${registry.digest}") + assertThat(testReport("latestDepForkedTest", "ImageForkedTest")) + .content() + .contains("library/cassandra@${registry.digest}") + + val reusedSuites = run("forkedTest", "latestDepTest", "latestDepTestForkedTest", "latestDepForkedTest", "replayedTests") + + assertThat(reusedSuites.output).contains("Reusing configuration cache") + assertThat(reusedSuites.task(":forkedTest")?.outcome).isEqualTo(TaskOutcome.UP_TO_DATE) + assertThat(reusedSuites.task(":latestDepTestForkedTest")?.outcome).isEqualTo(TaskOutcome.UP_TO_DATE) + assertThat(reusedSuites.task(":replayedTests")?.outcome).isEqualTo(TaskOutcome.UP_TO_DATE) + + assertThat(run("declaredTest").task(":declaredTest")?.outcome).isIn(TaskOutcome.SUCCESS, TaskOutcome.FROM_CACHE) + assertThat(testReport("declaredTest")) + .content() + .contains("library/cassandra@${registry.digest}") + + val requestsBeforeUnrelatedSuite = registry.requestCount + assertThat(run("isolatedTest").task(":isolatedTest")?.outcome).isEqualTo(TaskOutcome.SUCCESS) + assertThat(run("testForkedTest").task(":testForkedTest")?.outcome).isIn(TaskOutcome.SUCCESS, TaskOutcome.FROM_CACHE) + assertThat(run("emptyTest").task(":emptyTest")?.outcome).isEqualTo(TaskOutcome.NO_SOURCE) + assertThat(registry.requestCount).isEqualTo(requestsBeforeUnrelatedSuite) + + registry.unavailable = true + assertThat(runner("test").buildAndFail().output) + .contains("Cannot resolve test container image") + } + + @Test + fun `image configurations support shared parents and tests without matching source sets`() { + val image = "cassandra@sha256:${"a".repeat(64)}" + val extraImage = "redis@sha256:${"b".repeat(64)}" + fixture(image) + directory.resolve("build.gradle.kts").toFile().appendText( + """ + + val databaseImages = configurations.dependencyScope("databaseImages") + val sharedImplementation = configurations.dependencyScope("sharedImplementation") + + configurations.named("sharedContainerImage") { + extendsFrom(databaseImages.get()) + } + configurations.testContainerImage { dependencies.clear() } + configurations.testImplementation { extendsFrom(sharedImplementation.get()) } + configurations.dependencyScope("databaseCheckContainerImage") { + extendsFrom(databaseImages.get()) + } + + dependencies { + add(databaseImages.name, image("$image", "test.cassandra.image")) + add("databaseCheckContainerImage", image("$extraImage", "test.redis.image")) + } + + tasks.register("databaseCheck") { + testClassesDirs = sourceSets.test.get().output.classesDirs + classpath = sourceSets.test.get().runtimeClasspath + } + """.trimIndent(), + ) + + val first = run("test", "databaseCheck") + + assertThat(first.task(":test")?.outcome).isEqualTo(TaskOutcome.SUCCESS) + assertThat(first.task(":databaseCheck")?.outcome).isIn(TaskOutcome.SUCCESS, TaskOutcome.FROM_CACHE) + assertThat(testReport()).content().contains("registry-1.docker.io/library/$image") + assertThat(testReport("databaseCheck")) + .content() + .contains("registry-1.docker.io/library/$image") + .contains("registry-1.docker.io/library/$extraImage") + + val reused = run("test", "databaseCheck") + + assertThat(reused.output).contains("Reusing configuration cache") + assertThat(reused.task(":test")?.outcome).isEqualTo(TaskOutcome.UP_TO_DATE) + assertThat(reused.task(":databaseCheck")?.outcome).isEqualTo(TaskOutcome.UP_TO_DATE) + assertThat(registry.requestCount).isZero() + + run("databaseCheck") + Files.createDirectories(directory.resolve("src/test/resources")) + directory + .resolve("src/test/resources/testcontainers.properties") + .toFile() + .writeText("image.substitutor=untracked.CustomSubstitutor\n") + + val failed = runner("databaseCheck").buildAndFail() + + assertThat(failed.output) + .contains("Reusing configuration cache") + .contains("org.gradle.api.InvalidUserDataException: Custom Testcontainers image substitutions") + .contains("move image overrides into testContainerImage declarations") + } + + @Test + fun `tasks running multiple source sets reject conflicting inherited images`() { + fixture("cassandra@sha256:${"a".repeat(64)}") + directory.resolve("build.gradle.kts").toFile().appendText( + """ + + configurations.named("declaredTestContainerImage") { dependencies.clear() } + dependencies { + add("declaredTestContainerImage", image("cassandra@sha256:${"b".repeat(64)}", "test.cassandra.image")) + } + + tasks.register("combinedTests") { + testClassesDirs = files(sourceSets.test.get().output.classesDirs, sourceSets["declaredTest"].output.classesDirs) + classpath = sourceSets.test.get().runtimeClasspath + sourceSets["declaredTest"].runtimeClasspath + } + """.trimIndent(), + ) + + assertThat(runner("combinedTests").buildAndFail().output) + .contains("org.gradle.api.InvalidUserDataException: Conflicting container images for 'test.cassandra.image'") + assertThat(registry.requestCount).isZero() + } + + @Test + fun `different image attributes for the same property are rejected`() { + fixture("cassandra@sha256:${"a".repeat(64)}") + directory.resolve("build.gradle.kts").toFile().appendText( + """ + + dependencies { + testContainerImage(image("cassandra@sha256:${"b".repeat(64)}", "test.cassandra.image")) + } + """.trimIndent(), + ) + + assertThat(runner("test").buildAndFail().output) + .contains("org.gradle.api.InvalidUserDataException: Conflicting container images for 'test.cassandra.image'") + assertThat(registry.requestCount).isZero() + } + + @Test + fun `hub prefix is resolved and pinned references require no registry`() { + val digest = "sha256:${"a".repeat(64)}" + fixture("cassandra@$digest") + + val result = + runner("test") + .withEnvironment( + System.getenv() + + ("TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX" to "mirror.example/team/"), + ).build() + + assertThat(result.task(":test")?.outcome).isEqualTo(TaskOutcome.SUCCESS) + assertThat(testReport()).content().contains("mirror.example/team/cassandra@$digest") + + val changed = + runner("test") + .withEnvironment( + System.getenv() + + ("TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX" to "another.example/team/"), + ).build() + + assertThat(changed.task(":test")?.outcome).isEqualTo(TaskOutcome.SUCCESS) + assertThat(testReport()).content().contains("another.example/team/cassandra@$digest") + + directory.resolve("build.gradle.kts").toFile().appendText( + """ + + sourceSets.test { resources.setSrcDirs(listOf("lateResources")) } + tasks.named("test") { + environment("TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX", "task.example/") + } + """.trimIndent(), + ) + + assertThat(run("test").task(":test")?.outcome).isEqualTo(TaskOutcome.SUCCESS) + assertThat(testReport()).content().contains("task.example/cassandra@$digest") + + Files.createDirectories(directory.resolve("lateResources")) + directory + .resolve("lateResources/testcontainers.properties") + .toFile() + .writeText("image.substitutor=untracked.CustomSubstitutor\n") + + assertThat(runner("test").buildAndFail().output) + .contains("move image overrides into testContainerImage declarations") + } + + @Test + fun `task environment removals override inherited Testcontainers settings`() { + val digest = "sha256:${"a".repeat(64)}" + fixture("cassandra@$digest") + directory.resolve("build.gradle.kts").toFile().appendText( + """ + + tasks.named("test") { + environment.remove("TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX") + environment.remove("TESTCONTAINERS_IMAGE_SUBSTITUTOR") + } + """.trimIndent(), + ) + val environment = + System.getenv() + + mapOf( + "TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX" to "removed.example/team/", + "TESTCONTAINERS_IMAGE_SUBSTITUTOR" to "removed.CustomSubstitutor", + ) + + val first = runner("test").withEnvironment(environment).build() + + assertThat(first.task(":test")?.outcome).isEqualTo(TaskOutcome.SUCCESS) + assertThat(testReport()).content().contains("registry-1.docker.io/library/cassandra@$digest") + + val reused = runner("test").withEnvironment(environment).build() + + assertThat(reused.output).contains("Reusing configuration cache") + assertThat(reused.task(":test")?.outcome).isEqualTo(TaskOutcome.UP_TO_DATE) + } + + @Test + fun `one can write Groovy DSL dependency`() { + val image = "cassandra@sha256:${"a".repeat(64)}" + fixture(image) + Files.delete(directory.resolve("build.gradle.kts")) + directory.resolve("build.gradle").toFile().writeText( + """ + plugins { + id 'java' + id 'dd-trace-java.testcontainers' + } + + sourceSets { + integrationTest { java.srcDirs = sourceSets.test.java.srcDirs } + } + + dependencies { + testImplementation files(${junitClasspath()}) + integrationTestImplementation files(${junitClasspath()}) + testContainerImage(image('$image', 'test.cassandra.image')) + integrationTestContainerImage(image('$image', 'test.cassandra.image')) + } + + tasks.register('integrationTest', Test) { + testClassesDirs = sourceSets.integrationTest.output.classesDirs + classpath = sourceSets.integrationTest.runtimeClasspath + } + tasks.withType(Test).configureEach { useJUnitPlatform() } + """.trimIndent(), + ) + + val result = run("test", "integrationTest") + + assertThat(result.task(":test")?.outcome).isIn(TaskOutcome.SUCCESS, TaskOutcome.FROM_CACHE) + assertThat(result.task(":integrationTest")?.outcome).isIn(TaskOutcome.SUCCESS, TaskOutcome.FROM_CACHE) + assertThat(testReport()).content().contains("registry-1.docker.io/library/$image") + assertThat(testReport("integrationTest")) + .content() + .contains("registry-1.docker.io/library/$image") + + val reused = run("test", "integrationTest") + + assertThat(reused.output).contains("Reusing configuration cache") + assertThat(reused.task(":test")?.outcome).isEqualTo(TaskOutcome.UP_TO_DATE) + assertThat(reused.task(":integrationTest")?.outcome).isEqualTo(TaskOutcome.UP_TO_DATE) + } + + private fun fixture(image: String) { + directory.resolve("settings.gradle.kts").toFile().writeText( + """ + rootProject.name = "container-image-fixture" + buildCache { local { directory = file("cache") } } + """.trimIndent(), + ) + directory.resolve("build.gradle.kts").toFile().writeText( + """ + import datadog.buildlogic.testcontainers.image + + plugins { + java + id("dd-trace-java.testcontainers") + } + + val latestDepTest = sourceSets.create("latestDepTest") { + java.setSrcDirs(sourceSets.test.get().java.srcDirs) + } + val latestDepForkedTest = sourceSets.create("latestDepForkedTest") { + java.setSrcDirs(sourceSets.test.get().java.srcDirs) + } + val declaredTest = sourceSets.create("declaredTest") { + java.setSrcDirs(sourceSets.test.get().java.srcDirs) + } + val isolatedTest = sourceSets.create("isolatedTest") + val emptyTest = sourceSets.create("emptyTest") + + tasks.register("forkedTest") { + testClassesDirs = sourceSets.test.get().output.classesDirs + classpath = sourceSets.test.get().runtimeClasspath + } + + tasks.register("latestDepTest") { + testClassesDirs = latestDepTest.output.classesDirs + classpath = latestDepTest.runtimeClasspath + } + + tasks.register("latestDepTestForkedTest") { + testClassesDirs = latestDepTest.output.classesDirs + classpath = latestDepTest.runtimeClasspath + } + + tasks.register("replayedTests") { + testClassesDirs = latestDepTest.output.classesDirs + classpath = latestDepTest.runtimeClasspath + } + + tasks.register("latestDepForkedTest") { + testClassesDirs = latestDepForkedTest.output.classesDirs + classpath = latestDepForkedTest.runtimeClasspath + } + + tasks.register("declaredTest") { + testClassesDirs = declaredTest.output.classesDirs + classpath = declaredTest.runtimeClasspath + } + + tasks.register("isolatedTest") { + testClassesDirs = isolatedTest.output.classesDirs + classpath = isolatedTest.runtimeClasspath + } + + tasks.register("testForkedTest") { + testClassesDirs = isolatedTest.output.classesDirs + classpath = isolatedTest.runtimeClasspath + } + + tasks.register("emptyTest") { + testClassesDirs = emptyTest.output.classesDirs + classpath = emptyTest.runtimeClasspath + } + + tasks.withType().configureEach { + useJUnitPlatform() + val skip = providers.gradleProperty("skipTests") + onlyIf { !skip.isPresent } + + // Match dd-trace-java.configure-tests: split regular and forked test classes. + if (name.startsWith("forkedTest") || name.endsWith("ForkedTest")) { + setExcludes(emptyList()) + setIncludes(listOf("**/*ForkedTest*")) + forkEvery = 1 + } else { + exclude("**/*ForkedTest*") + failOnNoDiscoveredTests = false + } + } + + // Plugins must see declarations and inheritance added after tasks are realized. + tasks.named("test").get() + tasks.named("latestDepTest").get() + tasks.named("latestDepTestForkedTest").get() + tasks.named("replayedTests").get() + configurations.named(latestDepTest.implementationConfigurationName) { + extendsFrom(configurations.testImplementation.get()) + } + configurations.named(latestDepForkedTest.implementationConfigurationName) { + extendsFrom(configurations.getByName(latestDepTest.implementationConfigurationName)) + } + configurations.named(emptyTest.implementationConfigurationName) { + extendsFrom(configurations.testImplementation.get()) + } + + dependencies { + testImplementation(files(${junitClasspath()})) + add(declaredTest.implementationConfigurationName, files(${junitClasspath()})) + add(isolatedTest.implementationConfigurationName, files(${junitClasspath()})) + testContainerImage(image("$image", "test.cassandra.image")) + add("declaredTestContainerImage", image("$image", "test.cassandra.image")) + } + """.trimIndent(), + ) + + Files.createDirectories(directory.resolve("src/test/java")) + directory.resolve("src/test/java/ImageTest.java").toFile().writeText( + """ + import org.junit.jupiter.api.Test; + import static org.junit.jupiter.api.Assertions.assertTrue; + + public class ImageTest { + @Test public void imageIsPinned() { + String image = System.getProperty("test.cassandra.image"); + assertTrue(image.matches(".+@sha256:[a-f0-9]{64}"), image); + System.out.println(image); + System.out.println(System.getProperty("test.redis.image", "")); + } + } + """.trimIndent(), + ) + directory.resolve("src/test/java/ImageForkedTest.java").toFile().writeText( + "public class ImageForkedTest extends ImageTest {}\n", + ) + + Files.createDirectories(directory.resolve("src/isolatedTest/java")) + directory.resolve("src/isolatedTest/java/PlainTest.java").toFile().writeText( + """ + import org.junit.jupiter.api.Test; + import static org.junit.jupiter.api.Assertions.assertNull; + + public class PlainTest { + @Test public void hasNoContainerDependency() { + assertNull(System.getProperty("test.cassandra.image")); + } + } + """.trimIndent(), + ) + directory.resolve("src/isolatedTest/java/PlainForkedTest.java").toFile().writeText( + "public class PlainForkedTest extends PlainTest {}\n", + ) + } + + private fun junitClasspath() = + listOf( + "org.junit.jupiter.api.Test", + "org.junit.jupiter.engine.JupiterTestEngine", + "org.junit.platform.engine.TestEngine", + "org.junit.platform.launcher.Launcher", + "org.junit.platform.commons.JUnitException", + "org.opentest4j.AssertionFailedError", + ).joinToString(", ") { + "\"${Path.of( + Class + .forName(it) + .protectionDomain.codeSource.location + .toURI(), + )}\"" + } + + private fun runner(vararg arguments: String) = GradleRunner + .create() + .withProjectDir(directory.toFile()) + .withPluginClasspath() + .withArguments( + *arguments, + "--build-cache", + "--configuration-cache", + "--stacktrace", + "--max-workers=2", + "-Dorg.gradle.jvmargs=-Xmx512m", + ) + + private fun run(vararg arguments: String) = runner(*arguments).build() + + private fun testReport( + task: String = "test", + testClass: String = "ImageTest", + ) = directory.resolve("build/test-results/$task/TEST-$testClass.xml").toFile() +} diff --git a/buildSrc/src/test/resources/repository-proxy.init.gradle.kts b/buildSrc/src/test/resources/repository-proxy.init.gradle.kts index f4b18536966..08c3e8ef446 100644 --- a/buildSrc/src/test/resources/repository-proxy.init.gradle.kts +++ b/buildSrc/src/test/resources/repository-proxy.init.gradle.kts @@ -5,6 +5,7 @@ import org.gradle.api.artifacts.repositories.MavenArtifactRepository import org.gradle.api.initialization.Settings import java.net.URI +// Requires Gradle 6.8+ for Settings.dependencyResolutionManagement. // Routes the public repositories declared by TestKit builds (see GradleFixture) through the // mirrors CI configures, so buildSrc tests do not hit Maven Central and get rate limited. // diff --git a/dd-java-agent/instrumentation/aerospike-4.0/build.gradle b/dd-java-agent/instrumentation/aerospike-4.0/build.gradle index b67da684832..87de14495cd 100644 --- a/dd-java-agent/instrumentation/aerospike-4.0/build.gradle +++ b/dd-java-agent/instrumentation/aerospike-4.0/build.gradle @@ -1,5 +1,6 @@ plugins { id 'dd-trace-java.module.instrumentation' + id 'dd-trace-java.testcontainers' } muzzle { @@ -30,6 +31,7 @@ tasks.named("latestDepForkedTest", Test) { dependencies { compileOnly group: 'com.aerospike', name: 'aerospike-client', version: '4.0.0' + testContainerImage(image('aerospike:ce-6.2.0.2', 'test.aerospike.image')) testImplementation group: 'com.aerospike', name: 'aerospike-client', version: '4.0.0' testImplementation libs.testcontainers diff --git a/dd-java-agent/instrumentation/aerospike-4.0/src/test/groovy/datadog/trace/instrumentation/aerospike4/AerospikeBaseTest.groovy b/dd-java-agent/instrumentation/aerospike-4.0/src/test/groovy/datadog/trace/instrumentation/aerospike4/AerospikeBaseTest.groovy index 746f57577c8..274da89bc4b 100644 --- a/dd-java-agent/instrumentation/aerospike-4.0/src/test/groovy/datadog/trace/instrumentation/aerospike4/AerospikeBaseTest.groovy +++ b/dd-java-agent/instrumentation/aerospike-4.0/src/test/groovy/datadog/trace/instrumentation/aerospike4/AerospikeBaseTest.groovy @@ -11,6 +11,7 @@ import datadog.trace.api.DDSpanTypes import datadog.trace.bootstrap.instrumentation.api.Tags import datadog.trace.core.DDSpan import org.testcontainers.containers.GenericContainer +import org.testcontainers.utility.DockerImageName import spock.lang.Shared abstract class AerospikeBaseTest extends VersionedNamingTestBase { @@ -26,7 +27,7 @@ abstract class AerospikeBaseTest extends VersionedNamingTestBase { def setup() throws Exception { // Linux arm64 supported since `ce-6.2.0.2` - aerospike = new GenericContainer('aerospike:ce-6.2.0.2') + aerospike = new GenericContainer(DockerImageName.parse(System.getProperty('test.aerospike.image'))) .withExposedPorts(3000) // proto-fd-max default is 15000, but container default is 1024. // see: https://aerospike.com/docs/database/reference/config#service__proto-fd-max diff --git a/dd-java-agent/instrumentation/aws-java/aws-java-dynamodb-2.0/build.gradle b/dd-java-agent/instrumentation/aws-java/aws-java-dynamodb-2.0/build.gradle index 7e419d43131..7ff7c472ef1 100644 --- a/dd-java-agent/instrumentation/aws-java/aws-java-dynamodb-2.0/build.gradle +++ b/dd-java-agent/instrumentation/aws-java/aws-java-dynamodb-2.0/build.gradle @@ -1,5 +1,6 @@ plugins { id 'dd-trace-java.module.instrumentation' + id 'dd-trace-java.testcontainers' } muzzle { @@ -16,6 +17,7 @@ addTestSuiteExtendingForDir('latestDepForkedTest', 'latestDepTest', 'test') dependencies { compileOnly group: 'software.amazon.awssdk', name: 'dynamodb', version: '2.30.22' + testContainerImage(image('localstack/localstack:4.2.0', 'test.localstack.image')) // Include httpclient instrumentation for testing because it is a dependency for aws-sdk. testImplementation project(':dd-java-agent:instrumentation:apache-httpclient:apache-httpclient-4.0') diff --git a/dd-java-agent/instrumentation/aws-java/aws-java-dynamodb-2.0/src/test/groovy/DynamoDbClientTest.groovy b/dd-java-agent/instrumentation/aws-java/aws-java-dynamodb-2.0/src/test/groovy/DynamoDbClientTest.groovy index 02acd67be37..60ee72c6e0f 100644 --- a/dd-java-agent/instrumentation/aws-java/aws-java-dynamodb-2.0/src/test/groovy/DynamoDbClientTest.groovy +++ b/dd-java-agent/instrumentation/aws-java/aws-java-dynamodb-2.0/src/test/groovy/DynamoDbClientTest.groovy @@ -29,7 +29,7 @@ import spock.lang.Shared import java.time.Duration class DynamoDbClientTest extends InstrumentationSpecification { - static final LOCALSTACK = new GenericContainer(DockerImageName.parse("localstack/localstack:4.2.0")) + static final LOCALSTACK = new GenericContainer(DockerImageName.parse(System.getProperty("test.localstack.image"))) .withExposedPorts(4566) .withEnv("SERVICES", "dynamodb") .withReuse(true) diff --git a/dd-java-agent/instrumentation/aws-java/aws-java-eventbridge-2.0/build.gradle b/dd-java-agent/instrumentation/aws-java/aws-java-eventbridge-2.0/build.gradle index f21a227f1df..217dd6dc439 100644 --- a/dd-java-agent/instrumentation/aws-java/aws-java-eventbridge-2.0/build.gradle +++ b/dd-java-agent/instrumentation/aws-java/aws-java-eventbridge-2.0/build.gradle @@ -1,5 +1,6 @@ plugins { id 'dd-trace-java.module.instrumentation' + id 'dd-trace-java.testcontainers' } muzzle { @@ -16,6 +17,7 @@ addTestSuiteExtendingForDir('latestDepForkedTest', 'latestDepTest', 'test') dependencies { compileOnly group: 'software.amazon.awssdk', name: 'eventbridge', version: '2.27.19' + testContainerImage(image('localstack/localstack:4.2.0', 'test.localstack.image')) // Include httpclient instrumentation for testing because it is a dependency for aws-sdk. testImplementation project(':dd-java-agent:instrumentation:apache-httpclient:apache-httpclient-4.0') diff --git a/dd-java-agent/instrumentation/aws-java/aws-java-eventbridge-2.0/src/test/groovy/EventBridgeClientTest.groovy b/dd-java-agent/instrumentation/aws-java/aws-java-eventbridge-2.0/src/test/groovy/EventBridgeClientTest.groovy index 3362d3f4b97..19d158d3adf 100644 --- a/dd-java-agent/instrumentation/aws-java/aws-java-eventbridge-2.0/src/test/groovy/EventBridgeClientTest.groovy +++ b/dd-java-agent/instrumentation/aws-java/aws-java-eventbridge-2.0/src/test/groovy/EventBridgeClientTest.groovy @@ -20,7 +20,7 @@ import software.amazon.awssdk.services.sqs.model.QueueAttributeName import spock.lang.Shared class EventBridgeClientTest extends InstrumentationSpecification { - static final LOCALSTACK = new GenericContainer(DockerImageName.parse("localstack/localstack:4.2.0")) + static final LOCALSTACK = new GenericContainer(DockerImageName.parse(System.getProperty("test.localstack.image"))) .withExposedPorts(4566) .withEnv("SERVICES", "sns,sqs,events") .withReuse(true) diff --git a/dd-java-agent/instrumentation/aws-java/aws-java-s3-2.0/build.gradle b/dd-java-agent/instrumentation/aws-java/aws-java-s3-2.0/build.gradle index de35825b2e5..bfe17e9d31b 100644 --- a/dd-java-agent/instrumentation/aws-java/aws-java-s3-2.0/build.gradle +++ b/dd-java-agent/instrumentation/aws-java/aws-java-s3-2.0/build.gradle @@ -1,5 +1,6 @@ plugins { id 'dd-trace-java.module.instrumentation' + id 'dd-trace-java.testcontainers' } muzzle { @@ -16,6 +17,7 @@ addTestSuiteExtendingForDir('latestDepForkedTest', 'latestDepTest', 'test') dependencies { compileOnly group: 'software.amazon.awssdk', name: 's3', version: '2.29.26' + testContainerImage(image('localstack/localstack:4.2.0', 'test.localstack.image')) // Include httpclient instrumentation for testing because it is a dependency for aws-sdk. testRuntimeOnly project(':dd-java-agent:instrumentation:apache-httpclient:apache-httpclient-4.0') diff --git a/dd-java-agent/instrumentation/aws-java/aws-java-s3-2.0/src/test/groovy/S3ClientTest.groovy b/dd-java-agent/instrumentation/aws-java/aws-java-s3-2.0/src/test/groovy/S3ClientTest.groovy index c2071fb7858..76dc369ed01 100644 --- a/dd-java-agent/instrumentation/aws-java/aws-java-s3-2.0/src/test/groovy/S3ClientTest.groovy +++ b/dd-java-agent/instrumentation/aws-java/aws-java-s3-2.0/src/test/groovy/S3ClientTest.groovy @@ -18,7 +18,7 @@ import spock.lang.Shared import java.time.Duration class S3ClientTest extends InstrumentationSpecification { - static final LOCALSTACK = new GenericContainer(DockerImageName.parse("localstack/localstack:4.2.0")) + static final LOCALSTACK = new GenericContainer(DockerImageName.parse(System.getProperty("test.localstack.image"))) .withExposedPorts(4566) .withEnv("SERVICES", "s3") .withReuse(true) diff --git a/dd-java-agent/instrumentation/aws-java/aws-java-sdk-2.2/build.gradle b/dd-java-agent/instrumentation/aws-java/aws-java-sdk-2.2/build.gradle index 7945dca62b9..cfdc266d8ec 100644 --- a/dd-java-agent/instrumentation/aws-java/aws-java-sdk-2.2/build.gradle +++ b/dd-java-agent/instrumentation/aws-java/aws-java-sdk-2.2/build.gradle @@ -1,5 +1,6 @@ plugins { id 'dd-trace-java.module.instrumentation' + id 'dd-trace-java.testcontainers' } muzzle { @@ -26,10 +27,15 @@ addForkedTestTask('payloadTaggingTest') addTestSuiteForDir('latestPayloadTaggingTest', 'payloadTaggingTest') addTestSuiteExtendingForDir('latestPayloadTaggingForkedTest', 'latestPayloadTaggingTest', 'payloadTaggingTest') +configurations.named('latestPayloadTaggingTestContainerImage') { + extendsFrom(configurations.named('payloadTaggingTestContainerImage').get()) +} + def fixedSdkVersion = '2.20.33' // 2.20.34 is missing and breaks IDEA import dependencies { compileOnly group: 'software.amazon.awssdk', name: 'aws-core', version: '2.2.0' + payloadTaggingTestContainerImage(image('localstack/localstack:4.2.0', 'test.localstack.image')) testImplementation project(':dd-java-agent:instrumentation:aws-java:aws-java-common') // Include httpclient instrumentation for testing because it is a dependency for aws-sdk. diff --git a/dd-java-agent/instrumentation/aws-java/aws-java-sdk-2.2/src/payloadTaggingTest/groovy/PayloadTaggingTest.groovy b/dd-java-agent/instrumentation/aws-java/aws-java-sdk-2.2/src/payloadTaggingTest/groovy/PayloadTaggingTest.groovy index 2f8a4ee613e..ea4719db72d 100644 --- a/dd-java-agent/instrumentation/aws-java/aws-java-sdk-2.2/src/payloadTaggingTest/groovy/PayloadTaggingTest.groovy +++ b/dd-java-agent/instrumentation/aws-java/aws-java-sdk-2.2/src/payloadTaggingTest/groovy/PayloadTaggingTest.groovy @@ -26,7 +26,7 @@ abstract class AbstractPayloadTaggingTest extends InstrumentationSpecification { static final Object NA = {} static final int DEFAULT_PORT = 4566 - static final LOCALSTACK = new GenericContainer(DockerImageName.parse("localstack/localstack:4.2.0")) + static final LOCALSTACK = new GenericContainer(DockerImageName.parse(System.getProperty("test.localstack.image"))) .withExposedPorts(DEFAULT_PORT) .withEnv("SERVICES", "apigateway,events,s3,sns,sqs,kinesis") .withReuse(true) @@ -364,4 +364,3 @@ class PayloadTaggingMaxTagsForkedTest extends AbstractPayloadTaggingTest { ] } } - diff --git a/dd-java-agent/instrumentation/aws-java/aws-java-sfn-2.0/build.gradle b/dd-java-agent/instrumentation/aws-java/aws-java-sfn-2.0/build.gradle index b9307d3a0a9..1d2dd96574e 100644 --- a/dd-java-agent/instrumentation/aws-java/aws-java-sfn-2.0/build.gradle +++ b/dd-java-agent/instrumentation/aws-java/aws-java-sfn-2.0/build.gradle @@ -1,5 +1,6 @@ plugins { id 'dd-trace-java.module.instrumentation' + id 'dd-trace-java.testcontainers' } muzzle { @@ -17,6 +18,7 @@ addTestSuiteExtendingForDir('latestDepForkedTest', 'latestDepTest', 'test') dependencies { compileOnly group: 'software.amazon.awssdk', name: 'sfn', version: '2.15.35' + testContainerImage(image('localstack/localstack:4.2.0', 'test.localstack.image')) // Include httpclient instrumentation for testing because it is a dependency for aws-sdk. testImplementation project(':dd-java-agent:instrumentation:apache-httpclient:apache-httpclient-4.0') diff --git a/dd-java-agent/instrumentation/aws-java/aws-java-sfn-2.0/src/test/groovy/SfnClientTest.groovy b/dd-java-agent/instrumentation/aws-java/aws-java-sfn-2.0/src/test/groovy/SfnClientTest.groovy index 18b2c36ee06..74f9c4e183c 100644 --- a/dd-java-agent/instrumentation/aws-java/aws-java-sfn-2.0/src/test/groovy/SfnClientTest.groovy +++ b/dd-java-agent/instrumentation/aws-java/aws-java-sfn-2.0/src/test/groovy/SfnClientTest.groovy @@ -24,7 +24,7 @@ abstract class SfnClientTest extends VersionedNamingTestBase { @Shared Object endPoint def setupSpec() { - localStack = new GenericContainer(DockerImageName.parse("localstack/localstack:4.2.0")) + localStack = new GenericContainer(DockerImageName.parse(System.getProperty("test.localstack.image"))) .withExposedPorts(4566) .withEnv("SERVICES", "stepfunctions") .withReuse(true) diff --git a/dd-java-agent/instrumentation/aws-java/aws-java-sns-1.0/build.gradle b/dd-java-agent/instrumentation/aws-java/aws-java-sns-1.0/build.gradle index 93f5d7abd56..d39b71702d8 100644 --- a/dd-java-agent/instrumentation/aws-java/aws-java-sns-1.0/build.gradle +++ b/dd-java-agent/instrumentation/aws-java/aws-java-sns-1.0/build.gradle @@ -1,5 +1,6 @@ plugins { id 'dd-trace-java.module.instrumentation' + id 'dd-trace-java.testcontainers' } muzzle { @@ -17,6 +18,7 @@ addTestSuiteExtendingForDir('latestDepForkedTest', 'latestDepTest', 'test') dependencies { compileOnly group: 'com.amazonaws', name: 'aws-java-sdk-sns', version: '1.12.710' + testContainerImage(image('localstack/localstack:4.2.0', 'test.localstack.image')) testImplementation project(':dd-java-agent:instrumentation:aws-java:aws-java-common') // Include httpclient instrumentation for testing because it is a dependency for aws-sdk. diff --git a/dd-java-agent/instrumentation/aws-java/aws-java-sns-1.0/src/test/groovy/SnsClientTest.groovy b/dd-java-agent/instrumentation/aws-java/aws-java-sns-1.0/src/test/groovy/SnsClientTest.groovy index 4aa1e6e2ddc..a4154bc543d 100644 --- a/dd-java-agent/instrumentation/aws-java/aws-java-sns-1.0/src/test/groovy/SnsClientTest.groovy +++ b/dd-java-agent/instrumentation/aws-java/aws-java-sns-1.0/src/test/groovy/SnsClientTest.groovy @@ -25,7 +25,7 @@ import spock.lang.Shared abstract class SnsClientTest extends VersionedNamingTestBase { - static final LOCALSTACK = new GenericContainer(DockerImageName.parse("localstack/localstack:4.2.0")) + static final LOCALSTACK = new GenericContainer(DockerImageName.parse(System.getProperty("test.localstack.image"))) .withExposedPorts(4566) // Default LocalStack port .withEnv("SERVICES", "sns,sqs") // Enable SNS and SQS service .withReuse(true) diff --git a/dd-java-agent/instrumentation/aws-java/aws-java-sns-2.0/build.gradle b/dd-java-agent/instrumentation/aws-java/aws-java-sns-2.0/build.gradle index 63610c31c8c..ad24d8fbefe 100644 --- a/dd-java-agent/instrumentation/aws-java/aws-java-sns-2.0/build.gradle +++ b/dd-java-agent/instrumentation/aws-java/aws-java-sns-2.0/build.gradle @@ -1,5 +1,6 @@ plugins { id 'dd-trace-java.module.instrumentation' + id 'dd-trace-java.testcontainers' } muzzle { @@ -16,6 +17,7 @@ addTestSuiteExtendingForDir('latestDepForkedTest', 'latestDepTest', 'test') dependencies { compileOnly group: 'software.amazon.awssdk', name: 'sns', version: '2.25.40' + testContainerImage(image('localstack/localstack:4.2.0', 'test.localstack.image')) testImplementation project(':dd-java-agent:instrumentation:aws-java:aws-java-common') // Include httpclient instrumentation for testing because it is a dependency for aws-sdk. diff --git a/dd-java-agent/instrumentation/aws-java/aws-java-sns-2.0/src/test/groovy/SnsClientTest.groovy b/dd-java-agent/instrumentation/aws-java/aws-java-sns-2.0/src/test/groovy/SnsClientTest.groovy index d340515d7a1..e6c6930f681 100644 --- a/dd-java-agent/instrumentation/aws-java/aws-java-sns-2.0/src/test/groovy/SnsClientTest.groovy +++ b/dd-java-agent/instrumentation/aws-java/aws-java-sns-2.0/src/test/groovy/SnsClientTest.groovy @@ -23,7 +23,7 @@ import java.time.Duration import static datadog.trace.agent.test.utils.TraceUtils.basicSpan abstract class SnsClientTest extends VersionedNamingTestBase { - static final LOCALSTACK = new GenericContainer(DockerImageName.parse("localstack/localstack:4.2.0")) + static final LOCALSTACK = new GenericContainer(DockerImageName.parse(System.getProperty("test.localstack.image"))) .withExposedPorts(4566) // Default LocalStack port .withEnv("SERVICES", "sns,sqs") // Enable SNS and SQS service .withReuse(true) diff --git a/dd-java-agent/instrumentation/couchbase/couchbase-3.1/build.gradle b/dd-java-agent/instrumentation/couchbase/couchbase-3.1/build.gradle index 9be16049b71..36543313dfe 100644 --- a/dd-java-agent/instrumentation/couchbase/couchbase-3.1/build.gradle +++ b/dd-java-agent/instrumentation/couchbase/couchbase-3.1/build.gradle @@ -1,5 +1,8 @@ +import org.gradle.nativeplatform.platform.internal.DefaultNativePlatform + plugins { id 'dd-trace-java.module.instrumentation' + id 'dd-trace-java.testcontainers' } muzzle { @@ -17,6 +20,11 @@ addTestSuiteForDir('latestDepTest', 'test') dependencies { compileOnly group: 'com.couchbase.client', name: 'java-client', version: '3.1.0' + testContainerImage(image( + DefaultNativePlatform.getCurrentArchitecture().isArm64() + ? 'couchbase/server:7.1.0-aarch64' + : 'couchbase/server:7.1.0', + 'test.couchbase.image')) testImplementation group: 'com.couchbase.client', name: 'java-client', version: '3.1.0' testImplementation group: 'org.testcontainers', name: 'couchbase', version: libs.versions.testcontainers.get() diff --git a/dd-java-agent/instrumentation/couchbase/couchbase-3.1/src/test/groovy/CouchbaseClient31Test.groovy b/dd-java-agent/instrumentation/couchbase/couchbase-3.1/src/test/groovy/CouchbaseClient31Test.groovy index e37cda84896..fc531250e73 100644 --- a/dd-java-agent/instrumentation/couchbase/couchbase-3.1/src/test/groovy/CouchbaseClient31Test.groovy +++ b/dd-java-agent/instrumentation/couchbase/couchbase-3.1/src/test/groovy/CouchbaseClient31Test.groovy @@ -21,6 +21,7 @@ import org.slf4j.LoggerFactory import java.time.Duration import org.testcontainers.couchbase.BucketDefinition import org.testcontainers.couchbase.CouchbaseContainer +import org.testcontainers.utility.DockerImageName import spock.lang.Shared import static datadog.trace.agent.test.utils.TraceUtils.basicSpan @@ -39,8 +40,9 @@ abstract class CouchbaseClient31Test extends VersionedNamingTestBase { Bucket bucket def setupSpec() { - def arch = System.getProperty("os.arch") == "aarch64" ? "-aarch64" : "" - couchbase = new CouchbaseContainer("couchbase/server:7.1.0${arch}") + couchbase = new CouchbaseContainer( + DockerImageName.parse(System.getProperty("test.couchbase.image")) + .asCompatibleSubstituteFor("couchbase/server")) .withBucket(new BucketDefinition(BUCKET).withPrimaryIndex(true)) .withStartupTimeout(Duration.ofSeconds(240)) .withStartupAttempts(3) diff --git a/dd-java-agent/instrumentation/couchbase/couchbase-3.2/build.gradle b/dd-java-agent/instrumentation/couchbase/couchbase-3.2/build.gradle index 60459d3f2b2..0a20c86d311 100644 --- a/dd-java-agent/instrumentation/couchbase/couchbase-3.2/build.gradle +++ b/dd-java-agent/instrumentation/couchbase/couchbase-3.2/build.gradle @@ -1,5 +1,8 @@ +import org.gradle.nativeplatform.platform.internal.DefaultNativePlatform + plugins { id 'dd-trace-java.module.instrumentation' + id 'dd-trace-java.testcontainers' } muzzle { @@ -17,6 +20,11 @@ addTestSuiteForDir('latestDepTest', 'test') dependencies { compileOnly group: 'com.couchbase.client', name: 'java-client', version: '3.2.0' + testContainerImage(image( + DefaultNativePlatform.getCurrentArchitecture().isArm64() + ? 'couchbase/server:7.1.0-aarch64' + : 'couchbase/server:7.1.0', + 'test.couchbase.image')) testImplementation group: 'com.couchbase.client', name: 'java-client', version: '3.2.0' testImplementation group: 'org.testcontainers', name: 'couchbase', version: libs.versions.testcontainers.get() diff --git a/dd-java-agent/instrumentation/couchbase/couchbase-3.2/src/test/groovy/CouchbaseClient32Test.groovy b/dd-java-agent/instrumentation/couchbase/couchbase-3.2/src/test/groovy/CouchbaseClient32Test.groovy index ff7e9968cc8..fe11f51f2ff 100644 --- a/dd-java-agent/instrumentation/couchbase/couchbase-3.2/src/test/groovy/CouchbaseClient32Test.groovy +++ b/dd-java-agent/instrumentation/couchbase/couchbase-3.2/src/test/groovy/CouchbaseClient32Test.groovy @@ -28,6 +28,7 @@ import org.slf4j.Logger import org.slf4j.LoggerFactory import org.testcontainers.couchbase.BucketDefinition import org.testcontainers.couchbase.CouchbaseContainer +import org.testcontainers.utility.DockerImageName import reactor.core.publisher.Mono import spock.lang.Shared @@ -45,8 +46,9 @@ abstract class CouchbaseClient32Test extends VersionedNamingTestBase { Bucket bucket def setupSpec() { - def arch = System.getProperty("os.arch") == "aarch64" ? "-aarch64" : "" - couchbase = new CouchbaseContainer("couchbase/server:7.1.0${arch}") + couchbase = new CouchbaseContainer( + DockerImageName.parse(System.getProperty("test.couchbase.image")) + .asCompatibleSubstituteFor("couchbase/server")) .withBucket(new BucketDefinition(BUCKET).withPrimaryIndex(true)) .withStartupTimeout(Duration.ofSeconds(240)) .withStartupAttempts(3) diff --git a/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-3.0/build.gradle b/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-3.0/build.gradle index f74982c2038..9718e557981 100644 --- a/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-3.0/build.gradle +++ b/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-3.0/build.gradle @@ -1,5 +1,6 @@ plugins { id 'dd-trace-java.module.instrumentation' + id 'dd-trace-java.testcontainers' } muzzle { @@ -50,6 +51,8 @@ testJvmConstraints { addTestSuiteForDir('latestDepTest', 'test') dependencies { + testContainerImage(image('cassandra:3', 'test.cassandra.image')) + constraints { testImplementation("com.google.guava:guava") { version { strictly "19.0" } diff --git a/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-3.0/src/test/groovy/CassandraClientTest.groovy b/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-3.0/src/test/groovy/CassandraClientTest.groovy index d4e0ac21065..88c820b9dc3 100644 --- a/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-3.0/src/test/groovy/CassandraClientTest.groovy +++ b/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-3.0/src/test/groovy/CassandraClientTest.groovy @@ -12,7 +12,8 @@ import datadog.trace.api.DDSpanTypes import datadog.trace.bootstrap.instrumentation.api.InstrumentationTags import datadog.trace.bootstrap.instrumentation.api.Tags import datadog.trace.core.DDSpan -import org.testcontainers.containers.CassandraContainer +import org.testcontainers.cassandra.CassandraContainer +import org.testcontainers.utility.DockerImageName import spock.lang.Shared import java.time.Duration @@ -42,10 +43,16 @@ abstract class CassandraClientTest extends VersionedNamingTestBase { CassandraContainer container def setupSpec() { - container = new CassandraContainer("cassandra:3").withStartupTimeout(Duration.ofSeconds(120)) + def image = DockerImageName.parse(System.getProperty("test.cassandra.image")) + .asCompatibleSubstituteFor("cassandra") + container = new CassandraContainer(image).withStartupTimeout(Duration.ofSeconds(120)) container.start() - cluster = container.getCluster() port = container.getMappedPort(9042) + cluster = Cluster.builder() + .addContactPoint(container.getHost()) + .withPort(port) + .withoutJMXReporting() + .build() // Looks like sometimes our requests fail because Cassandra takes to long to respond, // Increase this timeout as well to try to cope with this. cluster.getConfiguration().getSocketOptions().setReadTimeoutMillis(120000) diff --git a/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-3.8/build.gradle b/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-3.8/build.gradle index 535c56510e1..5e74728b352 100644 --- a/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-3.8/build.gradle +++ b/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-3.8/build.gradle @@ -1,5 +1,6 @@ plugins { id 'dd-trace-java.module.instrumentation' + id 'dd-trace-java.testcontainers' } muzzle { @@ -26,6 +27,8 @@ testJvmConstraints { addTestSuiteForDir('latestDepTest', 'test') dependencies { + testContainerImage(image('cassandra:3', 'test.cassandra.image')) + compileOnly group: 'com.datastax.cassandra', name: 'cassandra-driver-core', version: '3.8.0' compileOnly group: 'com.google.guava', name: 'guava', version: '18.0' diff --git a/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-3.8/src/test/groovy/CassandraClientTest.groovy b/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-3.8/src/test/groovy/CassandraClientTest.groovy index d4e0ac21065..88c820b9dc3 100644 --- a/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-3.8/src/test/groovy/CassandraClientTest.groovy +++ b/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-3.8/src/test/groovy/CassandraClientTest.groovy @@ -12,7 +12,8 @@ import datadog.trace.api.DDSpanTypes import datadog.trace.bootstrap.instrumentation.api.InstrumentationTags import datadog.trace.bootstrap.instrumentation.api.Tags import datadog.trace.core.DDSpan -import org.testcontainers.containers.CassandraContainer +import org.testcontainers.cassandra.CassandraContainer +import org.testcontainers.utility.DockerImageName import spock.lang.Shared import java.time.Duration @@ -42,10 +43,16 @@ abstract class CassandraClientTest extends VersionedNamingTestBase { CassandraContainer container def setupSpec() { - container = new CassandraContainer("cassandra:3").withStartupTimeout(Duration.ofSeconds(120)) + def image = DockerImageName.parse(System.getProperty("test.cassandra.image")) + .asCompatibleSubstituteFor("cassandra") + container = new CassandraContainer(image).withStartupTimeout(Duration.ofSeconds(120)) container.start() - cluster = container.getCluster() port = container.getMappedPort(9042) + cluster = Cluster.builder() + .addContactPoint(container.getHost()) + .withPort(port) + .withoutJMXReporting() + .build() // Looks like sometimes our requests fail because Cassandra takes to long to respond, // Increase this timeout as well to try to cope with this. cluster.getConfiguration().getSocketOptions().setReadTimeoutMillis(120000) diff --git a/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-4.0/build.gradle b/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-4.0/build.gradle index 595c7874922..5a87a224cd0 100644 --- a/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-4.0/build.gradle +++ b/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-4.0/build.gradle @@ -1,5 +1,6 @@ plugins { id 'dd-trace-java.module.instrumentation' + id 'dd-trace-java.testcontainers' } muzzle { @@ -20,6 +21,8 @@ testJvmConstraints { addTestSuiteForDir('latestDepTest', 'test') dependencies { + testContainerImage(image('cassandra:4', 'test.cassandra.image')) + compileOnly group: 'com.datastax.oss', name: 'java-driver-core', version: '4.0.0' // ProgrammaticConfig, required to set the timeout, wasn't added until 4.0.1 diff --git a/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-4.0/src/test/groovy/CassandraClientTest.groovy b/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-4.0/src/test/groovy/CassandraClientTest.groovy index ba3aadb8773..49cae522fdf 100644 --- a/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-4.0/src/test/groovy/CassandraClientTest.groovy +++ b/dd-java-agent/instrumentation/datastax-cassandra/datastax-cassandra-4.0/src/test/groovy/CassandraClientTest.groovy @@ -12,7 +12,8 @@ import datadog.trace.api.DDSpanTypes import datadog.trace.bootstrap.instrumentation.api.InstrumentationTags import datadog.trace.bootstrap.instrumentation.api.Tags import datadog.trace.core.DDSpan -import org.testcontainers.containers.CassandraContainer +import org.testcontainers.cassandra.CassandraContainer +import org.testcontainers.utility.DockerImageName import spock.lang.Shared import spock.util.concurrent.BlockingVariable @@ -44,7 +45,9 @@ abstract class CassandraClientTest extends VersionedNamingTestBase { CassandraContainer container def setupSpec() { - container = new CassandraContainer("cassandra:4").withStartupTimeout(Duration.ofSeconds(120)) + def image = DockerImageName.parse(System.getProperty("test.cassandra.image")) + .asCompatibleSubstituteFor("cassandra") + container = new CassandraContainer(image).withStartupTimeout(Duration.ofSeconds(120)) container.start() port = container.getMappedPort(9042) address = new InetSocketAddress(container.getHost(), port) diff --git a/dd-java-agent/instrumentation/google-pubsub-1.116/build.gradle b/dd-java-agent/instrumentation/google-pubsub-1.116/build.gradle index 03f919585a7..a2b03630339 100644 --- a/dd-java-agent/instrumentation/google-pubsub-1.116/build.gradle +++ b/dd-java-agent/instrumentation/google-pubsub-1.116/build.gradle @@ -1,5 +1,6 @@ plugins { id 'dd-trace-java.module.instrumentation' + id 'dd-trace-java.testcontainers' } muzzle { @@ -16,6 +17,8 @@ addTestSuiteExtendingForDir('latestDepForkedTest', 'latestDepTest', 'test') dependencies { + testContainerImage(image('gcr.io/google.com/cloudsdktool/google-cloud-cli:emulators', 'test.pubsub.image')) + compileOnly group: 'com.google.cloud', name: 'google-cloud-pubsub', version: '1.116.0' testImplementation group: 'org.testcontainers', name: 'gcloud', version: libs.versions.testcontainers.get() testImplementation group: 'com.google.cloud', name: 'google-cloud-pubsub', version: '1.116.0' diff --git a/dd-java-agent/instrumentation/google-pubsub-1.116/src/test/groovy/PubSubTest.groovy b/dd-java-agent/instrumentation/google-pubsub-1.116/src/test/groovy/PubSubTest.groovy index 3317485f031..7c40ed076ae 100644 --- a/dd-java-agent/instrumentation/google-pubsub-1.116/src/test/groovy/PubSubTest.groovy +++ b/dd-java-agent/instrumentation/google-pubsub-1.116/src/test/groovy/PubSubTest.groovy @@ -90,7 +90,9 @@ abstract class PubSubTest extends VersionedNamingTestBase { } def setupSpec() { - emulator = new PubSubEmulatorContainer(DockerImageName.parse("gcr.io/google.com/cloudsdktool/google-cloud-cli:emulators")) + def image = DockerImageName.parse(System.getProperty("test.pubsub.image")) + .asCompatibleSubstituteFor("gcr.io/google.com/cloudsdktool/google-cloud-cli") + emulator = new PubSubEmulatorContainer(image) emulator.start() channel = ManagedChannelBuilder.forTarget(emulator.getEmulatorEndpoint()).usePlaintext().build() transportChannelProvider = FixedTransportChannelProvider.create(GrpcTransportChannel.create(channel)) diff --git a/dd-java-agent/instrumentation/jdbc/build.gradle b/dd-java-agent/instrumentation/jdbc/build.gradle index 6ee3987ca2d..dc2622fdec5 100644 --- a/dd-java-agent/instrumentation/jdbc/build.gradle +++ b/dd-java-agent/instrumentation/jdbc/build.gradle @@ -1,6 +1,7 @@ plugins { id 'java-test-fixtures' id 'dd-trace-java.module.instrumentation' + id 'dd-trace-java.testcontainers' id 'dd-trace-java.call-site-instrumentation' id 'dd-trace-java.jmh-conventions' } @@ -18,7 +19,17 @@ addTestSuiteForDir('oldPostgresTest', 'test') addTestSuiteForDir('latestDepTest', 'test') addTestSuiteExtendingForDir('latestDepJava11Test', 'latestDepTest', 'test') +// This non-Docker-Hub image has a separate mirror in CI. +def sqlServerImage = providers.environmentVariable('CI').isPresent() + ? 'registry.ddbuild.io/images/mirror/sqlserver:latest' + : 'mcr.microsoft.com/mssql/server:latest' + dependencies { + testContainerImage(image('postgres:11.2', 'test.postgres.image')) + testContainerImage(image('mysql:8.0', 'test.mysql.image')) + testContainerImage(image(sqlServerImage, 'test.sqlserver.image')) + testContainerImage(image('gvenzl/oracle-free:23.5-slim-faststart', 'test.oracle.image')) + compileOnly group: 'com.zaxxer', name: 'HikariCP', version: '2.4.0' testImplementation project(':dd-java-agent:agent-iast:iast-test-fixtures') @@ -79,7 +90,9 @@ tasks.named("check") { } tasks.named("latestDepJava11Test", Test) { - javaLauncher = getJavaLauncherFor(11) + if (!providers.gradleProperty('testJvm').isPresent()) { + javaLauncher = getJavaLauncherFor(11) + } } tasks.withType(GroovyCompile).configureEach { diff --git a/dd-java-agent/instrumentation/jdbc/src/test/groovy/RemoteJDBCInstrumentationTest.groovy b/dd-java-agent/instrumentation/jdbc/src/test/groovy/RemoteJDBCInstrumentationTest.groovy index 069971e97ef..2870bca13d3 100644 --- a/dd-java-agent/instrumentation/jdbc/src/test/groovy/RemoteJDBCInstrumentationTest.groovy +++ b/dd-java-agent/instrumentation/jdbc/src/test/groovy/RemoteJDBCInstrumentationTest.groovy @@ -154,7 +154,9 @@ abstract class RemoteJDBCInstrumentationTest extends VersionedNamingTestBase { return } - PostgreSQLContainer server = new PostgreSQLContainer("postgres:11.2") + def image = DockerImageName.parse(System.getProperty("test.postgres.image")) + .asCompatibleSubstituteFor("postgres") + PostgreSQLContainer server = new PostgreSQLContainer(image) .withDatabaseName(dbName.get(POSTGRESQL)) .withUsername(jdbcUserNames.get(POSTGRESQL)) .withPassword(jdbcPasswords.get(POSTGRESQL)) @@ -175,7 +177,9 @@ abstract class RemoteJDBCInstrumentationTest extends VersionedNamingTestBase { return } - MySQLContainer server = new MySQLContainer("mysql:8.0") + def image = DockerImageName.parse(System.getProperty("test.mysql.image")) + .asCompatibleSubstituteFor("mysql") + MySQLContainer server = new MySQLContainer(image) .withDatabaseName(dbName.get(MYSQL)) .withUsername(jdbcUserNames.get(MYSQL)) .withPassword(jdbcPasswords.get(MYSQL)) @@ -198,7 +202,9 @@ abstract class RemoteJDBCInstrumentationTest extends VersionedNamingTestBase { return } - MSSQLServerContainer server = new MSSQLServerContainer(MSSQLServerContainer.IMAGE) + def image = DockerImageName.parse(System.getProperty("test.sqlserver.image")) + .asCompatibleSubstituteFor(MSSQLServerContainer.IMAGE) + def server = new MSSQLServerContainer(image) .acceptLicense() .withPassword(jdbcPasswords.get(SQLSERVER)) // SQL Server can occasionally abort while booting on virtualized CI hosts. @@ -221,7 +227,8 @@ abstract class RemoteJDBCInstrumentationTest extends VersionedNamingTestBase { } // Earlier Oracle version images (oracle-xe) don't work on arm64 - DockerImageName oracleImage = DockerImageName.parse("gvenzl/oracle-free:23.5-slim-faststart").asCompatibleSubstituteFor("gvenzl/oracle-xe") + DockerImageName oracleImage = DockerImageName.parse(System.getProperty("test.oracle.image")) + .asCompatibleSubstituteFor("gvenzl/oracle-xe") OracleContainer server = new OracleContainer(oracleImage) .withStartupTimeout(Duration.ofMinutes(5)) .withUsername(jdbcUserNames.get(ORACLE)) diff --git a/dd-java-agent/instrumentation/jdbc/src/test/java/test/DataDogRegistryImageNameSubstitutor.java b/dd-java-agent/instrumentation/jdbc/src/test/java/test/DataDogRegistryImageNameSubstitutor.java deleted file mode 100644 index 8f7693f49d1..00000000000 --- a/dd-java-agent/instrumentation/jdbc/src/test/java/test/DataDogRegistryImageNameSubstitutor.java +++ /dev/null @@ -1,35 +0,0 @@ -package test; - -import org.testcontainers.utility.DockerImageName; -import org.testcontainers.utility.ImageNameSubstitutor; - -/** - * A custom {@link ImageNameSubstitutor} implementation that rewrites Docker image names to use - * Datadog's internal registry {@code registry.ddbuild.io} when running in a CI environment. - * - *

Images from DockerHub already mirrored by {@code registry.ddbuild.io} via environment variable - * {@code TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX} - * - *

For images from other repositories custom image name substitutor should be implemented. - * Internal registry is faster and not affected by rate limiting. - */ -public class DataDogRegistryImageNameSubstitutor extends ImageNameSubstitutor { - @Override - public DockerImageName apply(DockerImageName original) { - String name = original.asCanonicalNameString(); - - if (System.getenv("CI") != null) { - // For now, we need to mirror Microsoft SQL Server images only. - name = - name.replace( - "mcr.microsoft.com/mssql/server:", "registry.ddbuild.io/images/mirror/sqlserver:"); - } - - return DockerImageName.parse(name); - } - - @Override - protected String getDescription() { - return "Image name substitutor to load images from registry.ddbuild.io"; - } -} diff --git a/dd-java-agent/instrumentation/jdbc/src/test/resources/testcontainers.properties b/dd-java-agent/instrumentation/jdbc/src/test/resources/testcontainers.properties deleted file mode 100644 index 03fb660c6d1..00000000000 --- a/dd-java-agent/instrumentation/jdbc/src/test/resources/testcontainers.properties +++ /dev/null @@ -1 +0,0 @@ -image.substitutor=test.DataDogRegistryImageNameSubstitutor diff --git a/dd-java-agent/instrumentation/lettuce/lettuce-5.0/build.gradle b/dd-java-agent/instrumentation/lettuce/lettuce-5.0/build.gradle index 14f2768a709..8c75a2232d6 100644 --- a/dd-java-agent/instrumentation/lettuce/lettuce-5.0/build.gradle +++ b/dd-java-agent/instrumentation/lettuce/lettuce-5.0/build.gradle @@ -1,6 +1,7 @@ plugins { id 'dd-trace-java.module.instrumentation' id 'dd-trace-java.jmh-conventions' + id 'dd-trace-java.testcontainers' } muzzle { @@ -22,6 +23,7 @@ addTestSuiteForDir('lettuce62Test', 'test') dependencies { compileOnly group: 'io.lettuce', name: 'lettuce-core', version: '5.0.0.RELEASE' jmh group: 'io.lettuce', name: 'lettuce-core', version: '5.0.0.RELEASE' + testContainerImage(image('redis:6.2.6', 'test.redis.image')) testImplementation group: 'com.redis.testcontainers', name: 'testcontainers-redis', version: '1.6.2' testImplementation libs.testcontainers diff --git a/dd-java-agent/instrumentation/lettuce/lettuce-5.0/src/test/groovy/Lettuce5ClientTestBase.groovy b/dd-java-agent/instrumentation/lettuce/lettuce-5.0/src/test/groovy/Lettuce5ClientTestBase.groovy index 9749dbdfb0f..50ccbdbb137 100644 --- a/dd-java-agent/instrumentation/lettuce/lettuce-5.0/src/test/groovy/Lettuce5ClientTestBase.groovy +++ b/dd-java-agent/instrumentation/lettuce/lettuce-5.0/src/test/groovy/Lettuce5ClientTestBase.groovy @@ -1,3 +1,5 @@ +import static datadog.trace.agent.test.utils.TraceUtils.runUnderTrace + import com.redis.testcontainers.RedisContainer import datadog.trace.agent.test.naming.VersionedNamingTestBase import datadog.trace.agent.test.utils.PortUtils @@ -12,8 +14,6 @@ import org.testcontainers.utility.DockerImageName import spock.lang.Shared import spock.util.concurrent.PollingConditions -import static datadog.trace.agent.test.utils.TraceUtils.runUnderTrace - abstract class Lettuce5ClientTestBase extends VersionedNamingTestBase { public static final int DB_INDEX = 0 // Disable autoreconnect so we do not get stray traces popping up on server shutdown @@ -33,7 +33,9 @@ abstract class Lettuce5ClientTestBase extends VersionedNamingTestBase { String dbUriNonExistent String embeddedDbUri - RedisContainer redisServer = new RedisContainer(DockerImageName.parse("redis:6.2.6")) + RedisContainer redisServer = new RedisContainer( + DockerImageName.parse(System.getProperty("test.redis.image")) + .asCompatibleSubstituteFor("redis")) .waitingFor(Wait.forListeningPort()) RedisClient redisClient diff --git a/dd-java-agent/instrumentation/lettuce/lettuce-5.0/src/test/java/Lettuce5ClusterTest.java b/dd-java-agent/instrumentation/lettuce/lettuce-5.0/src/test/java/Lettuce5ClusterTest.java index ccefb34e0b7..70f56efaf5f 100644 --- a/dd-java-agent/instrumentation/lettuce/lettuce-5.0/src/test/java/Lettuce5ClusterTest.java +++ b/dd-java-agent/instrumentation/lettuce/lettuce-5.0/src/test/java/Lettuce5ClusterTest.java @@ -26,6 +26,7 @@ import org.junit.jupiter.api.Test; import org.testcontainers.containers.GenericContainer; import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.utility.DockerImageName; class Lettuce5ClusterTest extends AbstractInstrumentationTest { private static final String TEST_SET_KEY = "TESTSETKEY"; @@ -53,7 +54,8 @@ void setUpRedis() throws Exception { // Redis cluster discovery returns the announced node port, so the host-side port must be // stable. Use the same random ports inside the container so cluster nodes can also reach each // other at their announced addresses. - redisCluster = new GenericContainer<>("redis:6.2.6"); + redisCluster = + new GenericContainer<>(DockerImageName.parse(System.getProperty("test.redis.image"))); redisCluster.setPortBindings( Arrays.asList( redisClusterMasterPort + ":" + redisClusterMasterPort, diff --git a/dd-java-agent/instrumentation/lettuce/lettuce-5.0/src/test/java/Lettuce5MasterReplicaTest.java b/dd-java-agent/instrumentation/lettuce/lettuce-5.0/src/test/java/Lettuce5MasterReplicaTest.java index 0b1e0a1c3f3..c7f01bbd0f4 100644 --- a/dd-java-agent/instrumentation/lettuce/lettuce-5.0/src/test/java/Lettuce5MasterReplicaTest.java +++ b/dd-java-agent/instrumentation/lettuce/lettuce-5.0/src/test/java/Lettuce5MasterReplicaTest.java @@ -33,7 +33,9 @@ class Lettuce5MasterReplicaTest extends AbstractInstrumentationTest { @BeforeEach void setUpRedis() throws Exception { redisServer = - new RedisContainer(DockerImageName.parse("redis:6.2.6")) + new RedisContainer( + DockerImageName.parse(System.getProperty("test.redis.image")) + .asCompatibleSubstituteFor("redis")) .waitingFor(Wait.forListeningPort()); redisServer.start(); diff --git a/dd-java-agent/instrumentation/mongo/mongo-common/src/testFixtures/groovy/MongoBaseTest.groovy b/dd-java-agent/instrumentation/mongo/mongo-common/src/testFixtures/groovy/MongoBaseTest.groovy index 55f686f712b..e45286d6438 100644 --- a/dd-java-agent/instrumentation/mongo/mongo-common/src/testFixtures/groovy/MongoBaseTest.groovy +++ b/dd-java-agent/instrumentation/mongo/mongo-common/src/testFixtures/groovy/MongoBaseTest.groovy @@ -7,6 +7,7 @@ import datadog.trace.bootstrap.instrumentation.api.Tags import datadog.trace.core.DDSpan import org.slf4j.LoggerFactory import org.testcontainers.containers.MongoDBContainer +import org.testcontainers.utility.DockerImageName import spock.lang.Shared abstract class MongoBaseTest extends VersionedNamingTestBase { @@ -32,12 +33,10 @@ abstract class MongoBaseTest extends VersionedNamingTestBase { abstract String dbType() - def mongodbImageName() { - return "mongo:4.4.29" - } - def setupSpec() throws Exception { - mongoDbContainer = new MongoDBContainer(mongodbImageName()) + mongoDbContainer = new MongoDBContainer( + DockerImageName.parse(System.getProperty("test.mongo.image")) + .asCompatibleSubstituteFor("mongo")) mongoDbContainer.start() port = mongoDbContainer.getMappedPort(27017) logger.info("MongoDB started on port {}", port) diff --git a/dd-java-agent/instrumentation/mongo/mongo-driver/mongo-driver-3/mongo-driver-3.1/build.gradle b/dd-java-agent/instrumentation/mongo/mongo-driver/mongo-driver-3/mongo-driver-3.1/build.gradle index 29a894a4d1d..54fd338b83c 100644 --- a/dd-java-agent/instrumentation/mongo/mongo-driver/mongo-driver-3/mongo-driver-3.1/build.gradle +++ b/dd-java-agent/instrumentation/mongo/mongo-driver/mongo-driver-3/mongo-driver-3.1/build.gradle @@ -1,5 +1,6 @@ plugins { id 'dd-trace-java.module.instrumentation' + id 'dd-trace-java.testcontainers' } muzzle { @@ -29,6 +30,7 @@ dependencies { testImplementation libs.bundles.junit5 compileOnly group: 'org.mongodb', name: 'mongo-java-driver', version: '3.1.0' + testContainerImage(image('mongo:4.4.29', 'test.mongo.image')) implementation(project(':dd-java-agent:instrumentation:mongo:mongo-common')) { transitive = false diff --git a/dd-java-agent/instrumentation/mongo/mongo-driver/mongo-driver-3/mongo-driver-3.4/build.gradle b/dd-java-agent/instrumentation/mongo/mongo-driver/mongo-driver-3/mongo-driver-3.4/build.gradle index ac693dbfd96..67d1debc317 100644 --- a/dd-java-agent/instrumentation/mongo/mongo-driver/mongo-driver-3/mongo-driver-3.4/build.gradle +++ b/dd-java-agent/instrumentation/mongo/mongo-driver/mongo-driver-3/mongo-driver-3.4/build.gradle @@ -1,5 +1,6 @@ plugins { id 'dd-trace-java.module.instrumentation' + id 'dd-trace-java.testcontainers' } muzzle { @@ -36,6 +37,7 @@ dependencies { compileOnly group: 'org.mongodb', name: 'mongo-java-driver', version: '3.4.0' compileOnly group: 'org.mongodb', name: 'mongodb-driver-core', version: '3.4.0' + testContainerImage(image('mongo:4.4.29', 'test.mongo.image')) implementation(project(':dd-java-agent:instrumentation:mongo:mongo-common')) { transitive = false diff --git a/dd-java-agent/instrumentation/mongo/mongo-driver/mongo-driver-3/mongo-driver-3.6/build.gradle b/dd-java-agent/instrumentation/mongo/mongo-driver/mongo-driver-3/mongo-driver-3.6/build.gradle index bbc11a5a13c..88c0800beeb 100644 --- a/dd-java-agent/instrumentation/mongo/mongo-driver/mongo-driver-3/mongo-driver-3.6/build.gradle +++ b/dd-java-agent/instrumentation/mongo/mongo-driver/mongo-driver-3/mongo-driver-3.6/build.gradle @@ -1,5 +1,6 @@ plugins { id 'dd-trace-java.module.instrumentation' + id 'dd-trace-java.testcontainers' } muzzle { @@ -34,6 +35,7 @@ addTestSuiteForDir('latestDepTest', 'test') dependencies { compileOnly group: 'org.mongodb', name: 'mongo-java-driver', version: '3.6.0' compileOnly group: 'org.mongodb', name: 'mongodb-driver-core', version: '3.6.0' + testContainerImage(image('mongo:4.4.29', 'test.mongo.image')) implementation(project(':dd-java-agent:instrumentation:mongo:mongo-common')) { transitive = false diff --git a/dd-java-agent/instrumentation/mongo/mongo-driver/mongo-driver-3/mongo-driver-3.8/build.gradle b/dd-java-agent/instrumentation/mongo/mongo-driver/mongo-driver-3/mongo-driver-3.8/build.gradle index 71a55e33118..174c7c45621 100644 --- a/dd-java-agent/instrumentation/mongo/mongo-driver/mongo-driver-3/mongo-driver-3.8/build.gradle +++ b/dd-java-agent/instrumentation/mongo/mongo-driver/mongo-driver-3/mongo-driver-3.8/build.gradle @@ -1,5 +1,6 @@ plugins { id 'dd-trace-java.module.instrumentation' + id 'dd-trace-java.testcontainers' } muzzle { @@ -34,6 +35,7 @@ addTestSuiteForDir('latestDepTest', 'test') dependencies { compileOnly group: 'org.mongodb', name: 'mongo-java-driver', version: '3.8.0' compileOnly group: 'org.mongodb', name: 'mongodb-driver-core', version: '3.8.0' + testContainerImage(image('mongo:4.4.29', 'test.mongo.image')) implementation(project(':dd-java-agent:instrumentation:mongo:mongo-common')) { transitive = false diff --git a/dd-java-agent/instrumentation/mongo/mongo-driver/mongo-driver-4.0/build.gradle b/dd-java-agent/instrumentation/mongo/mongo-driver/mongo-driver-4.0/build.gradle index b4ee761702c..5e47265167a 100644 --- a/dd-java-agent/instrumentation/mongo/mongo-driver/mongo-driver-4.0/build.gradle +++ b/dd-java-agent/instrumentation/mongo/mongo-driver/mongo-driver-4.0/build.gradle @@ -1,5 +1,6 @@ plugins { id 'dd-trace-java.module.instrumentation' + id 'dd-trace-java.testcontainers' } muzzle { @@ -27,6 +28,7 @@ addTestSuiteForDir('mongo410ForkedTest', 'test') dependencies { compileOnly group: 'org.mongodb', name: 'mongodb-driver-sync', version: '4.0.0' compileOnly group: 'org.mongodb', name: 'mongodb-driver-reactivestreams', version: '4.0.0' + testContainerImage(image('mongo:4.4.29', 'test.mongo.image')) implementation(project(':dd-java-agent:instrumentation:mongo:mongo-common')) { transitive = false diff --git a/dd-java-agent/instrumentation/mongo/mongo-test/mongo-test-async-3.3/build.gradle b/dd-java-agent/instrumentation/mongo/mongo-test/mongo-test-async-3.3/build.gradle index 9c87d0f1187..8f6f239b6b1 100644 --- a/dd-java-agent/instrumentation/mongo/mongo-test/mongo-test-async-3.3/build.gradle +++ b/dd-java-agent/instrumentation/mongo/mongo-test/mongo-test-async-3.3/build.gradle @@ -1,10 +1,13 @@ plugins { id 'dd-trace-java.module.instrumentation' + id 'dd-trace-java.testcontainers' } addTestSuiteForDir('latestDepTest', 'test') dependencies { + testContainerImage(image('mongo:4.4.29', 'test.mongo.image')) + testImplementation(project(':dd-java-agent:instrumentation:mongo:mongo-common')) { transitive = false } diff --git a/dd-java-agent/instrumentation/mongo/mongo-test/mongo-test-core-3.1/build.gradle b/dd-java-agent/instrumentation/mongo/mongo-test/mongo-test-core-3.1/build.gradle index ccf44f9650e..48d6f90e7a0 100644 --- a/dd-java-agent/instrumentation/mongo/mongo-test/mongo-test-core-3.1/build.gradle +++ b/dd-java-agent/instrumentation/mongo/mongo-test/mongo-test-core-3.1/build.gradle @@ -1,10 +1,13 @@ plugins { id 'dd-trace-java.module.instrumentation' + id 'dd-trace-java.testcontainers' } addTestSuiteForDir('latestDepTest', 'test') dependencies { + testContainerImage(image('mongo:4.4.29', 'test.mongo.image')) + testImplementation group: 'org.testcontainers', name: 'mongodb', version: libs.versions.testcontainers.get() // We need to pull in this dependency to get the 'suspend span' instrumentation for spock tests @@ -22,4 +25,3 @@ dependencies { testImplementation group: 'org.mongodb', name: 'mongodb-driver', version: '3.1.0' latestDepTestImplementation group: 'org.mongodb', name: 'mongodb-driver', version: '3.6+' } - diff --git a/dd-java-agent/instrumentation/mongo/mongo-test/mongo-test-core-3.7/build.gradle b/dd-java-agent/instrumentation/mongo/mongo-test/mongo-test-core-3.7/build.gradle index f436e6a8905..4bbfb9a8b96 100644 --- a/dd-java-agent/instrumentation/mongo/mongo-test/mongo-test-core-3.7/build.gradle +++ b/dd-java-agent/instrumentation/mongo/mongo-test/mongo-test-core-3.7/build.gradle @@ -1,10 +1,13 @@ plugins { id 'dd-trace-java.module.instrumentation' + id 'dd-trace-java.testcontainers' } addTestSuiteForDir('latestDepTest', 'test') dependencies { + testContainerImage(image('mongo:4.4.29', 'test.mongo.image')) + testImplementation group: 'org.testcontainers', name: 'mongodb', version: libs.versions.testcontainers.get() // We need to pull in this dependency to get the 'suspend span' instrumentation for spock tests diff --git a/dd-java-agent/instrumentation/mongo/mongo-test/mongo-test-sync-3.10/build.gradle b/dd-java-agent/instrumentation/mongo/mongo-test/mongo-test-sync-3.10/build.gradle index 5a5b016f198..e6878f999b8 100644 --- a/dd-java-agent/instrumentation/mongo/mongo-test/mongo-test-sync-3.10/build.gradle +++ b/dd-java-agent/instrumentation/mongo/mongo-test/mongo-test-sync-3.10/build.gradle @@ -1,10 +1,13 @@ plugins { id 'dd-trace-java.module.instrumentation' + id 'dd-trace-java.testcontainers' } addTestSuiteForDir('latestDepTest', 'test') dependencies { + testContainerImage(image('mongo:4.4.29', 'test.mongo.image')) + testImplementation(project(':dd-java-agent:instrumentation:mongo:mongo-common')) { transitive = false } diff --git a/dd-java-agent/instrumentation/rabbitmq-amqp-2.7/build.gradle b/dd-java-agent/instrumentation/rabbitmq-amqp-2.7/build.gradle index c7b17611d5b..01fd1960aff 100644 --- a/dd-java-agent/instrumentation/rabbitmq-amqp-2.7/build.gradle +++ b/dd-java-agent/instrumentation/rabbitmq-amqp-2.7/build.gradle @@ -1,5 +1,6 @@ plugins { id 'dd-trace-java.module.instrumentation' + id 'dd-trace-java.testcontainers' } muzzle { @@ -18,6 +19,7 @@ addTestSuite('reactorTest') dependencies { compileOnly group: 'com.rabbitmq', name: 'amqp-client', version: '2.7.0' + testContainerImage(image('rabbitmq:3.9.20-alpine', 'test.rabbitmq.image')) testImplementation group: 'com.rabbitmq', name: 'amqp-client', version: '2.7.0' testImplementation group: 'org.springframework.amqp', name: 'spring-rabbit', version: '1.1.0.RELEASE' diff --git a/dd-java-agent/instrumentation/rabbitmq-amqp-2.7/src/reactorTest/groovy/ReactorRabbitMQTest.groovy b/dd-java-agent/instrumentation/rabbitmq-amqp-2.7/src/reactorTest/groovy/ReactorRabbitMQTest.groovy index da056550c8f..f0289cf5007 100644 --- a/dd-java-agent/instrumentation/rabbitmq-amqp-2.7/src/reactorTest/groovy/ReactorRabbitMQTest.groovy +++ b/dd-java-agent/instrumentation/rabbitmq-amqp-2.7/src/reactorTest/groovy/ReactorRabbitMQTest.groovy @@ -4,6 +4,7 @@ import com.rabbitmq.client.ConnectionFactory import datadog.trace.agent.test.InstrumentationSpecification import datadog.trace.agent.test.utils.PortUtils import org.testcontainers.containers.RabbitMQContainer +import org.testcontainers.utility.DockerImageName import reactor.core.publisher.Mono import reactor.core.scheduler.Schedulers import reactor.rabbitmq.RabbitFlux @@ -29,7 +30,9 @@ class ReactorRabbitMQTest extends InstrumentationSpecification { } def setupSpec() { - rabbitMQContainer = new RabbitMQContainer('rabbitmq:3.9.20-alpine') + rabbitMQContainer = new RabbitMQContainer( + DockerImageName.parse(System.getProperty("test.rabbitmq.image")) + .asCompatibleSubstituteFor("rabbitmq")) .withExposedPorts(defaultRabbitMQPort) .withStartupTimeout(Duration.ofSeconds(120)) rabbitMQContainer.start() diff --git a/dd-java-agent/instrumentation/rabbitmq-amqp-2.7/src/test/groovy/RabbitMQTest.groovy b/dd-java-agent/instrumentation/rabbitmq-amqp-2.7/src/test/groovy/RabbitMQTest.groovy index 966ab9d4b80..7c638baac62 100644 --- a/dd-java-agent/instrumentation/rabbitmq-amqp-2.7/src/test/groovy/RabbitMQTest.groovy +++ b/dd-java-agent/instrumentation/rabbitmq-amqp-2.7/src/test/groovy/RabbitMQTest.groovy @@ -26,6 +26,7 @@ import org.springframework.amqp.rabbit.connection.CachingConnectionFactory import org.springframework.amqp.rabbit.core.RabbitAdmin import org.springframework.amqp.rabbit.core.RabbitTemplate import org.testcontainers.containers.RabbitMQContainer +import org.testcontainers.utility.DockerImageName import spock.lang.Shared import spock.util.concurrent.PollingConditions @@ -70,7 +71,9 @@ abstract class RabbitMQTestBase extends VersionedNamingTestBase { } def setupSpec() { - rabbitMQContainer = new RabbitMQContainer('rabbitmq:3.9.20-alpine') + rabbitMQContainer = new RabbitMQContainer( + DockerImageName.parse(System.getProperty("test.rabbitmq.image")) + .asCompatibleSubstituteFor("rabbitmq")) .withExposedPorts(defaultRabbitMQPort) .withStartupTimeout(Duration.ofSeconds(120)) rabbitMQContainer.start() diff --git a/dd-java-agent/instrumentation/redisson/redisson-2.0.0/build.gradle b/dd-java-agent/instrumentation/redisson/redisson-2.0.0/build.gradle index 3b79ecba0f6..e977ce3a118 100644 --- a/dd-java-agent/instrumentation/redisson/redisson-2.0.0/build.gradle +++ b/dd-java-agent/instrumentation/redisson/redisson-2.0.0/build.gradle @@ -1,5 +1,6 @@ plugins { id 'dd-trace-java.module.instrumentation' + id 'dd-trace-java.testcontainers' } muzzle { @@ -17,6 +18,7 @@ addTestSuiteExtendingForDir('latestDepForkedTest', 'latestDepTest', 'test') dependencies { compileOnly group: 'org.redisson', name: 'redisson', version: '2.0.0' + testContainerImage(image('redis:6.2.6', 'test.redis.image')) testImplementation group: 'com.redis.testcontainers', name: 'testcontainers-redis', version: '1.6.2' testImplementation libs.testcontainers diff --git a/dd-java-agent/instrumentation/redisson/redisson-2.0.0/src/test/groovy/RedissonClientTest.groovy b/dd-java-agent/instrumentation/redisson/redisson-2.0.0/src/test/groovy/RedissonClientTest.groovy index 3201a5f6969..6fae340e06e 100644 --- a/dd-java-agent/instrumentation/redisson/redisson-2.0.0/src/test/groovy/RedissonClientTest.groovy +++ b/dd-java-agent/instrumentation/redisson/redisson-2.0.0/src/test/groovy/RedissonClientTest.groovy @@ -19,7 +19,9 @@ import spock.lang.Shared abstract class RedissonClientTest extends VersionedNamingTestBase { @Shared - RedisServer redisServer = new RedisContainer(DockerImageName.parse("redis:6.2.6")).waitingFor(Wait.forListeningPort()) + RedisServer redisServer = new RedisContainer( + DockerImageName.parse(System.getProperty("test.redis.image"))) + .waitingFor(Wait.forListeningPort()) @Shared Config config = new Config() diff --git a/dd-java-agent/instrumentation/redisson/redisson-2.3.0/build.gradle b/dd-java-agent/instrumentation/redisson/redisson-2.3.0/build.gradle index 8ac63db2bdb..ae779c8397c 100644 --- a/dd-java-agent/instrumentation/redisson/redisson-2.3.0/build.gradle +++ b/dd-java-agent/instrumentation/redisson/redisson-2.3.0/build.gradle @@ -1,5 +1,6 @@ plugins { id 'dd-trace-java.module.instrumentation' + id 'dd-trace-java.testcontainers' } muzzle { @@ -17,6 +18,7 @@ addTestSuiteExtendingForDir('latestDepForkedTest', 'latestDepTest', 'test') dependencies { compileOnly group: 'org.redisson', name: 'redisson', version: '2.3.0' + testContainerImage(image('redis:6.2.6', 'test.redis.image')) testImplementation group: 'com.redis.testcontainers', name: 'testcontainers-redis', version: '1.6.2' testImplementation libs.testcontainers diff --git a/dd-java-agent/instrumentation/redisson/redisson-2.3.0/src/test/groovy/RedissonClientTest.groovy b/dd-java-agent/instrumentation/redisson/redisson-2.3.0/src/test/groovy/RedissonClientTest.groovy index d8e0c5bbc78..dac92fa960e 100644 --- a/dd-java-agent/instrumentation/redisson/redisson-2.3.0/src/test/groovy/RedissonClientTest.groovy +++ b/dd-java-agent/instrumentation/redisson/redisson-2.3.0/src/test/groovy/RedissonClientTest.groovy @@ -16,7 +16,9 @@ import static datadog.trace.api.config.TraceInstrumentationConfig.DB_CLIENT_HOST abstract class RedissonClientTest extends VersionedNamingTestBase { @Shared - RedisServer redisServer = new RedisContainer(DockerImageName.parse("redis:6.2.6")).waitingFor(Wait.forListeningPort()) + RedisServer redisServer = new RedisContainer( + DockerImageName.parse(System.getProperty("test.redis.image"))) + .waitingFor(Wait.forListeningPort()) @Shared Config config = new Config() diff --git a/dd-java-agent/instrumentation/redisson/redisson-3.10.3/build.gradle b/dd-java-agent/instrumentation/redisson/redisson-3.10.3/build.gradle index 159330befa4..00894b36155 100644 --- a/dd-java-agent/instrumentation/redisson/redisson-3.10.3/build.gradle +++ b/dd-java-agent/instrumentation/redisson/redisson-3.10.3/build.gradle @@ -1,5 +1,6 @@ plugins { id 'dd-trace-java.module.instrumentation' + id 'dd-trace-java.testcontainers' } muzzle { @@ -19,6 +20,7 @@ dependencies { compileOnly group: 'org.redisson', name: 'redisson', version: '3.10.3', { exclude group: 'org.slf4j', module: 'slf4j-api' } + testContainerImage(image('redis:6.2.6', 'test.redis.image')) testImplementation group: 'com.redis.testcontainers', name: 'testcontainers-redis', version: '1.6.2' testImplementation libs.testcontainers diff --git a/dd-java-agent/instrumentation/redisson/redisson-3.10.3/src/test/groovy/RedissonClientTest.groovy b/dd-java-agent/instrumentation/redisson/redisson-3.10.3/src/test/groovy/RedissonClientTest.groovy index d1292ab48fd..b426c37d8ea 100644 --- a/dd-java-agent/instrumentation/redisson/redisson-3.10.3/src/test/groovy/RedissonClientTest.groovy +++ b/dd-java-agent/instrumentation/redisson/redisson-3.10.3/src/test/groovy/RedissonClientTest.groovy @@ -16,7 +16,9 @@ import static datadog.trace.api.config.TraceInstrumentationConfig.DB_CLIENT_HOST abstract class RedissonClientTest extends VersionedNamingTestBase { @Shared - RedisServer redisServer = new RedisContainer(DockerImageName.parse("redis:6.2.6")).waitingFor(Wait.forListeningPort()) + RedisServer redisServer = new RedisContainer( + DockerImageName.parse(System.getProperty("test.redis.image"))) + .waitingFor(Wait.forListeningPort()) @Shared Config config = new Config() diff --git a/dd-java-agent/instrumentation/spring/spring-rabbit-1.5/build.gradle b/dd-java-agent/instrumentation/spring/spring-rabbit-1.5/build.gradle index 78cfe257bbc..b7270924bee 100644 --- a/dd-java-agent/instrumentation/spring/spring-rabbit-1.5/build.gradle +++ b/dd-java-agent/instrumentation/spring/spring-rabbit-1.5/build.gradle @@ -1,5 +1,6 @@ plugins { id 'dd-trace-java.module.instrumentation' + id 'dd-trace-java.testcontainers' } muzzle { @@ -17,6 +18,7 @@ addTestSuiteForDir('latestDepTest', 'test') dependencies { compileOnly group: 'org.springframework.amqp', name: 'spring-rabbit', version: '2.0.0.RELEASE' compileOnly group: 'org.springframework.amqp', name: 'spring-amqp', version: '2.0.0.RELEASE' + testContainerImage(image('rabbitmq:3.9.20-alpine', 'test.rabbitmq.image')) testImplementation project(':dd-java-agent:instrumentation:datadog:tracing:trace-annotation') testImplementation project(':dd-java-agent:instrumentation:rabbitmq-amqp-2.7') diff --git a/dd-java-agent/instrumentation/spring/spring-rabbit-1.5/src/test/groovy/SpringAmqpTest.groovy b/dd-java-agent/instrumentation/spring/spring-rabbit-1.5/src/test/groovy/SpringAmqpTest.groovy index c76dda008c3..af075e759e8 100644 --- a/dd-java-agent/instrumentation/spring/spring-rabbit-1.5/src/test/groovy/SpringAmqpTest.groovy +++ b/dd-java-agent/instrumentation/spring/spring-rabbit-1.5/src/test/groovy/SpringAmqpTest.groovy @@ -1,6 +1,7 @@ import datadog.trace.agent.test.InstrumentationSpecification import datadog.trace.agent.test.utils.PortUtils import org.testcontainers.containers.RabbitMQContainer +import org.testcontainers.utility.DockerImageName import rabbit.MessagingRabbitMQApplication import rabbit.Receiver import rabbit.Sender @@ -17,7 +18,9 @@ class SpringAmqpTest extends InstrumentationSpecification { @Override def setupSpec() { - rabbit = new RabbitMQContainer("rabbitmq:3.9.20-alpine") + rabbit = new RabbitMQContainer( + DockerImageName.parse(System.getProperty("test.rabbitmq.image")) + .asCompatibleSubstituteFor("rabbitmq")) rabbit.start() def hostName = rabbit.getHost() def port = rabbit.getMappedPort(MessagingRabbitMQApplication.port) diff --git a/dd-java-agent/instrumentation/spymemcached-2.10/build.gradle b/dd-java-agent/instrumentation/spymemcached-2.10/build.gradle index 6563b7c276d..01eec3adc73 100644 --- a/dd-java-agent/instrumentation/spymemcached-2.10/build.gradle +++ b/dd-java-agent/instrumentation/spymemcached-2.10/build.gradle @@ -1,5 +1,6 @@ plugins { id 'dd-trace-java.module.instrumentation' + id 'dd-trace-java.testcontainers' } muzzle { @@ -20,6 +21,7 @@ tasks.named("latestDepTest", Test) { dependencies { compileOnly group: 'net.spy', name: 'spymemcached', version: '2.10.4' + testContainerImage(image('library/memcached:1.6.14-alpine', 'test.memcached.image')) testImplementation group: 'net.spy', name: 'spymemcached', version: '2.10.4' testImplementation libs.testcontainers diff --git a/dd-java-agent/instrumentation/spymemcached-2.10/src/test/groovy/datadog/trace/instrumentation/spymemcached/SpymemcachedTest.groovy b/dd-java-agent/instrumentation/spymemcached-2.10/src/test/groovy/datadog/trace/instrumentation/spymemcached/SpymemcachedTest.groovy index f8d8d3d593d..9c16f96d8e8 100644 --- a/dd-java-agent/instrumentation/spymemcached-2.10/src/test/groovy/datadog/trace/instrumentation/spymemcached/SpymemcachedTest.groovy +++ b/dd-java-agent/instrumentation/spymemcached-2.10/src/test/groovy/datadog/trace/instrumentation/spymemcached/SpymemcachedTest.groovy @@ -15,6 +15,7 @@ import net.spy.memcached.internal.CheckedOperationTimeoutException import net.spy.memcached.ops.Operation import net.spy.memcached.ops.OperationQueueFactory import org.testcontainers.containers.GenericContainer +import org.testcontainers.utility.DockerImageName import spock.lang.Shared import java.time.Duration @@ -56,7 +57,7 @@ abstract class SpymemcachedTest extends VersionedNamingTestBase { } def setupSpec() { - memcachedContainer = new GenericContainer('library/memcached:1.6.14-alpine') + memcachedContainer = new GenericContainer(DockerImageName.parse(System.getProperty('test.memcached.image'))) .withExposedPorts(defaultMemcachedPort) .withStartupTimeout(Duration.ofSeconds(120)) memcachedContainer.start() diff --git a/dd-java-agent/instrumentation/vertx/vertx-mysql-client/vertx-mysql-client-3.9/build.gradle b/dd-java-agent/instrumentation/vertx/vertx-mysql-client/vertx-mysql-client-3.9/build.gradle index 4485ea3aa32..1f8423384f0 100644 --- a/dd-java-agent/instrumentation/vertx/vertx-mysql-client/vertx-mysql-client-3.9/build.gradle +++ b/dd-java-agent/instrumentation/vertx/vertx-mysql-client/vertx-mysql-client-3.9/build.gradle @@ -1,5 +1,6 @@ plugins { id 'dd-trace-java.module.instrumentation' + id 'dd-trace-java.testcontainers' } muzzle { @@ -20,6 +21,8 @@ tasks.named("latestDepTest", Test) { } dependencies { + testContainerImage(image('mysql:8.0', 'test.mysql.image')) + compileOnly group: 'io.vertx', name: 'vertx-mysql-client', version: '3.9.0' testImplementation group: 'io.vertx', name: 'vertx-mysql-client', version: '3.9.0' diff --git a/dd-java-agent/instrumentation/vertx/vertx-mysql-client/vertx-mysql-client-3.9/src/test/java/TestDatabases.java b/dd-java-agent/instrumentation/vertx/vertx-mysql-client/vertx-mysql-client-3.9/src/test/java/TestDatabases.java index 47793bd21cc..27fecadc279 100644 --- a/dd-java-agent/instrumentation/vertx/vertx-mysql-client/vertx-mysql-client-3.9/src/test/java/TestDatabases.java +++ b/dd-java-agent/instrumentation/vertx/vertx-mysql-client/vertx-mysql-client-3.9/src/test/java/TestDatabases.java @@ -1,11 +1,11 @@ import datadog.trace.agent.test.utils.PortUtils; import java.io.Closeable; -import java.io.IOException; import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.concurrent.TimeUnit; import org.testcontainers.containers.MySQLContainer; +import org.testcontainers.utility.DockerImageName; public class TestDatabases implements Closeable { @@ -13,14 +13,17 @@ public static TestDatabases initialise(String dbName) { return new TestDatabases(dbName); } - private final MySQLContainer mysql; + private final MySQLContainer mysql; private final Map dbInfos; + @SuppressWarnings("resource") private TestDatabases(String dbName) { Map infos = new HashMap<>(); mysql = - new MySQLContainer("mysql:8.0") + new MySQLContainer<>( + DockerImageName.parse(System.getProperty("test.mysql.image")) + .asCompatibleSubstituteFor("mysql")) .withDatabaseName(dbName) .withUsername("sa") .withPassword("sa"); @@ -41,7 +44,7 @@ private TestDatabases(String dbName) { } @Override - public void close() throws IOException { + public void close() { if (null != mysql) { mysql.close(); } diff --git a/dd-java-agent/instrumentation/vertx/vertx-mysql-client/vertx-mysql-client-4.0/build.gradle b/dd-java-agent/instrumentation/vertx/vertx-mysql-client/vertx-mysql-client-4.0/build.gradle index ecdd088f75a..c314173c643 100644 --- a/dd-java-agent/instrumentation/vertx/vertx-mysql-client/vertx-mysql-client-4.0/build.gradle +++ b/dd-java-agent/instrumentation/vertx/vertx-mysql-client/vertx-mysql-client-4.0/build.gradle @@ -1,5 +1,6 @@ plugins { id 'dd-trace-java.module.instrumentation' + id 'dd-trace-java.testcontainers' } muzzle { @@ -19,6 +20,8 @@ tasks.named("latestDepTest", Test) { } dependencies { + testContainerImage(image('mysql:8.0', 'test.mysql.image')) + compileOnly group: 'io.vertx', name: 'vertx-mysql-client', version: '4.0.0' testImplementation group: 'io.vertx', name: 'vertx-mysql-client', version: '4.0.0' diff --git a/dd-java-agent/instrumentation/vertx/vertx-mysql-client/vertx-mysql-client-4.0/src/test/java/TestDatabases.java b/dd-java-agent/instrumentation/vertx/vertx-mysql-client/vertx-mysql-client-4.0/src/test/java/TestDatabases.java index d3b86359399..78ad9a82b20 100644 --- a/dd-java-agent/instrumentation/vertx/vertx-mysql-client/vertx-mysql-client-4.0/src/test/java/TestDatabases.java +++ b/dd-java-agent/instrumentation/vertx/vertx-mysql-client/vertx-mysql-client-4.0/src/test/java/TestDatabases.java @@ -1,21 +1,24 @@ import datadog.trace.agent.test.utils.PortUtils; import java.io.Closeable; -import java.io.IOException; import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.concurrent.TimeUnit; import org.testcontainers.containers.MySQLContainer; +import org.testcontainers.utility.DockerImageName; public class TestDatabases implements Closeable { - private final MySQLContainer mysql; + private final MySQLContainer mysql; private final Map dbInfos; + @SuppressWarnings("resource") private TestDatabases(String dbName) { Map infos = new HashMap<>(); mysql = - new MySQLContainer("mysql:8.0") + new MySQLContainer<>( + DockerImageName.parse(System.getProperty("test.mysql.image")) + .asCompatibleSubstituteFor("mysql")) .withDatabaseName(dbName) .withUsername("sa") .withPassword("sa"); @@ -40,7 +43,7 @@ public static TestDatabases initialise(String dbName) { } @Override - public void close() throws IOException { + public void close() { if (null != mysql) { mysql.close(); } diff --git a/dd-java-agent/instrumentation/vertx/vertx-pg-client/vertx-pg-client-4.0/build.gradle b/dd-java-agent/instrumentation/vertx/vertx-pg-client/vertx-pg-client-4.0/build.gradle index 99a8aee5236..0db168b53ad 100644 --- a/dd-java-agent/instrumentation/vertx/vertx-pg-client/vertx-pg-client-4.0/build.gradle +++ b/dd-java-agent/instrumentation/vertx/vertx-pg-client/vertx-pg-client-4.0/build.gradle @@ -1,5 +1,6 @@ plugins { id 'dd-trace-java.module.instrumentation' + id 'dd-trace-java.testcontainers' } muzzle { @@ -27,6 +28,8 @@ tasks.named("latestDepTest", Test) { } dependencies { + testContainerImage(image('postgres:16-alpine', 'test.postgres.image')) + compileOnly group: 'io.vertx', name: 'vertx-pg-client', version: '4.1.1' testImplementation group: 'io.vertx', name: 'vertx-pg-client', version: '4.1.1' diff --git a/dd-java-agent/instrumentation/vertx/vertx-pg-client/vertx-pg-client-4.0/src/test/java/TestDatabases.java b/dd-java-agent/instrumentation/vertx/vertx-pg-client/vertx-pg-client-4.0/src/test/java/TestDatabases.java index 1e31905cf79..3e1f72e8d41 100644 --- a/dd-java-agent/instrumentation/vertx/vertx-pg-client/vertx-pg-client-4.0/src/test/java/TestDatabases.java +++ b/dd-java-agent/instrumentation/vertx/vertx-pg-client/vertx-pg-client-4.0/src/test/java/TestDatabases.java @@ -1,21 +1,24 @@ import datadog.trace.agent.test.utils.PortUtils; import java.io.Closeable; -import java.io.IOException; import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.concurrent.TimeUnit; import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.utility.DockerImageName; public class TestDatabases implements Closeable { - private final PostgreSQLContainer pgsql; + private final PostgreSQLContainer pgsql; private final Map dbInfos; + @SuppressWarnings("resource") private TestDatabases(String dbName) { Map infos = new HashMap<>(); pgsql = - new PostgreSQLContainer("postgres:16-alpine") + new PostgreSQLContainer<>( + DockerImageName.parse(System.getProperty("test.postgres.image")) + .asCompatibleSubstituteFor("postgres")) .withDatabaseName(dbName) .withUsername("postgres") .withPassword("postgres"); @@ -38,7 +41,7 @@ public static TestDatabases initialise(String dbName) { } @Override - public void close() throws IOException { + public void close() { if (null != pgsql) { pgsql.close(); } diff --git a/dd-java-agent/instrumentation/vertx/vertx-redis-client/vertx-redis-client-3.9/build.gradle b/dd-java-agent/instrumentation/vertx/vertx-redis-client/vertx-redis-client-3.9/build.gradle index 77488806a6b..c5a9cb45db7 100644 --- a/dd-java-agent/instrumentation/vertx/vertx-redis-client/vertx-redis-client-3.9/build.gradle +++ b/dd-java-agent/instrumentation/vertx/vertx-redis-client/vertx-redis-client-3.9/build.gradle @@ -1,5 +1,6 @@ plugins { id 'dd-trace-java.module.instrumentation' + id 'dd-trace-java.testcontainers' } muzzle { @@ -21,6 +22,7 @@ addTestSuiteExtendingForDir('redis4xForkedTest', 'redis4xTest', 'test') dependencies { compileOnly project(':dd-java-agent:instrumentation:vertx:vertx-redis-client:vertx-redis-client-stubs') compileOnly group: 'io.vertx', name: 'vertx-redis-client', version: '3.9.0' + testContainerImage(image('redis:6.2.6', 'test.redis.image')) // only needed for the rx tests testImplementation project(':dd-java-agent:instrumentation:vertx:vertx-rx-3.5') diff --git a/dd-java-agent/instrumentation/vertx/vertx-redis-client/vertx-redis-client-3.9/src/test/groovy/VertxRedisTestBase.groovy b/dd-java-agent/instrumentation/vertx/vertx-redis-client/vertx-redis-client-3.9/src/test/groovy/VertxRedisTestBase.groovy index 8c26b18680a..dc8038613f8 100644 --- a/dd-java-agent/instrumentation/vertx/vertx-redis-client/vertx-redis-client-3.9/src/test/groovy/VertxRedisTestBase.groovy +++ b/dd-java-agent/instrumentation/vertx/vertx-redis-client/vertx-redis-client-3.9/src/test/groovy/VertxRedisTestBase.groovy @@ -36,7 +36,8 @@ abstract class VertxRedisTestBase extends VersionedNamingTestBase { @AutoCleanup(value = "stop") @Shared - def redisServer = new RedisContainer(DockerImageName.parse("redis:6.2.6")) + def redisServer = new RedisContainer( + DockerImageName.parse(System.getProperty("test.redis.image"))) .waitingFor(Wait.forListeningPort()) @Shared diff --git a/dd-smoke-tests/spring-boot-rabbit/build.gradle b/dd-smoke-tests/spring-boot-rabbit/build.gradle index 0c3f554d546..1fa4b6e5d7c 100644 --- a/dd-smoke-tests/spring-boot-rabbit/build.gradle +++ b/dd-smoke-tests/spring-boot-rabbit/build.gradle @@ -3,6 +3,7 @@ import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar plugins { id 'com.gradleup.shadow' id 'dd-trace-java.module.smoke-test' + id 'dd-trace-java.testcontainers' } description = 'SpringBoot RabbitMQ Smoke Tests.' @@ -20,6 +21,8 @@ tasks.named("shadowJar", ShadowJar) { } dependencies { + testContainerImage(image('rabbitmq:3.9.20-alpine', 'test.rabbitmq.image')) + implementation project(':dd-trace-api') implementation group: 'org.springframework.boot', name: 'spring-boot-starter-web', version: '2.5.4' implementation group: 'org.springframework.boot', name: 'spring-boot-starter-amqp', version: '2.5.4' diff --git a/dd-smoke-tests/spring-boot-rabbit/src/test/java/datadog/smoketest/SpringBootRabbitSmokeTest.java b/dd-smoke-tests/spring-boot-rabbit/src/test/java/datadog/smoketest/SpringBootRabbitSmokeTest.java index ed8d42c7c2b..b1213e7469d 100644 --- a/dd-smoke-tests/spring-boot-rabbit/src/test/java/datadog/smoketest/SpringBootRabbitSmokeTest.java +++ b/dd-smoke-tests/spring-boot-rabbit/src/test/java/datadog/smoketest/SpringBootRabbitSmokeTest.java @@ -85,7 +85,9 @@ class SpringBootRabbitSmokeTest { @Container private static final RabbitMQContainer RABBIT = - new RabbitMQContainer(DockerImageName.parse("rabbitmq:3.9.20-alpine")); + new RabbitMQContainer( + DockerImageName.parse(System.getProperty("test.rabbitmq.image")) + .asCompatibleSubstituteFor("rabbitmq")); @Order(1) @RegisterExtension diff --git a/dd-smoke-tests/springboot-mongo/build.gradle b/dd-smoke-tests/springboot-mongo/build.gradle index 4778f981281..48e2eb21aa4 100644 --- a/dd-smoke-tests/springboot-mongo/build.gradle +++ b/dd-smoke-tests/springboot-mongo/build.gradle @@ -3,6 +3,7 @@ import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar plugins { id 'com.gradleup.shadow' id 'dd-trace-java.module.smoke-test' + id 'dd-trace-java.testcontainers' } description = 'SpringBoot Spring-Data-Mongo Smoke Tests.' @@ -20,6 +21,8 @@ tasks.named("shadowJar", ShadowJar) { } dependencies { + testContainerImage(image('mongo:4.4.29', 'test.mongo.image')) + implementation group: 'org.springframework.boot', name: 'spring-boot-starter-web', version: '2.4.1' implementation group: 'org.springframework.boot', name: 'spring-boot-starter-data-mongodb', version: '2.4.1' diff --git a/dd-smoke-tests/springboot-mongo/src/test/groovy/datadog/smoketest/SpringBootMongoIntegrationTest.groovy b/dd-smoke-tests/springboot-mongo/src/test/groovy/datadog/smoketest/SpringBootMongoIntegrationTest.groovy index 76e8743fe00..032c583cc45 100644 --- a/dd-smoke-tests/springboot-mongo/src/test/groovy/datadog/smoketest/SpringBootMongoIntegrationTest.groovy +++ b/dd-smoke-tests/springboot-mongo/src/test/groovy/datadog/smoketest/SpringBootMongoIntegrationTest.groovy @@ -23,7 +23,9 @@ class SpringBootMongoIntegrationTest extends AbstractServerSmokeTest { @Override void beforeProcessBuilders() { - mongoDbContainer = new MongoDBContainer(DockerImageName.parse("mongo:4.4.29")) + mongoDbContainer = new MongoDBContainer( + DockerImageName.parse(System.getProperty("test.mongo.image")) + .asCompatibleSubstituteFor("mongo")) mongoDbContainer.start() mongoDbUri = mongoDbContainer.replicaSetUrl } diff --git a/dd-smoke-tests/websphere-jmx/build.gradle b/dd-smoke-tests/websphere-jmx/build.gradle index 33a96fb8465..b3ee93ac840 100644 --- a/dd-smoke-tests/websphere-jmx/build.gradle +++ b/dd-smoke-tests/websphere-jmx/build.gradle @@ -1,8 +1,11 @@ plugins { id 'dd-trace-java.module.smoke-test' + id 'dd-trace-java.testcontainers' } dependencies { + testContainerImage(image('icr.io/appcafe/websphere-traditional:latest', 'test.websphere.image')) + testImplementation project(':dd-smoke-tests') testImplementation libs.testcontainers } diff --git a/dd-smoke-tests/websphere-jmx/src/test/groovy/datadog/smoketest/WebSphereJmxSmokeTest.groovy b/dd-smoke-tests/websphere-jmx/src/test/groovy/datadog/smoketest/WebSphereJmxSmokeTest.groovy index 81dde64229d..cc329bc8056 100644 --- a/dd-smoke-tests/websphere-jmx/src/test/groovy/datadog/smoketest/WebSphereJmxSmokeTest.groovy +++ b/dd-smoke-tests/websphere-jmx/src/test/groovy/datadog/smoketest/WebSphereJmxSmokeTest.groovy @@ -71,7 +71,7 @@ class WebSphereJmxSmokeTest extends AbstractSmokeTest { } } - websphere = new GenericContainer("icr.io/appcafe/websphere-traditional:latest") + websphere = new GenericContainer(System.getProperty("test.websphere.image")) // inject wished jvm props for the server we are running .withCopyFileToContainer(MountableFile.forClasspathResource("jvm-config.props"), "/work/config/") // copy the agent jar diff --git a/dd-trace-core/build.gradle b/dd-trace-core/build.gradle index 5424650d961..46695a38059 100644 --- a/dd-trace-core/build.gradle +++ b/dd-trace-core/build.gradle @@ -2,6 +2,7 @@ plugins { id 'dd-trace-java.module.product-subsystem' id 'dd-trace-java.jmh-conventions' id 'dd-trace-java.version-file' + id 'dd-trace-java.testcontainers' } description = 'dd-trace-core' @@ -74,6 +75,9 @@ tasks.named('forkedTest', Test) { } dependencies { + testContainerImage(image('registry.ddbuild.io/images/mirror/dd-apm-test-agent/ddapm-test-agent:v1.64.1', 'test.ddapm.agent.image')) + traceAgentTestContainerImage(image('datadog/agent:7.40.1', 'test.datadog.agent.image')) + api project(':dd-trace-api') api project(':communication') api project(':internal-api') diff --git a/dd-trace-core/src/test/java/datadog/trace/TracerConnectionReliabilityTest.java b/dd-trace-core/src/test/java/datadog/trace/TracerConnectionReliabilityTest.java index cfda2f730c7..490df7f06a4 100644 --- a/dd-trace-core/src/test/java/datadog/trace/TracerConnectionReliabilityTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/TracerConnectionReliabilityTest.java @@ -131,8 +131,7 @@ GenericContainer startTestAgentContainer() { //noinspection GrDeprecatedAPIUsage Use FixedHostPortGenericContainer against deprecation // because we need to know the exposed to configure the tracer at start GenericContainer agentContainer = - new FixedHostPortGenericContainer( - "registry.ddbuild.io/images/mirror/dd-apm-test-agent/ddapm-test-agent:v1.64.1") + new FixedHostPortGenericContainer(System.getProperty("test.ddapm.agent.image")) .withFixedExposedPort(agentContainerPort, DEFAULT_TRACE_AGENT_PORT) .withEnv( "ENABLED_CHECKS", diff --git a/dd-trace-core/src/traceAgentTest/java/AbstractTraceAgentTest.java b/dd-trace-core/src/traceAgentTest/java/AbstractTraceAgentTest.java index 3c30bbe8cfa..d7885d6b0dd 100644 --- a/dd-trace-core/src/traceAgentTest/java/AbstractTraceAgentTest.java +++ b/dd-trace-core/src/traceAgentTest/java/AbstractTraceAgentTest.java @@ -12,6 +12,7 @@ import org.junit.jupiter.api.BeforeEach; import org.testcontainers.containers.GenericContainer; import org.testcontainers.containers.startupcheck.MinimumDurationRunningStartupCheckStrategy; +import org.testcontainers.utility.DockerImageName; abstract class AbstractTraceAgentTest extends DDJavaSpecification { @@ -30,7 +31,8 @@ static void setupSpec() { env.put("DD_HOSTNAME", "doesnotexist"); env.put("DD_LOGS_STDOUT", "yes"); agentContainer = - new GenericContainer<>("datadog/agent:7.40.1") + new GenericContainer<>( + DockerImageName.parse(System.getProperty("test.datadog.agent.image"))) .withEnv(env) .withExposedPorts(DEFAULT_TRACE_AGENT_PORT) .withStartupTimeout(Duration.ofSeconds(120)) diff --git a/dd-trace-core/src/traceAgentTest/java/datadog/trace/common/metrics/MetricsIntegrationTest.java b/dd-trace-core/src/traceAgentTest/java/datadog/trace/common/metrics/MetricsIntegrationTest.java index 607512427ae..537de40bfb7 100644 --- a/dd-trace-core/src/traceAgentTest/java/datadog/trace/common/metrics/MetricsIntegrationTest.java +++ b/dd-trace-core/src/traceAgentTest/java/datadog/trace/common/metrics/MetricsIntegrationTest.java @@ -32,6 +32,7 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.testcontainers.containers.GenericContainer; import org.testcontainers.containers.startupcheck.MinimumDurationRunningStartupCheckStrategy; +import org.testcontainers.utility.DockerImageName; @ExtendWith(WithConfigExtension.class) class MetricsIntegrationTest { @@ -55,7 +56,8 @@ static void setupSpec() { env.put("DD_HOSTNAME", "doesnotexist"); env.put("DD_LOGS_STDOUT", "yes"); agentContainer = - new GenericContainer<>("datadog/agent:7.40.1") + new GenericContainer<>( + DockerImageName.parse(System.getProperty("test.datadog.agent.image"))) .withEnv(env) .withExposedPorts(ConfigDefaults.DEFAULT_TRACE_AGENT_PORT) .withStartupTimeout(Duration.ofSeconds(120)) diff --git a/docs/how_to_smoke_test.md b/docs/how_to_smoke_test.md index 445b2565f43..55e0484e59c 100644 --- a/docs/how_to_smoke_test.md +++ b/docs/how_to_smoke_test.md @@ -233,6 +233,41 @@ static final TestAgentBackend agent = AgentBackend.testAgentBuilder().retainAcro static final SmokeServerApp sender = /* ... */; ``` +## Container images + +For application containers or dependencies such as RabbitMQ, apply +`dd-trace-java.testcontainers` and declare the image in the smoke-test module: + +```kotlin +import datadog.buildlogic.testcontainers.image + +plugins { + id("dd-trace-java.module.smoke-test") + id("dd-trace-java.testcontainers") +} + +dependencies { + testImplementation("org.testcontainers:rabbitmq:${libs.versions.testcontainers.get()}") + testImplementation("org.testcontainers:junit-jupiter:${libs.versions.testcontainers.get()}") + testContainerImage(image("rabbitmq:3.12-management-alpine", "test.rabbitmq.image")) +} +``` + +Read the property when creating the container. Annotate the test class with +`@Testcontainers` so its `@Container` fields are started and stopped automatically: + +```java +@Container +private static final RabbitMQContainer RABBIT_MQ_CONTAINER = + new RabbitMQContainer( + DockerImageName.parse(System.getProperty("test.rabbitmq.image")) + .asCompatibleSubstituteFor("rabbitmq")); +``` + +With this plugin Gradle can now fingerprint the resolved image digest and passes +the immutable image to the test. Also, see the [plugin reference](../build-logic/testcontainers/README.md) +for inheritance and shared configuration examples. + ## Choosing a backend The backend is the agent stand-in the app reports to. diff --git a/docs/how_to_test.md b/docs/how_to_test.md index a0bcefdb368..e809fc38e7c 100644 --- a/docs/how_to_test.md +++ b/docs/how_to_test.md @@ -55,6 +55,58 @@ In order to identify such tests and avoid the continuous integration to fail, th > * using the `Retry` button from the job view: > ![Rerun workflow from failed](how_to_test/retry-failed-job.png) +## Tests that use containers + +> [!IMPORTANT] +> Don't use image name in Test Container constructors like `new CassandraContainer("cassandra:4")`, +> or `new GenericContainer("icr.io/appcafe/websphere-traditional:latest")`. Image tags can change. +> Also, these are not properly tracked as _test_ task inputs and as such can't be fingerprinted. +> Instead, use the `dd-trace-java.testcontainers` plugin to declare these as dependencies, +> it will resolve the actual image digest before running the test. + +Declare container images in the module's Gradle build so a changed image cannot +silently reuse cached test results: + +```kotlin +import datadog.buildlogic.testcontainers.image + +plugins { + id("dd-trace-java.testcontainers") +} + +dependencies { + testImplementation(libs.testcontainers) + testImplementation("com.redis.testcontainers:testcontainers-redis:1.6.2") + testContainerImage(image("redis:7-alpine", "test.redis.image")) +} +``` + +Use `GenericContainer` or the dedicated container type and use its `DockerImageName` constructor +overload with the relevant system property, **without a fallback value**. The compatibility +declaration is important to let _testcontainer_ know it should accept it as a mirrored image. +For example with Redis: + +```java +import com.redis.testcontainers.RedisContainer; +import org.testcontainers.utility.DockerImageName; + +DockerImageName image = DockerImageName.parse(System.getProperty("test.redis.image")) + .asCompatibleSubstituteFor("redis"); +RedisContainer redis = new RedisContainer(image); +``` + +Essentially the plugin resolves tags to immutable registry digests before Gradle checks +whether test results are up to date or cached. Resolution failure stops the test task +earlier, rather than within the tests. + +The plugin feed the system property to both `test` and `forkedTest` tasks. + +`testContainerImage` is a "companion" for the `testImplementation` configuration. +The plugin automatically creates an image configuration for each `*Implementation` +configuration. For example, `integrationTestImplementation` gets +`integrationTestContainerImage`. Also, see the [plugin reference](../build-logic/testcontainers/README.md) +for inheritance and shared configuration examples. + ## Running Tests You can run the whole project test suite using `./gradlew test` but expect it to take a certain time.