Track Testcontainers images as Gradle test inputs

build-logic/settings.gradle.kts

@@ -55,3 +55,4 @@ rootProject.name = "build-logic"
 
 include(":conventions")
 include(":smoke-test")
+include(":testcontainers")

build-logic/smoke-test/src/main/resources/datadog/buildlogic/smoketest/proxy-repositories.init.gradle.kts

@@ -1,12 +1,12 @@
 import org.gradle.api.Action
 import org.gradle.api.Project
+import org.gradle.kotlin.dsl.*
 import org.gradle.api.artifacts.dsl.RepositoryHandler
 import org.gradle.api.artifacts.repositories.MavenArtifactRepository
 import org.gradle.api.initialization.Settings
 import java.net.URI
 
-// Requires Gradle 6.8+ and assumes project repositories are allowed by the target build.
-
+// Requires Gradle 6.8+ for Settings.providers and Settings.dependencyResolutionManagement.
 gradle.beforeSettings(Action<Settings> {
   val gradlePluginProxy = providers.gradleProperty("gradlePluginProxy").orNull
   val mavenRepositoryProxy = providers.gradleProperty("mavenRepositoryProxy").orNull
@@ -23,6 +23,8 @@ gradle.beforeSettings(Action<Settings> {
     withType(MavenArtifactRepository::class.java).configureEach {
       // A repository declared without a URL has a null one until Gradle validates it; leave it be
       // so the nested build reports that itself instead of failing inside this init script.
+      // See https://github.com/gradle/gradle/issues/37612
+      @Suppress("UNNECESSARY_SAFE_CALL")
       val repositoryUrl = url?.toString()?.trimEnd('/')
       if (repositoryUrl != null && repositoryUrl in mavenCentralUrls) {
         url = URI(proxy)
@@ -33,6 +35,8 @@ gradle.beforeSettings(Action<Settings> {
 
   fun RepositoryHandler.removeDuplicateMavenProxy() {
     val proxyUrl = mavenRepositoryProxy?.takeIf { it.isNotBlank() }?.trimEnd('/') ?: return
+    // see https://github.com/gradle/gradle/issues/37612
+    @Suppress("UNNECESSARY_SAFE_CALL")
     val proxies = withType(MavenArtifactRepository::class.java)
       .filter { it.url?.toString()?.trimEnd('/') == proxyUrl }
     // Keep the injected repository: it is the only one known to be unrestricted, since a declared
@@ -92,7 +96,7 @@ gradle.beforeSettings(Action<Settings> {
     }
   })
 
-  gradle.afterProject(Action<Project> {
+  gradle.afterProject(Action {
     repositories.removeDuplicateMavenProxy()
   })
 })

build-logic/testcontainers/README.md

@@ -0,0 +1,156 @@
+# Container images as test inputs
+
+Apply this plugin only in modules that use containers:
+
+```kotlin
+import datadog.buildlogic.testcontainers.image
+
+plugins {
+  id("dd-trace-java.testcontainers")
+}
+
+dependencies {
+  testContainerImage(image("cassandra:4", "test.cassandra.image"))
+}
+```
+
+> [!NOTE]
+> `testContainerImage` is a configuration.
+> Note the `image` function import.
+
+Use `GenericContainer` or the dedicated container type and pass the relevant 
+system property through its `DockerImageName` constructor. Note the compatibility 
+declaration that let it accept a mirrored image without changing the resolved
+reference:
+
+```java
+import org.testcontainers.cassandra.CassandraContainer;
+import org.testcontainers.utility.DockerImageName;
+
+CassandraContainer container = new CassandraContainer(
+    DockerImageName.parse(System.getProperty("test.cassandra.image"))
+        .asCompatibleSubstituteFor("cassandra"));
+```
+
+`testContainerImage` is a dependency-scope configuration: it stores declarations
+and is neither resolvable nor consumable. `image("image", "system property name")` 
+creates a Gradle module dependency. 
+Note that these declarations stay outside Java classpaths. Before test task runs, the plugin
+resolves the images digest.
+
+The plugin identifies a test task's source sets from its `testClassesDirs` and
+reads their image configurations. Forked or differently named tasks running the
+same compiled tests therefore inherit the same images. Additional images can be
+declared in `<taskName>ContainerImage`.
+In practice the plugin automatically creates an image configuration for each `*Implementation` 
+configuration, including those added by JVM Test Suites. For example:
+
+```kotlin
+testing {
+  suites {
+    register<JvmTestSuite>("integrationTest") {
+      useJUnitJupiter()
+      project.dependencies {
+        add("integrationTestContainerImage", image("redis:7-alpine", "test.redis.image"))
+      }
+    }
+  }
+}
+```
+
+With the repository's legacy Groovy `addTestSuite` helper, declare the image in
+the project's `dependencies` block:
+
+```groovy
+addTestSuite('integrationTest')
+
+dependencies {
+  integrationTestContainerImage(image('redis:7-alpine', 'test.redis.image'))
+}
+```
+
+Automatic inheritance follows Java dependency configurations: if
+`latestDepTestImplementation` extends `testImplementation`, its tests also read
+`testContainerImage`. The plugin maps each configuration in that hierarchy to its
+image counterpart. It also reads parents declared with `extendsFrom` on image
+configurations:
+
+```kotlin
+val databaseImages = configurations.dependencyScope("databaseImages")
+configurations.testContainerImage {
+  extendsFrom(databaseImages.get())
+}
+dependencies {
+  add(databaseImages.name, image("postgres:16-alpine", "test.postgres.image"))
+}
+```
+
+The plugin ensures there’s only one image per system property for a given test task. 
+Declaring both `image("redis:7", "test.redis.image")` and `image("redis:8", "test.redis.image")`
+for that task, directly or through inherited configurations, will fail.
+
+Groovy DSL is declared the same `dependencies { testContainerImage(image('cassandra:4', 'test.cassandra.image')) }`.
+
+## Resolution and test execution
+
+The plugin resolves declared tags to immutable reference like `registry/repository@sha256:...`
+before Gradle checks the build-cache for reusable test results. These references are
+both test inputs and system property values, so actual code in the tests use the exact
+images.
+
+Like when java dependencies are declared with `+`, the tags are resolved on each invocation,
+including when reusing the configuration cache. With a build cache this means unchanged 
+references allow to safely reuse test results, while Gradle is blind when the image is 
+resolving within test code.
+
+> [!NOTE]
+> Only explicitly declared images can be tracked. Implicit Testcontainers helpers such as
+> Alpine/Ryuk are not included. This can't be avoided.
+
+## Local registries and CI mirrors
+
+This plugin honors `TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX` environment variable.
+
+Locally, when asking for `cassandra:4` it is resolved from Docker Hub. 
+However, on [CI](../../.gitlab-ci.yml), the env var `TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX=...` is set, 
+so the plugin resolves and fingerprints from the prefix, instead, e.g.:
+
+```text
+Local: registry-1.docker.io/library/cassandra@sha256:...
+CI:    registry.ddbuild.io/images/mirror/cassandra@sha256:...
+```
+
+More precisely:
+- Prefix precedence: the test task's `TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX`, then
+  `hub.image.name.prefix` in `~/.testcontainers.properties`, then
+  `testcontainers.properties` in test classpath directories.
+- Explicit registry hosts bypass the prefix. Other mappings belong in the image
+  declaration, as in [JDBC's SQL Server example](../../dd-java-agent/instrumentation/jdbc/build.gradle).
+- Private registries use Docker's `config.json` and credential helpers, honoring
+  `DOCKER_CONFIG`. Remote registries require TLS; loopback registries also allow
+  local development certificates and HTTP.
+
+## Testcontainers configuration restrictions
+
+Testcontainers supports [configuration properties and environment variables](https://java.testcontainers.org/features/configuration/)
+that change its runtime behavior. In particular, its [image-substitution settings](https://java.testcontainers.org/features/image_name_substitution/)
+can replace the images used when containers start.
+
+> [!IMPORTANT]
+> This plugin forbids custom image substitutors (`image.substitutor`) and all
+> `*.container.image` overrides: **the test task fails when these settings are
+> detected**. Runtime replacements could make the tested image differ from the one
+> in Gradle's cache key. This restriction includes helper-image overrides such as
+> `ryuk.container.image`.
+
+The check covers environment variables, `~/.testcontainers.properties`, and
+`testcontainers.properties` in test classpath directories. The Docker Hub prefix
+described above remains supported.
+
+For declared test images, choose the replacement directly in the declaration,
+for example `image("registry.example/redis:7", "test.redis.image")`. This keeps the
+image used by the test consistent with the image in Gradle's cache key.
+
+The plugin does not inspect `testcontainers.properties` inside dependency JARs.
+Testcontainers can still load substitutions from those files; they escape this
+check and can invalidate cache correctness.

build-logic/testcontainers/build.gradle.kts

@@ -0,0 +1,54 @@
+plugins {
+  `java-gradle-plugin`
+  `kotlin-dsl`
+  alias(libs.plugins.shadow)
+}
+
+val jib = configurations.register("jib") {
+  dependencies.add(project.dependencies.create("com.google.cloud.tools:jib-core:0.28.2"))
+}
+configurations.compileOnly { extendsFrom(jib) }
+
+// buildSrc exports an older HttpClient through the parent classloader. Keep Jib's
+// dependencies private, including when this plugin is consumed as an included build.
+tasks.shadowJar {
+  configurations.add(jib)
+  enableAutoRelocation = true
+  relocationPrefix = "datadog.buildlogic.testcontainers.internal"
+  mergeServiceFiles()
+}
+configurations.apiElements {
+  outgoing.artifacts.clear()
+  outgoing.variants.clear()
+  outgoing.artifact(tasks.shadowJar)
+}
+configurations.runtimeElements {
+  outgoing.artifacts.clear()
+  outgoing.variants.clear()
+  outgoing.artifact(tasks.shadowJar)
+}
+tasks.pluginUnderTestMetadata {
+  pluginClasspath.setFrom(tasks.shadowJar)
+}
+
+gradlePlugin {
+  plugins {
+    create("testcontainers") {
+      id = "dd-trace-java.testcontainers"
+      implementationClass = "datadog.buildlogic.testcontainers.TestcontainersPlugin"
+    }
+  }
+}
+
+testing {
+  suites {
+    named<JvmTestSuite>("test") {
+      useJUnitJupiter(libs.versions.junit5)
+      dependencies {
+        implementation(libs.assertj.core)
+        implementation(libs.okhttp3.mockwebserver)
+        implementation("com.squareup.okhttp3:okhttp-tls:${libs.versions.okhttp3.testing.get()}")
+      }
+    }
+  }
+}

build-logic/testcontainers/src/main/kotlin/datadog/buildlogic/testcontainers/ContainerImageArguments.kt

@@ -0,0 +1,33 @@
+package datadog.buildlogic.testcontainers
+
+import org.gradle.api.provider.Provider
+import org.gradle.api.services.ServiceReference
+import org.gradle.api.tasks.Input
+import org.gradle.api.tasks.Internal
+import org.gradle.api.tasks.Nested
+import org.gradle.api.tasks.Optional
+import org.gradle.process.CommandLineArgumentProvider
+import java.io.File
+
+class ContainerImageArguments(
+  @get:Nested @get:Optional val containers: Provider<ContainerImageInputs>,
+) : CommandLineArgumentProvider {
+  override fun asArguments(): Iterable<String> =
+    containers.orNull
+      ?.images
+      ?.map { (name, image) -> "-D$name=$image" }
+      .orEmpty()
+}
+
+class ContainerImageInputs(
+  @get:Internal val declarations: Map<String, String>,
+  @get:Internal val imageEnvironment: Map<String, String>,
+  @get:Internal val configurationFiles: List<File>,
+  @get:ServiceReference("testContainerImageResolver") val resolver: Provider<ImageResolver>,
+) {
+  // Read during Test input snapshotting, after skip predicates and before cache lookup.
+  // Only the service reference is serialized by the configuration cache; its memo is per build.
+  @get:Input
+  val images: Map<String, String>
+    get() = resolver.get().resolve(declarations, imageEnvironment, configurationFiles)
+}

build-logic/testcontainers/src/main/kotlin/datadog/buildlogic/testcontainers/ContainerImages.kt

@@ -0,0 +1,27 @@
+package datadog.buildlogic.testcontainers
+
+import org.gradle.api.InvalidUserDataException
+import org.gradle.api.artifacts.ExternalModuleDependency
+import org.gradle.api.artifacts.dsl.DependencyHandler
+import org.gradle.api.attributes.Attribute
+
+internal const val IMAGE_DEPENDENCY_GROUP = "datadog.container-image"
+internal val IMAGE_REFERENCE = Attribute.of("datadog.container-image.reference", String::class.java)
+
+/** Creates an image declaration for the test JVM property that consumes it. */
+fun DependencyHandler.image(
+  reference: String,
+  systemProperty: String,
+): ExternalModuleDependency {
+  if (reference.isBlank()) {
+    throw InvalidUserDataException("Container image reference must not be empty")
+  }
+  if (!systemProperty.matches(Regex("[A-Za-z0-9_.-]+"))) {
+    throw InvalidUserDataException("Invalid container image system property: $systemProperty")
+  }
+
+  // These logical dependencies are read from declaration-only configurations, never resolved by Gradle.
+  return (create("$IMAGE_DEPENDENCY_GROUP:$systemProperty") as ExternalModuleDependency).apply {
+    attributes { attribute(IMAGE_REFERENCE, reference) }
+  }
+}