Experiment with testing of dd-trace-java on GitLab on Windows OS

.gitlab-ci.yml

@@ -1,5 +1,6 @@
 include:
   - local: ".gitlab/one-pipeline.locked.yml"
+  - local: ".gitlab/windows-tests.yml"
   - local: ".gitlab/benchmarks.yml"
   - local: ".gitlab/exploration-tests.yml"
   - local: ".gitlab/ci-visibility-tests.yml"
@@ -39,6 +40,7 @@ stages:
   - benchmarks
   - tests
   - tests-arm64
+  - tests-windows
   - test-summary
   - exploration-tests
   - ci-visibility-tests

.gitlab/windows-tests.yml

@@ -0,0 +1,269 @@
+variables:
+  WINDOWS_BUILD_IMAGE: "registry.ddbuild.io/ci/dd-trace-java/dd-trace-java-windows-docker-build:prototype-alexeyk-gitlab-windows-tests"
+
+.windows_gradle_cache_paths: &windows_gradle_cache_paths
+  - .gradle/wrapper/
+  - .gradle/caches/
+  - .gradle/notifications/
+
+build-windows-ci-image:
+  stage: build
+  timeout: 2h
+  tags: [ "windows-v2:2025" ]
+  when: manual
+  allow_failure: true
+  hooks:
+    pre_get_sources_script:
+      - git config --system core.longpaths true
+      - git config --global core.autocrlf false
+  script:
+    - |
+      . .gitlab/windows/ci-common.ps1
+
+      $image = Get-WindowsCiImage
+      Write-Output "Target image: $image"
+
+      $buildArguments = @(
+        "build",
+        "--file", ".gitlab/windows/image/Dockerfile",
+        "--tag", $image
+      )
+
+      Invoke-Native docker @("pull", "--quiet", $image)
+      if ($LASTEXITCODE -eq 0) {
+        $buildArguments += @("--cache-from", $image)
+      } else {
+        Write-Output "No existing prototype image found; building without a registry layer cache."
+      }
+
+      $buildArguments += ".gitlab/windows/image"
+      Invoke-Native docker $buildArguments
+      if ($LASTEXITCODE -ne 0) { throw "docker build failed" }
+
+      Invoke-Native docker @("push", $image)
+      if ($LASTEXITCODE -ne 0) { throw "docker push $image failed" }
+  id_tokens:
+    CI_IDENTITIES_GITLAB_ID_TOKEN:
+      aud: ci-identities
+
+test-base-windows:
+  stage: tests-windows
+  timeout: 2h
+  tags: [ "windows-v2:2025" ]
+  hooks:
+    pre_get_sources_script:
+      - git config --system core.longpaths true
+      - git config --global core.autocrlf false
+  needs:
+    - job: build
+      artifacts: false
+  # Reads the prototype image from registry.ddbuild.io, so it needs the same CI
+  # identity as the job that publishes it.
+  id_tokens:
+    CI_IDENTITIES_GITLAB_ID_TOKEN:
+      aud: ci-identities
+  variables:
+    GIT_SUBMODULE_STRATEGY: normal
+    GIT_SUBMODULE_DEPTH: 1
+    GRADLE_TARGET: ":baseTest"
+    GRADLE_WORKERS: "4"
+    CACHE_TYPE: "base"
+    # Feature branches only read the shared dependency seed; protected refs
+    # refresh it (overridden in rules below).
+    WINDOWS_SEED_CACHE_POLICY: pull
+  parallel:
+    matrix:
+      - testJvm: [ "21" ]
+        CI_SPLIT: [ "1/4", "2/4", "3/4", "4/4" ]
+  rules:
+    - if: '$POPULATE_CACHE'
+      when: never
+    - if: '$CI_COMMIT_BRANCH =~ /^mq-working-branch-/'
+      when: on_success
+      allow_failure: true
+      variables:
+        WINDOWS_SEED_CACHE_POLICY: pull-push
+    - if: '$CI_COMMIT_BRANCH == "master"'
+      when: on_success
+      allow_failure: true
+      variables:
+        WINDOWS_SEED_CACHE_POLICY: pull-push
+    - if: '$CI_COMMIT_BRANCH'
+      when: manual
+      allow_failure: true
+  cache:
+    # Shared dependency seed. Only protected refs write it, so a new feature
+    # branch does not resolve every dependency from scratch on all four
+    # partitions. unprotect:true is what makes the seed readable from
+    # unprotected refs; the pull-only policy keeps them from poisoning it.
+    - key:
+        files:
+          - gradle/wrapper/gradle-wrapper.properties
+          - settings.gradle.kts
+        prefix: "windows-gradle-seed-$CACHE_TYPE-$CI_NODE_INDEX"
+      paths: *windows_gradle_cache_paths
+      policy: $WINDOWS_SEED_CACHE_POLICY
+      unprotect: true
+    # Per-branch, per-partition cache. Restored after the seed, so it wins.
+    - key:
+        files:
+          - gradle/wrapper/gradle-wrapper.properties
+          - settings.gradle.kts
+        prefix: "windows-gradle-$CACHE_TYPE-$CI_COMMIT_REF_SLUG-$CI_NODE_INDEX"
+      paths: *windows_gradle_cache_paths
+      policy: pull-push
+  script:
+    - |
+      . .gitlab/windows/ci-common.ps1
+
+      $image = Get-WindowsCiImage
+      Write-Output "Expected Windows test image: $image"
+      Invoke-Native docker @("pull", "--quiet", $image)
+      if ($LASTEXITCODE -ne 0) {
+        Write-Host "ERROR: Could not pull Windows test image $image."
+        Write-Host "Manually run build-windows-ci-image, wait for it to finish, then retry this job."
+        exit 1
+      }
+
+      $diagnosticsDir = Join-Path $env:CI_PROJECT_DIR ".tmp"
+      New-Item -ItemType Directory -Force -Path $diagnosticsDir, ".gradle" | Out-Null
+
+      $containerName = "dd-trace-java-windows-$($env:CI_JOB_ID)"
+      $dockerArguments = @(
+        "run", "--rm",
+        "--name", $containerName,
+        "--memory", "20g",
+        # Windows reports the host's logical processors unless the count is
+        # capped, which makes thread-pool-sized assertions non-deterministic.
+        "--cpu-count", "4",
+        "--workdir", "C:\work",
+        "--mount", "type=bind,source=$($env:CI_PROJECT_DIR),target=C:\work",
+        "--mount", "type=bind,source=$diagnosticsDir,target=C:\tmp",
+        "--env", "CI=true",
+        "--env", "GITLAB_CI=true",
+        "--env", "CI_JOB_ID=$($env:CI_JOB_ID)",
+        "--env", "CI_PIPELINE_ID=$($env:CI_PIPELINE_ID)",
+        "--env", "CI_COMMIT_SHA=$($env:CI_COMMIT_SHA)",
+        "--env", "CI_COMMIT_BRANCH=$($env:CI_COMMIT_BRANCH)",
+        "--env", "CI_SPLIT=$($env:CI_SPLIT)",
+        "--env", "GRADLE_TARGET=$($env:GRADLE_TARGET)",
+        "--env", "GRADLE_WORKERS=$($env:GRADLE_WORKERS)",
+        "--env", "testJvm=$($env:testJvm)",
+        "--env", "MAVEN_REPOSITORY_PROXY=$($env:MAVEN_REPOSITORY_PROXY)",
+        "--env", "GRADLE_PLUGIN_PROXY=$($env:GRADLE_PLUGIN_PROXY)",
+        "--env", "MASS_READ_URL=$($env:MASS_READ_URL)",
+        # Parity with .test_job_common on Linux: both runtimes size pools from
+        # availableProcessors(), which is unreliable in containers.
+        "--env", "JAVA_TOOL_OPTIONS=-XX:ActiveProcessorCount=4",
+        "--env", "RUNTIME_AVAILABLE_PROCESSORS_OVERRIDE=4",
+        "--env", "JETTY_AVAILABLE_PROCESSORS=4",
+        "--env", "TESTCONTAINERS_CHECKS_DISABLE=true",
+        "--env", "TESTCONTAINERS_RYUK_DISABLED=true",
+        "--env", "TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX=registry.ddbuild.io/images/mirror/",
+        $image,
+        "powershell", "-NoLogo", "-NoProfile", "-ExecutionPolicy", "Bypass",
+        "-File", "C:\work\.gitlab\windows\run-base-tests.ps1"
+      )
+
+      Invoke-Native docker $dockerArguments
+      if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
+  after_script:
+    - |
+      $containerName = "dd-trace-java-windows-$($env:CI_JOB_ID)"
+      docker rm --force $containerName 2>$null
+      exit 0
+  artifacts:
+    when: always
+    paths:
+      - workspace/**/build/test-results/**
+      - workspace/**/build/reports/tests/**
+      - buildSrc/build/test-results/**
+      - buildSrc/build/reports/tests/**
+      - .gradle/daemon/*/*.out.log
+      - .gradle/workers/hs_err_pid*.log
+      - .tmp/**
+    reports:
+      junit:
+        - workspace/**/build/test-results/**/*.xml
+        - buildSrc/build/test-results/**/*.xml
+
+# Keep the smoke and instrumentation rollout opt-in on every ref. Unlike their Linux
+# counterparts, these jobs intentionally do not enable the APM Test Agent sidecar;
+# cross-container Test Agent connectivity can be added as a separate step.
+test-smoke-windows:
+  extends: test-base-windows
+  variables:
+    GRADLE_TARGET: "stageMainDist :smokeTest"
+    # Smoke tasks start additional JVMs and Docker processes, so leave one
+    # worker of headroom inside the 20 GiB Windows container.
+    GRADLE_WORKERS: "3"
+    CACHE_TYPE: "smoke"
+  parallel:
+    matrix:
+      - testJvm: [ "21" ]
+        CI_SPLIT: [ "1/8", "2/8", "3/8", "4/8", "5/8", "6/8", "7/8", "8/8" ]
+  rules:
+    - if: '$POPULATE_CACHE'
+      when: never
+    - if: '$CI_COMMIT_BRANCH =~ /^mq-working-branch-/'
+      when: manual
+      allow_failure: true
+      variables:
+        WINDOWS_SEED_CACHE_POLICY: pull-push
+    - if: '$CI_COMMIT_BRANCH == "master"'
+      when: manual
+      allow_failure: true
+      variables:
+        WINDOWS_SEED_CACHE_POLICY: pull-push
+    - when: manual
+      allow_failure: true
+
+test-inst-windows:
+  extends: test-base-windows
+  variables:
+    GRADLE_TARGET: ":instrumentationTest"
+    CACHE_TYPE: "inst"
+  parallel:
+    matrix:
+      - testJvm: [ "21" ]
+        CI_SPLIT: [ "1/8", "2/8", "3/8", "4/8", "5/8", "6/8", "7/8", "8/8" ]
+  rules:
+    - if: '$POPULATE_CACHE'
+      when: never
+    - if: '$CI_COMMIT_BRANCH =~ /^mq-working-branch-/'
+      when: manual
+      allow_failure: true
+      variables:
+        WINDOWS_SEED_CACHE_POLICY: pull-push
+    - if: '$CI_COMMIT_BRANCH == "master"'
+      when: manual
+      allow_failure: true
+      variables:
+        WINDOWS_SEED_CACHE_POLICY: pull-push
+    - when: manual
+      allow_failure: true
+
+test-inst-latest-windows:
+  extends: test-base-windows
+  variables:
+    GRADLE_TARGET: ":instrumentationLatestDepTest"
+    CACHE_TYPE: "latestdep"
+  parallel:
+    matrix:
+      - testJvm: [ "21" ]
+        CI_SPLIT: [ "1/6", "2/6", "3/6", "4/6", "5/6", "6/6" ]
+  rules:
+    - if: '$POPULATE_CACHE'
+      when: never
+    - if: '$CI_COMMIT_BRANCH =~ /^mq-working-branch-/'
+      when: manual
+      allow_failure: true
+      variables:
+        WINDOWS_SEED_CACHE_POLICY: pull-push
+    - if: '$CI_COMMIT_BRANCH == "master"'
+      when: manual
+      allow_failure: true
+      variables:
+        WINDOWS_SEED_CACHE_POLICY: pull-push
+    - when: manual
+      allow_failure: true

.gitlab/windows/README.md

@@ -0,0 +1,60 @@
+# Windows GitLab prototype
+
+This directory contains an experimental Windows test job and its repo-local CI image.
+The image contains MinGit and the Temurin 8, 11, 17, 21, and 25 JDK toolchains used by
+the Gradle build. JDK 21 is the default daemon and test JVM. The base test scope runs
+`:baseTest` on Java 21, split into the same four partitions as the existing `test_base`
+job. Three fully manual scopes run `stageMainDist :smokeTest` and `:instrumentationTest`
+on Java 21 in eight partitions, plus `:instrumentationLatestDepTest` on Java 21 in six
+partitions. Together they validate the execution model before moving the image to
+`dd-trace-java-docker-build`.
+
+## Running the prototype
+
+1. Push the branch and open its GitLab pipeline.
+2. Manually run `build-windows-ci-image`.
+3. Wait for the image to be pushed to
+   `registry.ddbuild.io/ci/dd-trace-java/dd-trace-java-windows-docker-build:prototype-alexeyk-gitlab-windows-tests`.
+4. Run or retry the four `test-base-windows` matrix jobs.
+5. Trigger the desired `test-smoke-windows` partitions manually. Starting with `1/8`
+   provides a small plumbing check before running the other seven partitions.
+6. Trigger the desired `test-inst-windows` partitions manually. Starting with `1/8`
+   provides a small plumbing check before running the other seven partitions.
+7. Trigger the desired `test-inst-latest-windows` partitions manually. Starting with
+   `1/6` provides the equivalent plumbing check before running the other five partitions.
+
+The image producer always overwrites this single mutable prototype tag and uses the
+previous image as its Docker layer cache. Test jobs explicitly pull the tag before use,
+so a long-lived Windows runner does not reuse a stale local copy.
+
+The base test job is manual and non-blocking on feature branches. It runs automatically
+but remains non-blocking on merge-queue branches and `master`. Smoke and instrumentation
+jobs are manual and non-blocking on every ref during the initial rollout.
+
+## Updating the image
+
+Run `build-windows-ci-image` after changing anything under `image/`. The same prototype
+tag is replaced after a successful build, which avoids accumulating per-experiment tags.
+The test job fails with an instruction to run the producer when it cannot pull that tag.
+
+## Caching
+
+Test jobs read `.gradle/{wrapper,caches,notifications}` from a category-specific seed
+cache that only protected refs write, then push a per-branch, per-partition cache on top.
+A new feature branch therefore starts from the last protected-ref dependency set instead
+of resolving everything separately in every partition.
+
+## Notes on Windows
+
+- All `docker` calls go through `Invoke-Native` in `ci-common.ps1`. PowerShell turns a
+  native command's stderr into a terminating error under `$ErrorActionPreference = 'Stop'`,
+  and docker writes progress and "manifest unknown" to stderr.
+- The container gets `LongPathsEnabled` and `core.longpaths`; the deepest relocated build
+  outputs under `C:\work\workspace\...` exceed the 260-character `MAX_PATH` default.
+- Temurin publishes no JDK 21 newer than `21.0.12+8` for `windowsservercore-ltsc2025`,
+  while 8, 11, 17, and 25 are current. Expect some failures on the Java 21 matrix to be
+  JDK-version artifacts rather than Windows-specific.
+- The initial smoke and instrumentation jobs do not set `CI_USE_TEST_AGENT`. The Linux
+  Test Agent sidecar is not reachable from the explicitly launched Windows test container
+  without additional networking support; instrumentation tests still use their in-memory
+  writer for their primary assertions.

.gitlab/windows/ci-common.ps1

@@ -0,0 +1,33 @@
+# Shared helpers for the Windows CI jobs in .gitlab/windows-tests.yml.
+# Dot-source this file at the top of a job script: . .gitlab/windows/ci-common.ps1
+
+$ErrorActionPreference = "Stop"
+
+# Runs a native command with its stderr merged into stdout.
+#
+# PowerShell wraps a native command's stderr in ErrorRecord objects, and under
+# $ErrorActionPreference = 'Stop' the first one becomes a terminating error.
+# docker writes build progress, push progress and "manifest unknown" to stderr,
+# so calling it directly would fail the job on paths that are expected to work.
+# Callers must gate on $LASTEXITCODE, which this function leaves untouched.
+#
+# Arguments are passed as a single array rather than splatted, so tokens such as
+# "--tag" reach the command instead of being bound as function parameters.
+function Invoke-Native {
+    param(
+        [Parameter(Mandatory = $true)][string] $Command,
+        [string[]] $Arguments = @()
+    )
+
+    # Function-scoped, so the caller keeps 'Stop' for cmdlet errors.
+    $ErrorActionPreference = "Continue"
+    & $Command @Arguments 2>&1 | ForEach-Object { Write-Host "$_" }
+}
+
+# Resolves the mutable image tag used by this personal prototype.
+function Get-WindowsCiImage {
+    if ([string]::IsNullOrWhiteSpace($env:WINDOWS_BUILD_IMAGE)) {
+        throw "WINDOWS_BUILD_IMAGE is not set"
+    }
+    return $env:WINDOWS_BUILD_IMAGE
+}