diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bf87d739e1d..064bd7f84f8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,7 +5,7 @@ include: - local: ".gitlab/ci-visibility-tests.yml" - project: 'DataDog/apm-reliability/apm-sdks-benchmarks' file: '.gitlab/ci-java-spring-petclinic-parallel.yml' - ref: &apm_sdks_benchmarks_sha '5e416ea523f39bbdf9e44f39e6895a07d3715ff5' # pinned by .github/workflows/update-apm-sdks-benchmarks-reference.yaml + ref: &apm_sdks_benchmarks_sha "12a3157e4e36ad5530c283c0e751f1fe71560192" # pinned by .github/workflows/update-apm-sdks-benchmarks-reference.yaml - project: 'DataDog/apm-reliability/apm-sdks-benchmarks' file: '.gitlab/ci-java-load-parallel.yml' ref: *apm_sdks_benchmarks_sha @@ -294,12 +294,7 @@ maven-central-pre-release-check: image: ${BUILDER_IMAGE_REPO}:${BUILDER_IMAGE_VERSION_PREFIX}base stage: .pre rules: - - if: '$CI_COMMIT_BRANCH == "master"' - when: on_success - allow_failure: false - - if: '$CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+$/' - when: on_success - allow_failure: false + - when: never script: - | MAVEN_CENTRAL_USERNAME=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.central_username --with-decryption --query "Parameter.Value" --out text) @@ -323,11 +318,7 @@ dd-octo-sts-pre-release-check: DDOCTOSTS_ID_TOKEN: aud: dd-octo-sts rules: - - if: '$POPULATE_CACHE' - when: never - - if: '$CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+$/' - when: on_success - allow_failure: false + - when: never before_script: - dd-octo-sts version - dd-octo-sts debug --scope DataDog/dd-trace-java --policy self.gitlab.release @@ -372,6 +363,8 @@ build: dotenv: build.env build_tests: + rules: + - when: never extends: .gradle_build variables: <<: *tier_xl_variables @@ -406,8 +399,7 @@ populate_plugin_cache: when: always unprotect: true rules: - - if: '$POPULATE_CACHE' - when: on_success + - when: never script: - mkdir -p .gradle-plugin-cache - sudo chown -R 1001:1001 .gradle-plugin-cache @@ -441,8 +433,7 @@ populate_dep_cache: - *dependency_cache - *build_cache rules: - - if: '$POPULATE_CACHE' - when: on_success + - when: never before_script: - !reference [.gradle_build, before_script] # Seed the otherwise cold writable cache, keeping the cache uploaded below self-contained. @@ -513,6 +504,8 @@ publish-artifacts-to-s3: spotless: + rules: + - when: never extends: .gradle_build stage: tests needs: [] @@ -526,6 +519,8 @@ spotless: - cd test-published-dependencies && ./gradlew spotlessCheck $GRADLE_ARGS check-instrumentation-naming: + rules: + - when: never extends: .gradle_build stage: tests needs: [ ] @@ -534,6 +529,8 @@ check-instrumentation-naming: - ./gradlew checkInstrumentationNaming config-inversion-linter: + rules: + - when: never extends: .gradle_build stage: tests needs: [] @@ -542,6 +539,8 @@ config-inversion-linter: - ./gradlew checkConfigurations test_published_artifacts: + rules: + - when: never extends: .gradle_build image: ${BUILDER_IMAGE_REPO}:${BUILDER_IMAGE_VERSION_PREFIX}7 # Needs Java7 for some tests stage: tests @@ -572,6 +571,8 @@ test_published_artifacts: - ./check_reports validate_build: + rules: + - when: never extends: .gradle_build stage: tests needs: [ build ] @@ -647,17 +648,23 @@ validate_build: - data_integrity_failure check_build_src: + rules: + - when: never extends: .check_job needs: [] variables: GRADLE_TARGET: ":buildSrc:build" check_base: + rules: + - when: never extends: .check_job variables: GRADLE_TARGET: ":baseCheck" check_inst: + rules: + - when: never extends: .check_job parallel: 4 variables: @@ -665,6 +672,8 @@ check_inst: CACHE_TYPE: "inst" check_smoke: + rules: + - when: never extends: .check_job parallel: 4 variables: @@ -672,11 +681,15 @@ check_smoke: CACHE_TYPE: "smoke" check_profiling: + rules: + - when: never extends: .check_job variables: GRADLE_TARGET: ":profilingCheck" check_debugger: + rules: + - when: never extends: .check_job variables: GRADLE_TARGET: ":debuggerCheck" @@ -694,11 +707,7 @@ muzzle: CACHE_TYPE: "inst" stage: tests rules: - - if: '$CI_COMMIT_BRANCH =~ /^mq-working-branch-/' - when: never - - if: '$CI_COMMIT_BRANCH =~ /^gh-readonly-queue/' - when: never - - when: on_success + - when: never parallel: matrix: - CI_SPLIT: ["1/8", "2/8", "3/8", "4/8", "5/8", "6/8", "7/8", "8/8"] @@ -732,11 +741,7 @@ muzzle-dep-report: needs: *needs_build_tests_inst stage: tests rules: - - if: '$CI_COMMIT_BRANCH =~ /^mq-working-branch-/' - when: never - - if: '$CI_COMMIT_BRANCH =~ /^gh-readonly-queue/' - when: never - - when: on_success + - when: never variables: CACHE_TYPE: "inst" script: @@ -938,6 +943,8 @@ muzzle-dep-report: - .test_job_with_test_agent_common agent_integration_tests: + rules: + - when: never extends: .test_job_amd64 tags: [ "arch:amd64" ] variables: @@ -955,6 +962,8 @@ agent_integration_tests: DD_API_KEY: "invalid_key_but_this_is_fine" test_base: + rules: + - when: never extends: .test_job_amd64 # needs:parallel:matrix limits this job to a specific build_tests combination. # Keep matrix vars exact and in build_tests declaration order: @@ -975,6 +984,8 @@ test_base: - !reference [.test_job_common, script] test_base_arm64: + rules: + - when: never extends: .test_job_arm64 variables: GRADLE_TARGET: ":baseTest" @@ -987,6 +998,8 @@ test_base_arm64: - !reference [.test_job_common, script] test_inst: + rules: + - when: never extends: .test_job_amd64_with_test_agent needs: *needs_build_tests_inst variables: @@ -997,6 +1010,8 @@ test_inst: matrix: *test_matrix_8 test_inst_arm64: + rules: + - when: never extends: .test_job_arm64_with_test_agent variables: <<: *tier_l_variables @@ -1006,6 +1021,8 @@ test_inst_arm64: matrix: *test_matrix_8 test_inst_latest: + rules: + - when: never extends: .test_job_amd64_with_test_agent # needs:parallel:matrix limits this job to a specific build_tests combination. # Keep matrix vars exact and in build_tests declaration order: @@ -1028,6 +1045,8 @@ test_inst_latest: CI_SPLIT: [ "1/6", "2/6", "3/6", "4/6", "5/6", "6/6"] test_inst_latest_arm64: + rules: + - when: never extends: .test_job_arm64_with_test_agent variables: <<: *tier_l_variables @@ -1048,9 +1067,7 @@ test_flaky: testJvm: "8" CONTINUE_ON_FAILURE: "true" rules: - - *master_only - - if: $RUN_FLAKY_TESTS == "true" - when: on_success + - when: never parallel: matrix: - GRADLE_TARGET: [":baseTest", ":smokeTest", ":debuggerTest"] @@ -1068,12 +1085,12 @@ test_flaky_inst: testJvm: "8" CONTINUE_ON_FAILURE: "true" rules: - - *master_only - - if: $RUN_FLAKY_TESTS == "true" - when: on_success + - when: never parallel: 6 test_profiling: + rules: + - when: never extends: .test_job_amd64 # needs:parallel:matrix limits this job to a specific build_tests combination. # Keep matrix vars exact and in build_tests declaration order: @@ -1091,6 +1108,8 @@ test_profiling: matrix: *test_matrix test_profiling_arm64: + rules: + - when: never extends: .test_job_arm64 variables: GRADLE_TARGET: ":profilingTest" @@ -1101,6 +1120,8 @@ test_profiling_arm64: # specific jvms list for debugger project because J9-based JVMs have issues with local vars # so need to test at least against one J9-based JVM test_debugger: + rules: + - when: never extends: .test_job_amd64 variables: GRADLE_TARGET: ":debuggerTest" @@ -1115,6 +1136,8 @@ test_debugger: - !reference [.test_job_common, script] test_debugger_arm64: + rules: + - when: never extends: .test_job_arm64 variables: GRADLE_TARGET: ":debuggerTest" @@ -1128,6 +1151,8 @@ test_debugger_arm64: - !reference [.test_job_common, script] test_smoke: + rules: + - when: never extends: .test_job_amd64_with_test_agent # needs:parallel:matrix limits this job to a specific build_tests combination. # Keep matrix vars exact and in build_tests declaration order: @@ -1148,6 +1173,8 @@ test_smoke: matrix: *test_matrix_8 test_smoke_arm64: + rules: + - when: never extends: .test_job_arm64_with_test_agent variables: <<: *tier_l_variables @@ -1161,12 +1188,7 @@ test_ssi_smoke: extends: .test_job_amd64 needs: *needs_build_tests_smoke rules: - - if: $CI_COMMIT_BRANCH == "master" - when: on_success - - if: '$CI_COMMIT_BRANCH =~ /^mq-working-branch-/' - when: on_success - - if: '$CI_COMMIT_BRANCH =~ /^gh-readonly-queue/' - when: on_success + - when: never variables: <<: *tier_l_variables GRADLE_TARGET: "stageMainDist :smokeTest" @@ -1177,6 +1199,8 @@ test_ssi_smoke: matrix: *test_matrix_8 test_ssi_smoke_arm64: + rules: + - when: never extends: .test_job_arm64 variables: <<: *tier_l_variables @@ -1188,6 +1212,8 @@ test_ssi_smoke_arm64: matrix: *test_matrix_8 test_smoke_graalvm: + rules: + - when: never extends: .test_job_amd64 needs: *needs_build_tests_smoke tags: [ "arch:amd64" ] @@ -1202,6 +1228,8 @@ test_smoke_graalvm: - testJvm: ["graalvm17", "graalvm21", "graalvm25"] test_smoke_graalvm_arm64: + rules: + - when: never extends: .test_job_arm64 tags: [ "arch:arm64" ] variables: @@ -1215,6 +1243,8 @@ test_smoke_graalvm_arm64: - testJvm: ["graalvm17", "graalvm21", "graalvm25"] test_smoke_semeru8_debugger: + rules: + - when: never extends: .test_job_amd64 needs: *needs_build_tests_smoke tags: [ "arch:amd64" ] @@ -1230,16 +1260,7 @@ aggregate_test_counts: # Keep stage ordering, but prevent GitLab from downloading all previous-stage artifacts. dependencies: [] rules: - - if: '$POPULATE_CACHE' - when: never - - if: '$CI_COMMIT_BRANCH == "master"' - when: on_success - - if: '$CI_COMMIT_BRANCH =~ /^mq-working-branch-/' - when: on_success - - if: '$CI_COMMIT_BRANCH =~ /^gh-readonly-queue/' - when: on_success - - if: '$CI_COMMIT_BRANCH' - when: on_success + - when: never script: - *set_datadog_api_keys - export TEST_COUNTS_S3_PREFIX="test-counts/${CI_PIPELINE_ID}" @@ -1259,14 +1280,7 @@ deploy_to_profiling_backend: stage: publish needs: [ build ] rules: - - if: '$POPULATE_CACHE' - when: never - - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' - when: on_success - - if: '$CI_COMMIT_TAG =~ /^v.*/' - when: on_success - - when: manual - allow_failure: true + - when: never trigger: project: DataDog/profiling-backend branch: dogfooding @@ -1301,6 +1315,8 @@ deploy_to_di_backend:manual: UPSTREAM_COMMIT_SHORT_SHA: $CI_COMMIT_SHORT_SHA deploy_to_reliability_env: + rules: + - when: never needs: [ build ] # If the deploy_to_maven_central job is re-run, re-trigger the deploy_artifacts_to_github job as well so that the artifacts match. @@ -1314,16 +1330,7 @@ deploy_to_maven_central: variables: CACHE_TYPE: "lib" rules: - - if: '$POPULATE_CACHE' - when: never - - if: '$CI_COMMIT_BRANCH == "master"' - when: on_success - # Do not deploy release candidate versions - - if: '$CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+$/' - when: on_success - allow_failure: true - - when: manual - allow_failure: true + - when: never script: - export MAVEN_CENTRAL_USERNAME=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.central_username --with-decryption --query "Parameter.Value" --out text) - export MAVEN_CENTRAL_PASSWORD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.central_password --with-decryption --query "Parameter.Value" --out text) @@ -1343,11 +1350,7 @@ deploy_snapshot_with_ddprof_snapshot: variables: CACHE_TYPE: "lib" rules: - - if: '$POPULATE_CACHE' - when: never - # Manual trigger only - for testing with ddprof snapshot versions - - when: manual - allow_failure: true + - when: never script: - export MAVEN_CENTRAL_USERNAME=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.central_username --with-decryption --query "Parameter.Value" --out text) - export MAVEN_CENTRAL_PASSWORD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.central_password --with-decryption --query "Parameter.Value" --out text) @@ -1369,10 +1372,7 @@ deploy_artifacts_to_github: DDOCTOSTS_ID_TOKEN: aud: dd-octo-sts rules: - - if: '$POPULATE_CACHE' - when: never - - if: '$CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+$/' - when: on_success + - when: never # Requires the deploy_to_maven_central job to have run first (the UP-TO-DATE gradle check across jobs is broken) # This will deploy the artifacts built from the publishToSonatype task to the GitHub release needs: @@ -1400,7 +1400,7 @@ deploy_artifacts_to_github: requirements_json_test: rules: - - when: on_success + - when: never variables: REQUIREMENTS_BLOCK_JSON_PATH: "metadata/requirements-block.json" REQUIREMENTS_ALLOW_JSON_PATH: "metadata/requirements-allow.json" @@ -1413,10 +1413,7 @@ override_verify_maven_central: stage: publish needs: [ ] rules: - - if: '$POPULATE_CACHE' - when: never - - when: manual - allow_failure: true + - when: never script: - touch OVERRIDE_MAVEN_VERIFY cache: # Cache is used to signal between the override_verify_maven_central and verify_maven_central_deployment jobs @@ -1432,10 +1429,7 @@ verify_maven_central_deployment: stage: publish-release-artifacts needs: [ deploy_to_maven_central ] rules: - - if: '$POPULATE_CACHE' - when: never - - if: '$CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+$/' - when: on_success + - when: never cache: # Cache is used to signal between the override_verify_maven_central and verify_maven_central_deployment jobs - key: $CI_PIPELINE_ID-OVERRIDE_SIGNAL paths: @@ -1486,6 +1480,8 @@ configure_system_tests: SYSTEM_TESTS_SCENARIOS_GROUPS: "simple_onboarding,simple_onboarding_profiling,simple_onboarding_appsec,docker-ssi,lib-injection" create_key: + rules: + - when: never stage: generate-signing-key when: manual needs: [ ] @@ -1501,12 +1497,16 @@ create_key: - pubkeys validate_supported_configurations_v2_local_file: + rules: + - when: never extends: .validate_supported_configurations_v2_local_file variables: LOCAL_JSON_PATH: "metadata/supported-configurations.json" BACKFILLED: "false" update_central_configurations_version_range_v2: + rules: + - when: never extends: .update_central_configurations_version_range_v2 variables: LOCAL_REPO_NAME: "dd-trace-java" diff --git a/.gitlab/benchmarks.yml b/.gitlab/benchmarks.yml index 2247a5b4aa3..df2e21c2ea6 100644 --- a/.gitlab/benchmarks.yml +++ b/.gitlab/benchmarks.yml @@ -33,11 +33,15 @@ FF_USE_LEGACY_KUBERNETES_EXECUTION_STRATEGY: "true" dsm-kafka-producer-benchmark: + rules: + - when: always extends: .dsm-kafka-benchmarks variables: BP_KAFKA_SCENARIO_DIR: producer-benchmark dsm-kafka-consumer-benchmark: + rules: + - when: always extends: .dsm-kafka-benchmarks variables: BP_KAFKA_SCENARIO_DIR: consumer-benchmark @@ -45,16 +49,7 @@ dsm-kafka-consumer-benchmark: debugger-benchmarks: stage: benchmarks rules: - - if: '$POPULATE_CACHE' - when: never - - if: $CI_PIPELINE_SOURCE != "schedule" - changes: - paths: - - dd-java-agent/agent-debugger/**/* - compare_to: "master" - when: on_success - - when: manual - allow_failure: true + - when: always tags: ["runner:apm-k8s-tweaked-metal"] interruptible: true timeout: 1h diff --git a/.gitlab/ci-visibility-tests.yml b/.gitlab/ci-visibility-tests.yml index d5379c8dbdb..691fbf1e97b 100644 --- a/.gitlab/ci-visibility-tests.yml +++ b/.gitlab/ci-visibility-tests.yml @@ -7,10 +7,6 @@ ci-visibility-tests-check: DDOCTOSTS_ID_TOKEN: aud: dd-octo-sts rules: - - if: '$POPULATE_CACHE' - when: never - - if: '$CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH !~ /^(master|release\/)/' - when: on_success - when: never before_script: - dd-octo-sts version @@ -32,10 +28,6 @@ ci-visibility-tests-trigger: stage: ci-visibility-tests needs: [ci-visibility-tests-check] rules: - - if: '$POPULATE_CACHE' - when: never - - if: '$CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH !~ /^(master|release\/)/' - when: on_success - when: never trigger: include: diff --git a/.gitlab/exploration-tests.yml b/.gitlab/exploration-tests.yml index f8d39acbbbb..cb2a310e48e 100644 --- a/.gitlab/exploration-tests.yml +++ b/.gitlab/exploration-tests.yml @@ -2,6 +2,8 @@ variables: EXPLORATION_TESTS_IMAGE: registry.ddbuild.io/ci/dd-trace-java:exploration-tests build-exploration-tests-image: + rules: + - when: never stage: exploration-tests when: manual needs: [] @@ -76,6 +78,8 @@ build-exploration-tests-image: - "*_debugger-dumps.tar.gz" exploration-tests-method-jsoup: + rules: + - when: never needs: [ build ] dependencies: - build @@ -86,6 +90,8 @@ exploration-tests-method-jsoup: - ./run-exploration-tests.sh "method" "$PROJECT" "mvn verify" "include_${PROJECT}.txt" "exclude_${PROJECT}.txt" exploration-tests-line-jsoup: + rules: + - when: never needs: [ build ] dependencies: - build @@ -96,6 +102,8 @@ exploration-tests-line-jsoup: - ./run-exploration-tests.sh "line" "$PROJECT" "mvn verify" "include_${PROJECT}.txt" "exclude_${PROJECT}.txt" exploration-tests-method-jackson-core: + rules: + - when: never needs: [ build ] dependencies: - build @@ -106,6 +114,8 @@ exploration-tests-method-jackson-core: - ./run-exploration-tests.sh "method" "$PROJECT" "mvn verify" "include_${PROJECT}.txt" "exclude_${PROJECT}.txt" exploration-tests-line-jackson-core: + rules: + - when: never needs: [ build ] dependencies: - build @@ -116,6 +126,8 @@ exploration-tests-line-jackson-core: - ./run-exploration-tests.sh "line" "$PROJECT" "mvn verify" "include_${PROJECT}.txt" "exclude_${PROJECT}.txt" exploration-tests-method-jackson-databind: + rules: + - when: never needs: [ build ] dependencies: - build @@ -126,6 +138,8 @@ exploration-tests-method-jackson-databind: - ./run-exploration-tests.sh "method" "$PROJECT" "./mvnw verify" "include_${PROJECT}.txt" "exclude_$PROJECT.txt" exploration-tests-line-jackson-databind: + rules: + - when: never needs: [ build ] dependencies: - build @@ -136,6 +150,8 @@ exploration-tests-line-jackson-databind: - ./run-exploration-tests.sh "line" "$PROJECT" "./mvnw verify" "include_${PROJECT}.txt" "exclude_line_$PROJECT.txt" exploration-tests-method-okhttp: + rules: + - when: never needs: [ build ] dependencies: - build diff --git a/.gitlab/java-benchmark-configs.yml b/.gitlab/java-benchmark-configs.yml index 8df9511b52d..0991e27f091 100644 --- a/.gitlab/java-benchmark-configs.yml +++ b/.gitlab/java-benchmark-configs.yml @@ -44,28 +44,35 @@ linux-java-spring-petclinic-parallel: linux-java-insecure-bank-load-parallel: needs: ["publish-artifacts-to-s3"] - rules: *parallel_benchmark_rules + rules: + - when: always linux-java-spring-petclinic-load-parallel: needs: ["publish-artifacts-to-s3"] - rules: *parallel_benchmark_rules + rules: + - when: always linux-java-spring-petclinic-sca-load-parallel: needs: ["publish-artifacts-to-s3"] - rules: *parallel_benchmark_rules + rules: + - when: always linux-java-insecure-bank-startup-parallel: needs: ["publish-artifacts-to-s3"] - rules: *parallel_startup_benchmark_rules + rules: + - when: always linux-java-spring-petclinic-startup-parallel: needs: ["publish-artifacts-to-s3"] - rules: *parallel_startup_benchmark_rules + rules: + - when: always linux-java-dacapo-parallel-1: needs: ["publish-artifacts-to-s3"] - rules: *parallel_benchmark_rules + rules: + - when: always linux-java-dacapo-parallel-2: needs: ["publish-artifacts-to-s3"] - rules: *parallel_benchmark_rules + rules: + - when: always