From e796bb8507024c5ec2a2e72f38408402e746dfd2 Mon Sep 17 00:00:00 2001 From: Brice Dutheil Date: Thu, 8 Oct 2026 15:13:25 +0200 Subject: [PATCH 1/4] fix: clarify nested Gradle cleanup failures --- .../buildlogic/smoketest/NestedGradleBuild.kt | 15 +++++++++- .../smoketest/SmokeTestAppExtension.kt | 2 +- .../smoketest/SmokeTestAppPluginTest.kt | 28 +++++++++++++++++++ 3 files changed, 43 insertions(+), 2 deletions(-) diff --git a/build-logic/smoke-test/src/main/kotlin/datadog/buildlogic/smoketest/NestedGradleBuild.kt b/build-logic/smoke-test/src/main/kotlin/datadog/buildlogic/smoketest/NestedGradleBuild.kt index 9219923e5fb..49fc7bcbb01 100644 --- a/build-logic/smoke-test/src/main/kotlin/datadog/buildlogic/smoketest/NestedGradleBuild.kt +++ b/build-logic/smoke-test/src/main/kotlin/datadog/buildlogic/smoketest/NestedGradleBuild.kt @@ -43,6 +43,8 @@ import javax.inject.Inject * from the root build can be forwarded via [projectJar]; each entry is passed as * `-P=` and tracked as a task input so the nested build re-runs * when the upstream jar changes. + * + * The temporary Gradle user home is preserved when daemon shutdown cannot be confirmed. */ @CacheableTask abstract class NestedGradleBuild @Inject constructor( @@ -137,7 +139,11 @@ abstract class NestedGradleBuild @Inject constructor( @get:Input abstract val buildCacheEnabled: Property - /** Timeout, in seconds, for stopping the nested Gradle daemon after the build. */ + /** + * Timeout, in seconds, applied separately to the `--stop` command and the subsequent daemon + * exit wait. The two waits can take up to twice this value in total. When unset, `--stop` + * waits indefinitely and daemon exit waits up to 30 seconds. + */ @get:Input @get:Optional abstract val stopTimeoutSeconds: Property @@ -356,6 +362,12 @@ abstract class NestedGradleBuild @Inject constructor( private fun createGradleUserHome(): File { val directory = temporaryDir.resolve("gradle-user-home") deleteGradleUserHome(directory) + if (directory.exists()) { + throw GradleException( + "Could not clean up existing nested Gradle user home: ${directory.absolutePath}. " + + "Check the preceding daemon shutdown and cleanup warnings.", + ) + } if (!directory.mkdirs()) { throw GradleException( "Could not create nested Gradle user home: ${directory.absolutePath}", @@ -364,6 +376,7 @@ abstract class NestedGradleBuild @Inject constructor( return directory } + // Gradle's internal log naming also drives buildSrc's GradleFixture.stopDaemonsIn. private fun findGradleDaemons(directory: File): List = directory.resolve("daemon").walkTopDown() .filter { it.isFile && it.name.startsWith("daemon-") && it.name.endsWith(".out.log") } diff --git a/build-logic/smoke-test/src/main/kotlin/datadog/buildlogic/smoketest/SmokeTestAppExtension.kt b/build-logic/smoke-test/src/main/kotlin/datadog/buildlogic/smoketest/SmokeTestAppExtension.kt index 71175e894e0..92a9551d068 100644 --- a/build-logic/smoke-test/src/main/kotlin/datadog/buildlogic/smoketest/SmokeTestAppExtension.kt +++ b/build-logic/smoke-test/src/main/kotlin/datadog/buildlogic/smoketest/SmokeTestAppExtension.kt @@ -312,7 +312,7 @@ abstract class GradleAppSpec @Inject constructor() : ApplicationSpec() { */ abstract val buildCacheEnabled: Property - /** Timeout, in seconds, for stopping the nested Gradle daemon after the build. */ + /** Timeout, in seconds, for each shutdown wait. See [NestedGradleBuild.stopTimeoutSeconds]. */ abstract val stopTimeoutSeconds: Property } diff --git a/build-logic/smoke-test/src/test/kotlin/datadog/buildlogic/smoketest/SmokeTestAppPluginTest.kt b/build-logic/smoke-test/src/test/kotlin/datadog/buildlogic/smoketest/SmokeTestAppPluginTest.kt index dee9db72e73..b1ee1333f8d 100644 --- a/build-logic/smoke-test/src/test/kotlin/datadog/buildlogic/smoketest/SmokeTestAppPluginTest.kt +++ b/build-logic/smoke-test/src/test/kotlin/datadog/buildlogic/smoketest/SmokeTestAppPluginTest.kt @@ -4,6 +4,8 @@ import datadog.buildlogic.smoketest.NestedGradleBuild.Companion.gradleExecutable import datadog.buildlogic.smoketest.NestedGradleBuild.Companion.isGradleDaemonCandidate import datadog.buildlogic.smoketest.NestedMavenBuild.Companion.mavenWrapperName import org.assertj.core.api.Assertions.assertThat +import org.assertj.core.api.Assertions.assertThatThrownBy +import org.gradle.api.GradleException import org.gradle.api.plugins.JavaPlugin import org.gradle.jvm.toolchain.JavaLanguageVersion import org.gradle.kotlin.dsl.apply @@ -111,6 +113,32 @@ class SmokeTestAppPluginTest { } } + @Test + fun `nested Gradle build reports a user home retained after cleanup`() { + val project = ProjectBuilder.builder().build() + project.apply() + val task = project.tasks.register("nestedBuild", NestedGradleBuild::class.java).get() + task.applicationDir.set(project.layout.projectDirectory) + task.applicationBuildDir.set(project.layout.buildDirectory.dir("application")) + // Resolve the toolchain before simulating interrupted cleanup. + task.javaLauncher.get() + val userHome = task.temporaryDir.resolve("gradle-user-home") + assertThat(userHome.mkdirs()).isTrue() + + Thread.currentThread().interrupt() + try { + assertThatThrownBy { task.runNestedBuild() } + .isInstanceOf(GradleException::class.java) + .hasMessageContaining("Could not clean up existing nested Gradle user home:") + .hasMessageContaining(userHome.absolutePath) + + assertThat(Thread.currentThread().isInterrupted).isTrue() + assertThat(userHome).exists() + } finally { + Thread.interrupted() + } + } + @Test fun `applying the plugin creates the smokeTestApp extension`() { val project = ProjectBuilder.builder().build() From 8f495d4abfd13e6f623b6b14dac44032b24c8637 Mon Sep 17 00:00:00 2001 From: Brice Dutheil Date: Thu, 8 Oct 2026 15:14:44 +0200 Subject: [PATCH 2/4] ci: focus nested Gradle cleanup checks --- .gitlab-ci.yml | 1263 +----------------------------------------------- 1 file changed, 2 insertions(+), 1261 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8dbde59e860..e17ad7618ae 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,25 +1,3 @@ -include: - - local: ".gitlab/one-pipeline.locked.yml" - - local: ".gitlab/benchmarks.yml" - - local: ".gitlab/exploration-tests.yml" - - local: ".gitlab/ci-visibility-tests.yml" - - project: 'DataDog/apm-reliability/apm-sdks-benchmarks' - file: '.gitlab/ci-java-spring-petclinic-parallel.yml' - ref: &apm_sdks_benchmarks_sha '454a97647cbc16ab7fd20e917aa31532ba7532eb' # pinned by .github/workflows/update-apm-sdks-benchmarks-reference.yaml - - project: 'DataDog/apm-reliability/apm-sdks-benchmarks' - file: '.gitlab/ci-java-load-parallel.yml' - ref: *apm_sdks_benchmarks_sha - - project: 'DataDog/apm-reliability/apm-sdks-benchmarks' - file: '.gitlab/ci-java-startup-parallel.yml' - ref: *apm_sdks_benchmarks_sha - - project: 'DataDog/apm-reliability/apm-sdks-benchmarks' - file: '.gitlab/ci-java-dacapo-parallel.yml' - ref: *apm_sdks_benchmarks_sha - - project: 'DataDog/apm-reliability/apm-sdks-benchmarks' - file: '.gitlab/ci-java-post-pr-comment.yml' - ref: *apm_sdks_benchmarks_sha - - local: ".gitlab/java-benchmark-configs.yml" - stages: - build - publish @@ -45,7 +23,6 @@ stages: - generate-signing-key variables: - APM_SDKS_BENCHMARKS_SHA: *apm_sdks_benchmarks_sha # Test the OpenTelemetry Operator-compatible Java image jobs from one-pipeline. CI_OTEL_OPERATOR_IMAGES_ENABLED: "true" CI_OTEL_OPERATOR_LANGUAGE: java @@ -97,58 +74,10 @@ workflow: on_new_commit: none - when: always -.test_matrix: &test_matrix - - testJvm: &test_jvms - - "8" - - "11" - - "17" - - "21" - - "25" - - "27" # JDK 27 TODO: remove after GA (tip will move to 27) - - "semeru11" - - "oracle8" - - "zulu8" - - "semeru8" - - "ibm8" - - "zulu11" - - "semeru17" - - "tip" - CI_SPLIT: ["1/1"] - -# Gitlab doesn't support "parallel" and "parallel:matrix" at the same time -# These blocks emulate "parallel" by including it in the matrix -.test_matrix_2: &test_matrix_2 - - testJvm: *test_jvms - CI_SPLIT: ["1/2", "2/2"] - -.test_matrix_4: &test_matrix_4 - - testJvm: *test_jvms - CI_SPLIT: ["1/4", "2/4", "3/4", "4/4"] - -.test_matrix_6: &test_matrix_6 - - testJvm: *test_jvms - CI_SPLIT: ["1/6", "2/6", "3/6", "4/6", "5/6", "6/6"] - -.test_matrix_8: &test_matrix_8 - - testJvm: *test_jvms - CI_SPLIT: ["1/8", "2/8", "3/8", "4/8", "5/8", "6/8", "7/8", "8/8"] - -.test_matrix_12: &test_matrix_12 - - testJvm: *test_jvms - CI_SPLIT: [ "1/12", "2/12", "3/12", "4/12", "5/12", "6/12", "7/12", "8/12", "9/12", "10/12", "11/12", "12/12" ] - -.master_only: &master_only - - if: $CI_COMMIT_BRANCH == "master" - when: on_success - default: tags: [ "arch:amd64" ] interruptible: true -.set_datadog_api_keys: &set_datadog_api_keys - - export DATADOG_API_KEY_PROD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.DATADOG_API_KEY_PROD --with-decryption --query "Parameter.Value" --out text) - -# CI_NODE_INDEX and CI_NODE_TOTAL are 1-indexed and not always set. These steps normalize the numbers for jobs .normalize_node_index: &normalize_node_index - if [ "$CI_NO_SPLIT" == "true" ] ; then CI_NODE_INDEX=1; CI_NODE_TOTAL=1; fi # A job uses parallel but doesn't intend to split by index - if [ -n "$CI_SPLIT" ]; then CI_NODE_INDEX="${CI_SPLIT%%/*}"; CI_NODE_TOTAL="${CI_SPLIT##*/}"; fi @@ -184,21 +113,6 @@ default: KUBERNETES_MEMORY_REQUEST: 16Gi KUBERNETES_MEMORY_LIMIT: 16Gi -.tier_l: - variables: &tier_l_variables - GRADLE_WORKERS: 6 - GRADLE_MEMORY_MIN: 1G - GRADLE_MEMORY_MAX: 6G - KUBERNETES_CPU_REQUEST: 10 - KUBERNETES_MEMORY_REQUEST: 20Gi - KUBERNETES_MEMORY_LIMIT: 20Gi - -.tier_xl: - variables: &tier_xl_variables - <<: *tier_l_variables - KUBERNETES_MEMORY_REQUEST: 32Gi - KUBERNETES_MEMORY_LIMIT: 32Gi - .gitlab_base_ref_params: &gitlab_base_ref_params - | export GIT_BASE_REF=$(.gitlab/find-gh-base-ref.sh) @@ -290,326 +204,6 @@ default: # Check and fail early if maven central credentials are incorrect. When a new token is generated # on the central publisher portal, it invalidates the old one. This check prevents going further. # See https://datadoghq.atlassian.net/wiki/x/Oog5OgE -maven-central-pre-release-check: - image: ${BUILDER_IMAGE_REPO}:${BUILDER_IMAGE_VERSION_PREFIX}base - stage: .pre - rules: - - if: '$CI_COMMIT_BRANCH == "master"' - when: on_success - allow_failure: false - - if: '$CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+$/' - when: on_success - allow_failure: false - script: - - | - MAVEN_CENTRAL_USERNAME=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.central_username --with-decryption --query "Parameter.Value" --out text) - MAVEN_CENTRAL_PASSWORD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.central_password --with-decryption --query "Parameter.Value" --out text) - # See https://central.sonatype.org/publish/publish-portal-ossrh-staging-api/ - # Use the staging API search endpoint to validate the tokens without relying on a specific deployment - AUTHORIZATION_HEADER="Authorization: Bearer $(printf '%s:%s' "$MAVEN_CENTRAL_USERNAME" "$MAVEN_CENTRAL_PASSWORD" | base64)" - if ! curl --silent --show-error --fail \ - "https://ossrh-staging-api.central.sonatype.com/manual/search/repositories?ip=any" \ - --header "$AUTHORIZATION_HEADER" \ - > /dev/null; then - echo "Failed to authenticate tokens against maven central staging API. Check credentials and see https://datadoghq.atlassian.net/wiki/x/Oog5OgE" - exit 1 - fi - -dd-octo-sts-pre-release-check: - image: registry.ddbuild.io/images/dd-octo-sts-ci-base:2025.06-1 - stage: .pre - tags: [ "arch:amd64" ] - id_tokens: - DDOCTOSTS_ID_TOKEN: - aud: dd-octo-sts - rules: - - if: '$POPULATE_CACHE' - when: never - - if: '$CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+$/' - when: on_success - allow_failure: false - before_script: - - dd-octo-sts version - - dd-octo-sts debug --scope DataDog/dd-trace-java --policy self.gitlab.release - - dd-octo-sts token --scope DataDog/dd-trace-java --policy self.gitlab.release > test-github-token.txt - script: - - gh auth login --with-token < test-github-token.txt - - gh auth status - after_script: - - dd-octo-sts revoke -t $(cat test-github-token.txt) - retry: - max: 2 - when: always - -build: - needs: - - job: maven-central-pre-release-check - optional: true - - job: dd-octo-sts-pre-release-check - optional: true - extends: .gradle_build - variables: - BUILD_CACHE_POLICY: push - CACHE_TYPE: "lib" - DEPENDENCY_CACHE_POLICY: pull - script: - - if [ $CI_PIPELINE_SOURCE == "schedule" ] ; then ./gradlew resolveAndLockAll --write-locks $GRADLE_ARGS; fi - - ./gradlew --version - - ./gradlew clean :dd-java-agent:shadowJar :dd-java-agent:check :dd-trace-api:jar :dd-trace-ot:shadowJar :products:feature-flagging:feature-flagging-api:jar -PskipTests -x spotlessCheck $GRADLE_ARGS - - echo UPSTREAM_TRACER_VERSION=$(java -jar workspace/dd-java-agent/build/libs/*.jar) >> upstream.env - - echo "BUILD_JOB_NAME=$CI_JOB_NAME" >> build.env - - echo "BUILD_JOB_ID=$CI_JOB_ID" >> build.env - artifacts: - when: always - paths: - - 'workspace/dd-java-agent/build/libs/*.jar' - - 'workspace/dd-trace-api/build/libs/*.jar' - - 'workspace/dd-trace-ot/build/libs/*.jar' - - 'workspace/products/feature-flagging/feature-flagging-api/build/libs/*.jar' - - 'upstream.env' - - '.gradle/daemon/*/*.out.log' - reports: - dotenv: build.env - -build_tests: - extends: .gradle_build - variables: - <<: *tier_xl_variables - BUILD_CACHE_POLICY: push - DEPENDENCY_CACHE_POLICY: pull - parallel: - matrix: - - GRADLE_TARGET: ":baseTest" - CACHE_TYPE: "base" - - GRADLE_TARGET: ":profilingTest" - CACHE_TYPE: "profiling" - - GRADLE_TARGET: ":instrumentationTest" - CACHE_TYPE: "inst" - - GRADLE_TARGET: ":instrumentationLatestDepTest" - CACHE_TYPE: "latestdep" - - GRADLE_TARGET: ":smokeTest" - CACHE_TYPE: "smoke" - MAVEN_OPTS: "-Xms256M -Xmx1024M" - script: - - *gitlab_base_ref_params - - ./gradlew --version - - ./gradlew clean $GRADLE_TARGET $GRADLE_PARAMS -PskipTests $GRADLE_ARGS - -populate_plugin_cache: - extends: .gradle_build - cache: - - &plugin_cache - key: dependency-plugins-v1 - paths: - - .gradle-plugin-cache/caches/modules-2 - policy: pull-push - when: always - unprotect: true - rules: - - if: '$POPULATE_CACHE' - when: on_success - script: - - mkdir -p .gradle-plugin-cache - - sudo chown -R 1001:1001 .gradle-plugin-cache - - export GRADLE_USER_HOME=$(pwd)/.gradle-plugin-cache - - | - gradle_status=0 - ./gradlew help -PskipTests $GRADLE_ARGS || gradle_status=$? - if ! find .gradle-plugin-cache/caches/modules-2/files-2.1 -type f -print -quit 2>/dev/null | grep -q .; then - echo "WARNING: No plugin artifacts were cached; skipping the cache upload." - rm -rf .gradle-plugin-cache/caches/modules-2 - fi - exit "$gradle_status" - # This cache warmer is an optimization; population jobs can continue with a cold plugin cache. - allow_failure: true - -populate_dep_cache: - extends: build_tests - # Wait for build-cache producers so their caches are uploaded before this job restores them. - needs: - - job: populate_plugin_cache - - job: build - artifacts: false - - job: build_tests - artifacts: false - variables: - BUILD_CACHE_POLICY: pull - DEPENDENCY_CACHE_POLICY: push - cache: - - <<: *plugin_cache - policy: pull - - *dependency_cache - - *build_cache - rules: - - if: '$POPULATE_CACHE' - when: on_success - before_script: - - !reference [.gradle_build, before_script] - # Seed the otherwise cold writable cache, keeping the cache uploaded below self-contained. - - mkdir -p .gradle/caches - - | - if [ -d .gradle-plugin-cache/caches/modules-2 ]; then - sudo chown -R 1001:1001 .gradle-plugin-cache - cp -a .gradle-plugin-cache/caches/modules-2 .gradle/caches/ - else - echo "Plugin cache is unavailable; continuing with a cold dependency cache." - fi - parallel: - matrix: - - GRADLE_TARGET: ":dd-java-agent:shadowJar :dd-trace-api:jar :dd-trace-ot:shadowJar" - CACHE_TYPE: "lib" - - GRADLE_TARGET: ":baseTest" - CACHE_TYPE: "base" - - GRADLE_TARGET: ":profilingTest" - CACHE_TYPE: "profiling" - - GRADLE_TARGET: ":instrumentationTest" - CACHE_TYPE: "inst" - - GRADLE_TARGET: ":instrumentationLatestDepTest" - CACHE_TYPE: "latestdep" - - GRADLE_TARGET: ":smokeTest" - CACHE_TYPE: "smoke" - - GRADLE_TARGET: "spotlessCheck" - CACHE_TYPE: "spotless" - GRADLE_MEMORY_MAX: "6G" - -publish-artifacts-to-s3: - image: registry.ddbuild.io/images/mirror/amazon/aws-cli:2.4.29 - stage: publish - needs: [ build ] - script: - - source upstream.env - - export VERSION="${UPSTREAM_TRACER_VERSION%~*}" # remove ~githash from the end of version - - aws s3 cp workspace/dd-java-agent/build/libs/dd-java-agent-${VERSION}.jar s3://dd-trace-java-builds/${CI_COMMIT_REF_NAME}/dd-java-agent.jar - - aws s3 cp workspace/dd-trace-api/build/libs/dd-trace-api-${VERSION}.jar s3://dd-trace-java-builds/${CI_COMMIT_REF_NAME}/dd-trace-api.jar - - aws s3 cp workspace/dd-trace-ot/build/libs/dd-trace-ot-${VERSION}.jar s3://dd-trace-java-builds/${CI_COMMIT_REF_NAME}/dd-trace-ot.jar - - aws s3 cp workspace/products/feature-flagging/feature-flagging-api/build/libs/dd-openfeature-${VERSION}.jar s3://dd-trace-java-builds/${CI_COMMIT_REF_NAME}/dd-openfeature.jar - - aws s3 cp workspace/dd-java-agent/build/libs/dd-java-agent-${VERSION}.jar s3://dd-trace-java-builds/${CI_PIPELINE_ID}/dd-java-agent.jar - - aws s3 cp workspace/dd-trace-api/build/libs/dd-trace-api-${VERSION}.jar s3://dd-trace-java-builds/${CI_PIPELINE_ID}/dd-trace-api.jar - - aws s3 cp workspace/dd-trace-ot/build/libs/dd-trace-ot-${VERSION}.jar s3://dd-trace-java-builds/${CI_PIPELINE_ID}/dd-trace-ot.jar - - aws s3 cp workspace/products/feature-flagging/feature-flagging-api/build/libs/dd-openfeature-${VERSION}.jar s3://dd-trace-java-builds/${CI_PIPELINE_ID}/dd-openfeature.jar - - | - cat << EOF > links.json - { - "S3 Links": [ - { - "external_link": { - "label": "Public Link to dd-java-agent.jar", - "url": "https://s3.us-east-1.amazonaws.com/dd-trace-java-builds/${CI_PIPELINE_ID}/dd-java-agent.jar" - } - }, - { - "external_link": { - "label": "Public Link to dd-openfeature.jar", - "url": "https://s3.us-east-1.amazonaws.com/dd-trace-java-builds/${CI_PIPELINE_ID}/dd-openfeature.jar" - } - } - ] - } - EOF - artifacts: - reports: - annotations: - - links.json - - -spotless: - extends: .gradle_build - stage: tests - needs: [] - variables: - GRADLE_MEMORY_MAX: 6G - CACHE_TYPE: "spotless" - script: - - ./gradlew --version - # test-published-dependencies's build file needs main version file - - ./gradlew spotlessCheck writeMainVersionFile $GRADLE_ARGS - - cd test-published-dependencies && ./gradlew spotlessCheck $GRADLE_ARGS - -check-instrumentation-naming: - extends: .gradle_build - stage: tests - needs: [ ] - script: - - ./gradlew --version - - ./gradlew checkInstrumentationNaming - -config-inversion-linter: - extends: .gradle_build - stage: tests - needs: [] - script: - - ./gradlew --version - - ./gradlew checkConfigurations - -test_published_artifacts: - extends: .gradle_build - image: ${BUILDER_IMAGE_REPO}:${BUILDER_IMAGE_VERSION_PREFIX}7 # Needs Java7 for some tests - stage: tests - needs: [ build ] - variables: - CACHE_TYPE: "lib" - script: - - mvn_local_repo=$(./mvnw help:evaluate -Dexpression=settings.localRepository -q -DforceStdout) - - rm -rf "${mvn_local_repo}/com/datadoghq" - - export GPG_PRIVATE_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.signing.gpg_private_key --with-decryption --query "Parameter.Value" --out text) - - export GPG_PASSWORD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.signing.gpg_passphrase --with-decryption --query "Parameter.Value" --out text) - - export GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xms2G -Xmx2G -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp'" - - ./gradlew publishToMavenLocal $GRADLE_ARGS - - cd test-published-dependencies - - printf '\norg.gradle.console=colored\n' >> gradle.properties - - export GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xms1G -Xmx1G -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp'" - - ./gradlew --version - - ./gradlew check --info $GRADLE_ARGS - after_script: - - *cgroup_info - - source .gitlab/gitlab-utils.sh - - gitlab_section_start "collect-reports" "Collecting reports" - - .gitlab/collect_reports.sh - - gitlab_section_end "collect-reports" - artifacts: - when: always - paths: - - ./check_reports - -validate_build: - extends: .gradle_build - stage: tests - needs: [ build ] - variables: - CACHE_TYPE: "lib" - script: - # Preserve the `build` job artifacts before Gradle rebuilds and overwrites them under - # workspace/**/build/libs, so jardiff can compare the rebuilt jars against them. - - mkdir -p reference-artifacts - - cp workspace/dd-java-agent/build/libs/*.jar reference-artifacts/ - - cp workspace/dd-trace-api/build/libs/*.jar reference-artifacts/ - - cp workspace/dd-trace-ot/build/libs/*.jar reference-artifacts/ - # Scheduled builds refresh dependency locks before creating the reference artifacts. - # Refresh them here as well so the candidate uses the same dependency resolution. - - if [ $CI_PIPELINE_SOURCE == "schedule" ] ; then ./gradlew resolveAndLockAll --write-locks $GRADLE_ARGS; fi - - ./gradlew --version - # This will run the shadowJar task and exercise the build cache, allowing to identify build-cache issues - # Keep both JAR sets for direct inspection when the comparison fails. - - | - if ! ./gradlew compareToReferenceJar -PjardiffReferenceDir="$CI_PROJECT_DIR/reference-artifacts" -PskipTests $GRADLE_ARGS; then - mkdir -p check_reports/jar-comparison-artifacts/reference check_reports/jar-comparison-artifacts/candidate - cp reference-artifacts/*.jar check_reports/jar-comparison-artifacts/reference/ - cp workspace/dd-java-agent/build/libs/*.jar check_reports/jar-comparison-artifacts/candidate/ - cp workspace/dd-trace-api/build/libs/*.jar check_reports/jar-comparison-artifacts/candidate/ - cp workspace/dd-trace-ot/build/libs/*.jar check_reports/jar-comparison-artifacts/candidate/ - exit 1 - fi - after_script: - - source .gitlab/gitlab-utils.sh - - gitlab_section_start "collect-reports" "Collecting reports" - - .gitlab/collect_reports.sh --destination ./check_reports - - gitlab_section_end "collect-reports" - artifacts: - when: always - paths: - - ./check_reports - - '.gradle/daemon/*/*.out.log' - .check_job: extends: .gradle_build needs: [ build ] @@ -650,861 +244,8 @@ check_build_src: extends: .check_job needs: [] variables: - GRADLE_TARGET: ":buildSrc:build :build-logic:check" - -check_base: - extends: .check_job - variables: - GRADLE_TARGET: ":baseCheck" - -check_inst: - extends: .check_job - parallel: 4 - variables: - GRADLE_TARGET: ":instrumentationCheck" - CACHE_TYPE: "inst" - -check_smoke: - extends: .check_job - parallel: 4 - variables: - GRADLE_TARGET: ":smokeCheck" - CACHE_TYPE: "smoke" - -check_profiling: - extends: .check_job - variables: - GRADLE_TARGET: ":profilingCheck" - -check_debugger: - extends: .check_job - variables: - GRADLE_TARGET: ":debuggerCheck" - -muzzle: - extends: .gradle_build - # needs:parallel:matrix limits this job to a specific build_tests combination. - # Keep matrix vars exact and in build_tests declaration order: - # https://docs.gitlab.com/ci/yaml/#needsparallelmatrix - needs: &needs_build_tests_inst - - job: build_tests - parallel: - matrix: - - GRADLE_TARGET: ":instrumentationTest" - CACHE_TYPE: "inst" - stage: tests - rules: - - if: '$CI_COMMIT_BRANCH =~ /^mq-working-branch-/' - when: never - - if: '$CI_COMMIT_BRANCH =~ /^gh-readonly-queue/' - when: never - - when: on_success - parallel: - matrix: - - CI_SPLIT: ["1/8", "2/8", "3/8", "4/8", "5/8", "6/8", "7/8", "8/8"] - variables: - CACHE_TYPE: "inst" - script: - - export SKIP_BUILDSCAN="true" - - ./gradlew --version - - ./gradlew :runMuzzle -Pslot=$CI_NODE_INDEX/$CI_NODE_TOTAL $GRADLE_ARGS - after_script: - - *container_info - - *cgroup_info - - *set_datadog_api_keys - - source .gitlab/gitlab-utils.sh - - gitlab_section_start "collect-reports" "Collecting reports" - - .gitlab/collect_reports.sh - - .gitlab/collect_results.sh - - .gitlab/upload_ciapp.sh $CACHE_TYPE - - gitlab_section_end "collect-reports" - artifacts: - when: always - paths: - - ./reports - - ./results - - '.gradle/daemon/*/*.out.log' - reports: - junit: results/*.xml - -muzzle-dep-report: - extends: .gradle_build - needs: *needs_build_tests_inst - stage: tests - rules: - - if: '$CI_COMMIT_BRANCH =~ /^mq-working-branch-/' - when: never - - if: '$CI_COMMIT_BRANCH =~ /^gh-readonly-queue/' - when: never - - when: on_success - variables: - CACHE_TYPE: "inst" - script: - - export SKIP_BUILDSCAN="true" - - ./gradlew --version - - ./gradlew generateMuzzleReport muzzleInstrumentationReport $GRADLE_ARGS - after_script: - - *container_info - - *cgroup_info - - .gitlab/collect_muzzle_deps.sh - artifacts: - when: always - paths: - - ./reports - - '.gradle/daemon/*/*.out.log' - -# In Gitlab, DD_* variables are set because the build runner is instrumented with Datadog telemetry -# To have a pristine environment for the tests, these variables are saved before the test run and restored afterwards -.prepare_test_env: &prepare_test_env - - export gitlabVariables=("DD_SERVICE" "DD_ENTITY_ID" "DD_SITE" "DD_ENV" "DD_DATACENTER" "DD_PARTITION" "DD_CLOUDPROVIDER") - - '[ ! -e pretest.env ] || rm pretest.env' - - | - for VARIABLE in "${gitlabVariables[@]}" - do - echo "export $VARIABLE=${!VARIABLE}" >> pretest.env - unset "$VARIABLE" - done - -.restore_pretest_env: &restore_pretest_env - - source pretest.env - -.test_job_common: - extends: .gradle_build - image: ${BUILDER_IMAGE_REPO}:${BUILDER_IMAGE_VERSION_PREFIX}$testJvm - variables: - <<: *tier_m_variables - GRADLE_PARAMS: "-PskipFlakyTests" - CONTINUE_ON_FAILURE: "false" - TEST_JVM_ARGS: "" - TESTCONTAINERS_CHECKS_DISABLE: "true" - TESTCONTAINERS_RYUK_DISABLED: "true" - TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX: "registry.ddbuild.io/images/mirror/" - JETTY_AVAILABLE_PROCESSORS: 4 # Jetty incorrectly calculates processor count in containers + GRADLE_TARGET: ":smoke-test:test" script: - *gitlab_base_ref_params - - > - if [ "$PROFILE_TESTS" == "true" ] && [ "$testJvm" != "ibm8" ] && [ "$testJvm" != "oracle8" ]; - then - export PROFILER_COMMAND="-XX:StartFlightRecording=settings=profile,filename=/tmp/${CI_JOB_NAME_SLUG}.jfr,dumponexit=true"; - fi - - *prepare_test_env - - export GRADLE_OPTS="-Dorg.gradle.jvmargs='$TEST_JVM_ARGS -Xms$GRADLE_MEMORY_MIN -Xmx$GRADLE_MEMORY_MAX $PROFILER_COMMAND -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp -Djava.util.prefs.userRoot=/tmp/.java/.userPrefs-${CI_JOB_ID}' -Ddatadog.forkedMinHeapSize=128M -Ddatadog.forkedMaxHeapSize=1024M" - ./gradlew --version - - ./gradlew $GRADLE_TARGET $GRADLE_PARAMS -PtestJvm=$testJvm -Pslot=$CI_NODE_INDEX/$CI_NODE_TOTAL $GRADLE_ARGS --continue || $CONTINUE_ON_FAILURE - after_script: - - *restore_pretest_env - - *set_datadog_api_keys - - *container_info - - *cgroup_info - - source .gitlab/gitlab-utils.sh - - gitlab_section_start "collect-reports" "Collecting reports" - - .gitlab/collect_reports.sh - - if [ "$PROFILE_TESTS" == "true" ]; then .gitlab/collect_profiles.sh; fi - - .gitlab/collect_results.sh - - .gitlab/upload_ciapp.sh $CACHE_TYPE $testJvm - - gitlab_section_end "collect-reports" - - .gitlab/count_tests.sh "$GRADLE_TARGET" "$testJvm" "./results" "./test_counts_${CI_JOB_ID}.json" - - export TEST_COUNTS_S3_PREFIX="test-counts/${CI_PIPELINE_ID}" - - export TEST_COUNTS_FILE="./test_counts_${CI_JOB_ID}.json" - # Use logical job identity in S3 so retried jobs overwrite stale attempts. - - export TEST_COUNTS_S3_FILE="test_counts_${CI_JOB_NAME_SLUG}_${CI_NODE_INDEX}-${CI_NODE_TOTAL}.json" - - export TEST_COUNTS_S3_URI="s3://${TEST_COUNTS_S3_BUCKET}/${TEST_COUNTS_S3_PREFIX}/${TEST_COUNTS_S3_FILE}" - - echo "Uploading ${TEST_COUNTS_FILE} to ${TEST_COUNTS_S3_URI}" - - aws s3 cp "$TEST_COUNTS_FILE" "$TEST_COUNTS_S3_URI" --only-show-errors - - URL_ENCODED_JOB_NAME=$(jq -rn --arg x "$CI_JOB_NAME" '$x|@uri') - - echo -e "${TEXT_BOLD}${TEXT_YELLOW}See test results in Datadog:${TEXT_CLEAR} https://app.datadoghq.com/ci/test/runs?query=test_level%3Atest%20%40test.service%3Add-trace-java%20%40ci.pipeline.id%3A${CI_PIPELINE_ID}%20%40ci.job.name%3A%22${URL_ENCODED_JOB_NAME}%22" - artifacts: - when: always - paths: - - ./reports.tar - - ./profiles.tar - - ./results - - './test_counts_*.json' - - '.gradle/daemon/*/*.out.log' - reports: - junit: results/*.xml - retry: - max: 2 - when: - - unknown_failure - - stuck_or_timeout_failure - - runner_system_failure - - unmet_prerequisites - - scheduler_failure - - data_integrity_failure - -.test_job_amd64: - extends: .test_job_common - tags: [ "docker-in-docker:amd64" ] # use docker-in-docker runner for testcontainers - needs: [ build_tests ] - stage: tests - rules: - # Protected branches (master/mq/gh-readonly): all JVMs run unconditionally - - if: '$CI_COMMIT_BRANCH == "master"' - when: on_success - - if: '$CI_COMMIT_BRANCH =~ /^mq-working-branch-/' - when: on_success - - if: '$CI_COMMIT_BRANCH =~ /^gh-readonly-queue/' - when: on_success - # Enable for default test JVMs or for NON_DEFAULT_JVMS - - if: '$NON_DEFAULT_JVMS == "true"' - when: on_success - - if: '$CI_COMMIT_MESSAGE =~ /\[ci: NON_DEFAULT_JVMS\]/' - when: on_success - - if: '$testJvm =~ $DEFAULT_TEST_JVMS' - when: on_success - -.test_job_arm64: - extends: .test_job_common - tags: [ "docker-in-docker:arm64" ] - stage: tests-arm64 - # Use the amd64 build only as a compilation gate. Do not download its platform-specific artifacts. - needs: - - job: build - artifacts: false - variables: - DEFAULT_TEST_JVMS: /^(11|17|21|25|27|tip)$/ # Java 27 TODO: remove 27 after GA (tip will move to 27) - TEST_JVM_ARGS: "-Xshare:off" - rules: - # Java 8 is excluded from the arm64 test matrix for all distributions. - # Java 8 JVMs are very slow on AARCH64. - - if: '$testJvm =~ /^(8|oracle8|zulu8|semeru8|ibm8)$/' - when: never - # arm64 tests are newly introduced to the merge queue and master. Keep them - # non-blocking (allow_failure) for now so we can collect stability stats and - # fix flaky/failing jobs without blocking the whole team. Remove allow_failure - # once the arm64 suite is proven stable. - - if: '$CI_COMMIT_BRANCH =~ /^mq-working-branch-/' - when: on_success - allow_failure: true - - if: '$CI_COMMIT_BRANCH == "master"' - when: on_success - allow_failure: true - # Enable non-default JVMs on demand. - - if: '$NON_DEFAULT_JVMS == "true"' - when: on_success - allow_failure: true - - if: '$CI_COMMIT_MESSAGE =~ /\[ci: NON_DEFAULT_JVMS\]/' - when: on_success - allow_failure: true - # Keep the default JVM subset available for manual runs on feature branches/PRs. - - if: '$testJvm =~ $DEFAULT_TEST_JVMS' - when: manual - allow_failure: true - cache: - - key: dependency-$CACHE_TYPE - paths: - - .gradle/wrapper - - .gradle/caches - - .gradle/notifications - - .mvn/caches - policy: pull - fallback_keys: - - dependency-base - - dependency-lib - unprotect: true - before_script: - - git config --global --add safe.directory "$CI_PROJECT_DIR" - - !reference [.gradle_build, before_script] - -.test_job_with_test_agent_common: - variables: - CI_USE_TEST_AGENT: "true" - CI_AGENT_HOST: local-agent - services: - - name: registry.ddbuild.io/images/mirror/dd-apm-test-agent/ddapm-test-agent:v1.64.1 - alias: local-agent - variables: - LOG_LEVEL: "DEBUG" - TRACE_LANGUAGE: "java" - DD_SUPPRESS_TRACE_PARSE_ERRORS: "true" - DD_POOL_TRACE_CHECK_FAILURES: "true" - DD_DISABLE_ERROR_RESPONSES: "true" - ENABLED_CHECKS: "trace_content_length,trace_stall,meta_tracer_version_header,trace_count_header,trace_peer_service,trace_dd_service" - script: - - !reference [.test_job_common, script] - - .gitlab/check_test_agent_results.sh - -.test_job_amd64_with_test_agent: - extends: - - .test_job_amd64 - - .test_job_with_test_agent_common - -.test_job_arm64_with_test_agent: - extends: - - .test_job_arm64 - - .test_job_with_test_agent_common - -agent_integration_tests: - extends: .test_job_amd64 - tags: [ "arch:amd64" ] - variables: - testJvm: "8" - CI_AGENT_HOST: local-agent - GRADLE_TARGET: "traceAgentTest" - CACHE_TYPE: "base" - services: - - name: registry.ddbuild.io/images/mirror/datadog/agent:7.40.1 - alias: local-agent - variables: - DD_APM_ENABLED: "true" - DD_BIND_HOST: "0.0.0.0" - DD_HOSTNAME: "local-agent" - DD_API_KEY: "invalid_key_but_this_is_fine" - -test_base: - extends: .test_job_amd64 - # needs:parallel:matrix limits this job to a specific build_tests combination. - # Keep matrix vars exact and in build_tests declaration order: - # https://docs.gitlab.com/ci/yaml/#needsparallelmatrix - needs: - - job: build_tests - parallel: - matrix: - - GRADLE_TARGET: ":baseTest" - CACHE_TYPE: "base" - variables: - GRADLE_TARGET: ":baseTest" - CACHE_TYPE: "base" - parallel: - matrix: *test_matrix_4 - script: - - if [ "$testJvm" == "8" ]; then export GRADLE_PARAMS="-PskipFlakyTests -PcheckCoverage"; fi - - !reference [.test_job_common, script] - -test_base_arm64: - extends: .test_job_arm64 - variables: - GRADLE_TARGET: ":baseTest" - CACHE_TYPE: "base" - parallel: - matrix: *test_matrix_4 - -test_inst: - extends: .test_job_amd64_with_test_agent - needs: *needs_build_tests_inst - variables: - <<: *tier_l_variables - GRADLE_TARGET: ":instrumentationTest" - CACHE_TYPE: "inst" - parallel: - matrix: *test_matrix_8 - -test_inst_arm64: - extends: .test_job_arm64_with_test_agent - variables: - <<: *tier_l_variables - GRADLE_TARGET: ":instrumentationTest" - CACHE_TYPE: "inst" - parallel: - matrix: *test_matrix_8 - -test_inst_latest: - extends: .test_job_amd64_with_test_agent - # needs:parallel:matrix limits this job to a specific build_tests combination. - # Keep matrix vars exact and in build_tests declaration order: - # https://docs.gitlab.com/ci/yaml/#needsparallelmatrix - needs: - - job: build_tests - parallel: - matrix: - - GRADLE_TARGET: ":instrumentationLatestDepTest" - CACHE_TYPE: "latestdep" - variables: - <<: *tier_l_variables - GRADLE_TARGET: ":instrumentationLatestDepTest" - CACHE_TYPE: "latestdep" - parallel: - matrix: - - testJvm: ["8", "17", "21", "25", "27", "tip"] # Java 27 TODO: remove 27 after GA (tip will move to 27) - # Gitlab doesn't support "parallel" and "parallel:matrix" at the same time - # This emulates "parallel" by including it in the matrix - CI_SPLIT: [ "1/6", "2/6", "3/6", "4/6", "5/6", "6/6"] - -test_inst_latest_arm64: - extends: .test_job_arm64_with_test_agent - variables: - <<: *tier_l_variables - GRADLE_TARGET: ":instrumentationLatestDepTest" - CACHE_TYPE: "latestdep" - parallel: - matrix: - - testJvm: ["17", "21", "25", "27", "tip"] # Java 27 TODO: remove 27 after GA (tip will move to 27) - # Gitlab doesn't support "parallel" and "parallel:matrix" at the same time - # This emulates "parallel" by including it in the matrix - CI_SPLIT: [ "1/6", "2/6", "3/6", "4/6", "5/6", "6/6"] - -test_flaky: - extends: .test_job_amd64_with_test_agent - variables: - GRADLE_PARAMS: "-PrunFlakyTests" - CACHE_TYPE: "smoke" - testJvm: "8" - CONTINUE_ON_FAILURE: "true" - rules: - - *master_only - - if: $RUN_FLAKY_TESTS == "true" - when: on_success - parallel: - matrix: - - GRADLE_TARGET: [":baseTest", ":smokeTest", ":debuggerTest"] - # Gitlab doesn't support "parallel" and "parallel:matrix" at the same time - # This emulates "parallel" by including it in the matrix - CI_SPLIT: [ "1/4", "2/4", "3/4", "4/4" ] - -test_flaky_inst: - extends: .test_job_amd64 - needs: *needs_build_tests_inst - variables: - GRADLE_TARGET: ":instrumentationTest" - GRADLE_PARAMS: "-PrunFlakyTests" - CACHE_TYPE: "inst" - testJvm: "8" - CONTINUE_ON_FAILURE: "true" - rules: - - *master_only - - if: $RUN_FLAKY_TESTS == "true" - when: on_success - parallel: 6 - -test_profiling: - extends: .test_job_amd64 - # needs:parallel:matrix limits this job to a specific build_tests combination. - # Keep matrix vars exact and in build_tests declaration order: - # https://docs.gitlab.com/ci/yaml/#needsparallelmatrix - needs: - - job: build_tests - parallel: - matrix: - - GRADLE_TARGET: ":profilingTest" - CACHE_TYPE: "profiling" - variables: - GRADLE_TARGET: ":profilingTest" - CACHE_TYPE: "profiling" - parallel: - matrix: *test_matrix - -test_profiling_arm64: - extends: .test_job_arm64 - variables: - GRADLE_TARGET: ":profilingTest" - CACHE_TYPE: "profiling" - parallel: - matrix: *test_matrix - -# specific jvms list for debugger project because J9-based JVMs have issues with local vars -# so need to test at least against one J9-based JVM -test_debugger: - extends: .test_job_amd64 - variables: - GRADLE_TARGET: ":debuggerTest" - CACHE_TYPE: "base" - DEFAULT_TEST_JVMS: /^(8|11|17|21|25|27|tip|semeru8)$/ # Java 27 TODO: remove 27 after GA (tip will move to 27) - parallel: - matrix: *test_matrix - # avoid running coverage for semeru8, semeru11, semeru17 and ibm8 as some tests are disabled and therefore cannot reach the - # exepected coverage - script: - - if [[ "$testJvm" != "semeru8" && "$testJvm" != "semeru11" && "$testJvm" != "semeru17" && "$testJvm" != "ibm8" ]]; then export GRADLE_PARAMS="$GRADLE_PARAMS -PcheckCoverage"; fi - - !reference [.test_job_common, script] - -test_debugger_arm64: - extends: .test_job_arm64 - variables: - GRADLE_TARGET: ":debuggerTest" - CACHE_TYPE: "base" - parallel: - matrix: *test_matrix - # avoid running coverage for semeru11 and semeru17 as some tests are disabled and therefore cannot reach the - # expected coverage - script: - - if [[ "$testJvm" != "semeru11" && "$testJvm" != "semeru17" ]]; then export GRADLE_PARAMS="$GRADLE_PARAMS -PcheckCoverage"; fi - - !reference [.test_job_common, script] - -test_smoke: - extends: .test_job_amd64_with_test_agent - # needs:parallel:matrix limits this job to a specific build_tests combination. - # Keep matrix vars exact and in build_tests declaration order: - # https://docs.gitlab.com/ci/yaml/#needsparallelmatrix - needs: &needs_build_tests_smoke - - job: build_tests - parallel: - matrix: - - GRADLE_TARGET: ":smokeTest" - CACHE_TYPE: "smoke" - MAVEN_OPTS: "-Xms256M -Xmx1024M" - variables: - <<: *tier_l_variables - GRADLE_TARGET: "stageMainDist :smokeTest" - GRADLE_PARAMS: "-PskipFlakyTests" - CACHE_TYPE: "smoke" - parallel: - matrix: *test_matrix_8 - -test_smoke_arm64: - extends: .test_job_arm64_with_test_agent - variables: - <<: *tier_l_variables - GRADLE_TARGET: "stageMainDist :smokeTest" - GRADLE_PARAMS: "-PskipFlakyTests" - CACHE_TYPE: "smoke" - parallel: - matrix: *test_matrix_8 - -test_ssi_smoke: - extends: .test_job_amd64 - needs: *needs_build_tests_smoke - rules: - - if: $CI_COMMIT_BRANCH == "master" - when: on_success - - if: '$CI_COMMIT_BRANCH =~ /^mq-working-branch-/' - when: on_success - - if: '$CI_COMMIT_BRANCH =~ /^gh-readonly-queue/' - when: on_success - variables: - <<: *tier_l_variables - GRADLE_TARGET: "stageMainDist :smokeTest" - CACHE_TYPE: "smoke" - DD_INJECT_FORCE: "true" - DD_INJECTION_ENABLED: "tracer" - parallel: - matrix: *test_matrix_8 - -test_ssi_smoke_arm64: - extends: .test_job_arm64 - variables: - <<: *tier_l_variables - GRADLE_TARGET: "stageMainDist :smokeTest" - CACHE_TYPE: "smoke" - DD_INJECT_FORCE: "true" - DD_INJECTION_ENABLED: "tracer" - parallel: - matrix: *test_matrix_8 - -test_smoke_graalvm: - extends: .test_job_amd64 - needs: *needs_build_tests_smoke - tags: [ "arch:amd64" ] - variables: - <<: *tier_l_variables - GRADLE_TARGET: "stageMainDist :dd-smoke-test:spring-boot-3.0-native:test :dd-smoke-test:quarkus-native:test" - CACHE_TYPE: "smoke" - CI_NO_SPLIT: "true" - NON_DEFAULT_JVMS: "true" - parallel: - matrix: - - testJvm: ["graalvm17", "graalvm21", "graalvm25"] - -test_smoke_graalvm_arm64: - extends: .test_job_arm64 - tags: [ "arch:arm64" ] - variables: - <<: *tier_l_variables - GRADLE_TARGET: "stageMainDist :dd-smoke-test:spring-boot-3.0-native:test :dd-smoke-test:quarkus-native:test" - CACHE_TYPE: "smoke" - CI_NO_SPLIT: "true" - NON_DEFAULT_JVMS: "true" - parallel: - matrix: - - testJvm: ["graalvm17", "graalvm21", "graalvm25"] - -test_smoke_semeru8_debugger: - extends: .test_job_amd64 - needs: *needs_build_tests_smoke - tags: [ "arch:amd64" ] - variables: - GRADLE_TARGET: "stageMainDist dd-smoke-tests:debugger-integration-tests:test" - CACHE_TYPE: "smoke" - NON_DEFAULT_JVMS: "true" - testJvm: "semeru8" - -aggregate_test_counts: - image: ${BUILDER_IMAGE_REPO}:${BUILDER_IMAGE_VERSION_PREFIX}base - stage: test-summary - # Keep stage ordering, but prevent GitLab from downloading all previous-stage artifacts. - dependencies: [] - rules: - - if: '$POPULATE_CACHE' - when: never - - if: '$CI_COMMIT_BRANCH == "master"' - when: on_success - - if: '$CI_COMMIT_BRANCH =~ /^mq-working-branch-/' - when: on_success - - if: '$CI_COMMIT_BRANCH =~ /^gh-readonly-queue/' - when: on_success - - if: '$CI_COMMIT_BRANCH' - when: on_success - script: - - *set_datadog_api_keys - - export TEST_COUNTS_S3_PREFIX="test-counts/${CI_PIPELINE_ID}" - - mkdir -p ./test_counts_aggregate - - echo "Downloading test count files from s3://${TEST_COUNTS_S3_BUCKET}/${TEST_COUNTS_S3_PREFIX}/" - - aws s3 cp "s3://${TEST_COUNTS_S3_BUCKET}/${TEST_COUNTS_S3_PREFIX}/" ./test_counts_aggregate/ --recursive --exclude "*" --include "test_counts_*.json" --only-show-errors - - find ./test_counts_aggregate -name 'test_counts_*.json' -type f -print | sort - - .gitlab/aggregate_test_counts.sh ./test_counts_aggregate - artifacts: - when: always - paths: - - test_counts_aggregate/test_counts_*.json - - test_counts_summary.json - - test_counts_report.md - -deploy_to_profiling_backend: - stage: publish - needs: [ build ] - rules: - - if: '$POPULATE_CACHE' - when: never - - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' - when: on_success - - if: '$CI_COMMIT_TAG =~ /^v.*/' - when: on_success - - when: manual - allow_failure: true - trigger: - project: DataDog/profiling-backend - branch: dogfooding - variables: - UPSTREAM_PACKAGE_JOB: $BUILD_JOB_NAME - UPSTREAM_PACKAGE_JOB_ID: $BUILD_JOB_ID - UPSTREAM_PROJECT_ID: $CI_PROJECT_ID - UPSTREAM_PROJECT_NAME: $CI_PROJECT_NAME - UPSTREAM_PIPELINE_ID: $CI_PIPELINE_ID - UPSTREAM_BRANCH: $CI_COMMIT_BRANCH - UPSTREAM_TAG: $CI_COMMIT_TAG - -deploy_to_di_backend:manual: - stage: publish - needs: [ build ] - rules: - - if: '$POPULATE_CACHE' - when: never - - when: manual - allow_failure: true - trigger: - project: DataDog/debugger-demos - branch: main - variables: - UPSTREAM_PACKAGE_JOB: build - UPSTREAM_PROJECT_ID: $CI_PROJECT_ID - UPSTREAM_PROJECT_NAME: $CI_PROJECT_NAME - UPSTREAM_PIPELINE_ID: $CI_PIPELINE_ID - UPSTREAM_BRANCH: $CI_COMMIT_BRANCH - UPSTREAM_TAG: $CI_COMMIT_TAG - UPSTREAM_COMMIT_AUTHOR: $CI_COMMIT_AUTHOR - UPSTREAM_COMMIT_SHORT_SHA: $CI_COMMIT_SHORT_SHA - -deploy_to_reliability_env: - needs: [ build ] - -# If the deploy_to_maven_central job is re-run, re-trigger the deploy_artifacts_to_github job as well so that the artifacts match. -deploy_to_maven_central: - extends: .gradle_build - stage: publish-release-artifacts - needs: - - job: build - - job: system_tests - optional: true - variables: - CACHE_TYPE: "lib" - rules: - - if: '$POPULATE_CACHE' - when: never - - if: '$CI_COMMIT_BRANCH == "master"' - when: on_success - # Do not deploy release candidate versions - - if: '$CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+$/' - when: on_success - allow_failure: true - - when: manual - allow_failure: true - script: - - export MAVEN_CENTRAL_USERNAME=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.central_username --with-decryption --query "Parameter.Value" --out text) - - export MAVEN_CENTRAL_PASSWORD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.central_password --with-decryption --query "Parameter.Value" --out text) - - export GPG_PRIVATE_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.signing.gpg_private_key --with-decryption --query "Parameter.Value" --out text) - - export GPG_PASSWORD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.signing.gpg_passphrase --with-decryption --query "Parameter.Value" --out text) - - ./gradlew publishToSonatype closeSonatypeStagingRepository -PskipTests $GRADLE_ARGS - artifacts: - paths: - - 'workspace/dd-java-agent/build/libs/*.jar' - - 'workspace/dd-trace-api/build/libs/*.jar' - - 'workspace/dd-trace-ot/build/libs/*.jar' - -deploy_snapshot_with_ddprof_snapshot: - extends: .gradle_build - stage: publish - needs: [ build ] - variables: - CACHE_TYPE: "lib" - rules: - - if: '$POPULATE_CACHE' - when: never - # Manual trigger only - for testing with ddprof snapshot versions - - when: manual - allow_failure: true - script: - - export MAVEN_CENTRAL_USERNAME=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.central_username --with-decryption --query "Parameter.Value" --out text) - - export MAVEN_CENTRAL_PASSWORD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.central_password --with-decryption --query "Parameter.Value" --out text) - - export GPG_PRIVATE_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.signing.gpg_private_key --with-decryption --query "Parameter.Value" --out text) - - export GPG_PASSWORD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.signing.gpg_passphrase --with-decryption --query "Parameter.Value" --out text) - - echo "Publishing dd-trace-java snapshot with ddprof snapshot dependency" - - ./gradlew -PbuildInfo.build.number=$CI_JOB_ID -PddprofUseSnapshot publishToSonatype -PskipTests $GRADLE_ARGS - artifacts: - paths: - - 'workspace/dd-java-agent/build/libs/*.jar' - - 'workspace/dd-trace-api/build/libs/*.jar' - - 'workspace/dd-trace-ot/build/libs/*.jar' - -deploy_artifacts_to_github: - stage: publish-release-artifacts - image: registry.ddbuild.io/images/dd-octo-sts-ci-base:2025.06-1 - tags: [ "arch:amd64" ] - id_tokens: - DDOCTOSTS_ID_TOKEN: - aud: dd-octo-sts - rules: - - if: '$POPULATE_CACHE' - when: never - - if: '$CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+$/' - when: on_success - # Requires the deploy_to_maven_central job to have run first (the UP-TO-DATE gradle check across jobs is broken) - # This will deploy the artifacts built from the publishToSonatype task to the GitHub release - needs: - - job: deploy_to_maven_central - # The deploy_to_maven_central job is not run for release candidate versions - optional: true - before_script: - - dd-octo-sts version - - dd-octo-sts debug --scope DataDog/dd-trace-java --policy self.gitlab.release - - dd-octo-sts token --scope DataDog/dd-trace-java --policy self.gitlab.release > github-token.txt - script: - - gh auth login --with-token < github-token.txt - - gh auth status - - export VERSION=${CI_COMMIT_TAG##v} # remove "v" from front of tag to get version - - cp workspace/dd-java-agent/build/libs/dd-java-agent-${VERSION}.jar workspace/dd-java-agent/build/libs/dd-java-agent.jar # upload two filenames - - gh release upload --clobber --repo DataDog/dd-trace-java $CI_COMMIT_TAG workspace/dd-java-agent/build/libs/dd-java-agent.jar - - gh release upload --clobber --repo DataDog/dd-trace-java $CI_COMMIT_TAG workspace/dd-java-agent/build/libs/dd-java-agent-${VERSION}.jar - - gh release upload --clobber --repo DataDog/dd-trace-java $CI_COMMIT_TAG workspace/dd-trace-api/build/libs/dd-trace-api-${VERSION}.jar - - gh release upload --clobber --repo DataDog/dd-trace-java $CI_COMMIT_TAG workspace/dd-trace-ot/build/libs/dd-trace-ot-${VERSION}.jar - after_script: - - dd-octo-sts revoke -t $(cat github-token.txt) - retry: - max: 2 - when: always - -requirements_json_test: - rules: - - when: on_success - variables: - REQUIREMENTS_BLOCK_JSON_PATH: "metadata/requirements-block.json" - REQUIREMENTS_ALLOW_JSON_PATH: "metadata/requirements-allow.json" - -package-oci: - needs: [ build ] - -override_verify_maven_central: - image: registry.ddbuild.io/images/base/gbi-ubuntu_2204:release - stage: publish - needs: [ ] - rules: - - if: '$POPULATE_CACHE' - when: never - - when: manual - allow_failure: true - script: - - touch OVERRIDE_MAVEN_VERIFY - cache: # Cache is used to signal between the override_verify_maven_central and verify_maven_central_deployment jobs - - key: $CI_PIPELINE_ID-OVERRIDE_SIGNAL - paths: - - OVERRIDE_MAVEN_VERIFY - policy: push - unprotect: true - -# Verify Maven Central deployment is publicly available before publishing OCI images -verify_maven_central_deployment: - image: registry.ddbuild.io/images/base/gbi-ubuntu_2204:release - stage: publish-release-artifacts - needs: [ deploy_to_maven_central ] - rules: - - if: '$POPULATE_CACHE' - when: never - - if: '$CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+$/' - when: on_success - cache: # Cache is used to signal between the override_verify_maven_central and verify_maven_central_deployment jobs - - key: $CI_PIPELINE_ID-OVERRIDE_SIGNAL - paths: - - OVERRIDE_MAVEN_VERIFY - policy: pull - unprotect: true - script: - - if [ -f OVERRIDE_MAVEN_VERIFY ]; then echo "SKIPPING MAVEN VERIFICATION"; exit 0; fi - - | - export VERSION=${CI_COMMIT_TAG##v} - ARTIFACT_URLS=( - "https://repo1.maven.org/maven2/com/datadoghq/dd-java-agent/${VERSION}/dd-java-agent-${VERSION}.jar" - "https://repo1.maven.org/maven2/com/datadoghq/dd-trace-api/${VERSION}/dd-trace-api-${VERSION}.jar" - "https://repo1.maven.org/maven2/com/datadoghq/dd-trace-ot/${VERSION}/dd-trace-ot-${VERSION}.jar" - ) - # Try once immediately (fast path on job retry), then every 5 mins for 45 mins. - TRY=0 - MAX_TRIES=10 - RETRY_DELAY=300 - while [ $TRY -lt $MAX_TRIES ]; do - ARTIFACTS_AVAILABLE=true - for URL in "${ARTIFACT_URLS[@]}"; do - if ! curl --location --fail --silent --show-error -I "$URL"; then - ARTIFACTS_AVAILABLE=false - break - fi - done - if [ "$ARTIFACTS_AVAILABLE" = true ]; then - break - fi - TRY=$((TRY + 1)) - if [ $TRY -eq $MAX_TRIES ]; then - echo "The release was not available after 45 mins. Manually re-run the job to try again." - exit 1 - fi - sleep $RETRY_DELAY - done - -publishing-gate: - stage: publish-release-artifacts - needs: - - job: verify_maven_central_deployment - optional: true # Required for releases only - -configure_system_tests: - variables: - SYSTEM_TESTS_REF: "main" # system tests are pinned on release branches only - SYSTEM_TESTS_SCENARIOS_GROUPS: "simple_onboarding,simple_onboarding_profiling,simple_onboarding_appsec,docker-ssi,lib-injection" - -create_key: - stage: generate-signing-key - when: manual - needs: [ ] - variables: - PROJECT_NAME: "dd-trace-java" - EXPORT_TO_KEYSERVER: "true" - image: $REGISTRY/ci/agent-key-management-tools/gpg:1 - script: - - /create.sh - artifacts: - expire_in: 13 mos - paths: - - pubkeys - -validate_supported_configurations_v2_local_file: - extends: .validate_supported_configurations_v2_local_file - variables: - LOCAL_JSON_PATH: "metadata/supported-configurations.json" - BACKFILLED: "false" - -update_central_configurations_version_range_v2: - extends: .update_central_configurations_version_range_v2 - variables: - LOCAL_REPO_NAME: "dd-trace-java" - LOCAL_JSON_PATH: "metadata/supported-configurations.json" - LANGUAGE_NAME: "java" - MULTIPLE_RELEASE_LINES: "false" - SEND_ALIASES: "true" + - ./gradlew -p build-logic :smoke-test:test --tests '*SmokeTestAppPluginTest' --tests '*SmokeTestAppEndToEndTest.nested build clears inherited Gradle launcher environment' --no-build-cache --rerun-tasks $GRADLE_PARAMS From 1cf27949ce531a267eac7ab4caaa80da8269cc02 Mon Sep 17 00:00:00 2001 From: Brice Dutheil Date: Thu, 8 Oct 2026 15:20:45 +0200 Subject: [PATCH 3/4] ci: select Java 25 for focused checks --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e17ad7618ae..b16e72aab6c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -247,5 +247,5 @@ check_build_src: GRADLE_TARGET: ":smoke-test:test" script: - *gitlab_base_ref_params - - ./gradlew --version - - ./gradlew -p build-logic :smoke-test:test --tests '*SmokeTestAppPluginTest' --tests '*SmokeTestAppEndToEndTest.nested build clears inherited Gradle launcher environment' --no-build-cache --rerun-tasks $GRADLE_PARAMS + - JAVA_HOME="$JAVA_25_HOME" ./gradlew --version + - JAVA_HOME="$JAVA_25_HOME" ./gradlew -p build-logic :smoke-test:test -Dorg.gradle.java.installations.fromEnv=JAVA_21_HOME,JAVA_25_HOME --tests '*SmokeTestAppPluginTest' --tests '*SmokeTestAppEndToEndTest.nested build clears inherited Gradle launcher environment' --no-build-cache --rerun-tasks --stacktrace --parallel --max-workers=$GRADLE_WORKERS $GRADLE_PARAMS From 480004d69624b2b35020e1532bfdd377f137f867 Mon Sep 17 00:00:00 2001 From: Brice Dutheil Date: Thu, 8 Oct 2026 15:31:33 +0200 Subject: [PATCH 4/4] ci: restore pipeline after cleanup checks --- .gitlab-ci.yml | 1265 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 1262 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b16e72aab6c..8dbde59e860 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,3 +1,25 @@ +include: + - local: ".gitlab/one-pipeline.locked.yml" + - local: ".gitlab/benchmarks.yml" + - local: ".gitlab/exploration-tests.yml" + - local: ".gitlab/ci-visibility-tests.yml" + - project: 'DataDog/apm-reliability/apm-sdks-benchmarks' + file: '.gitlab/ci-java-spring-petclinic-parallel.yml' + ref: &apm_sdks_benchmarks_sha '454a97647cbc16ab7fd20e917aa31532ba7532eb' # pinned by .github/workflows/update-apm-sdks-benchmarks-reference.yaml + - project: 'DataDog/apm-reliability/apm-sdks-benchmarks' + file: '.gitlab/ci-java-load-parallel.yml' + ref: *apm_sdks_benchmarks_sha + - project: 'DataDog/apm-reliability/apm-sdks-benchmarks' + file: '.gitlab/ci-java-startup-parallel.yml' + ref: *apm_sdks_benchmarks_sha + - project: 'DataDog/apm-reliability/apm-sdks-benchmarks' + file: '.gitlab/ci-java-dacapo-parallel.yml' + ref: *apm_sdks_benchmarks_sha + - project: 'DataDog/apm-reliability/apm-sdks-benchmarks' + file: '.gitlab/ci-java-post-pr-comment.yml' + ref: *apm_sdks_benchmarks_sha + - local: ".gitlab/java-benchmark-configs.yml" + stages: - build - publish @@ -23,6 +45,7 @@ stages: - generate-signing-key variables: + APM_SDKS_BENCHMARKS_SHA: *apm_sdks_benchmarks_sha # Test the OpenTelemetry Operator-compatible Java image jobs from one-pipeline. CI_OTEL_OPERATOR_IMAGES_ENABLED: "true" CI_OTEL_OPERATOR_LANGUAGE: java @@ -74,10 +97,58 @@ workflow: on_new_commit: none - when: always +.test_matrix: &test_matrix + - testJvm: &test_jvms + - "8" + - "11" + - "17" + - "21" + - "25" + - "27" # JDK 27 TODO: remove after GA (tip will move to 27) + - "semeru11" + - "oracle8" + - "zulu8" + - "semeru8" + - "ibm8" + - "zulu11" + - "semeru17" + - "tip" + CI_SPLIT: ["1/1"] + +# Gitlab doesn't support "parallel" and "parallel:matrix" at the same time +# These blocks emulate "parallel" by including it in the matrix +.test_matrix_2: &test_matrix_2 + - testJvm: *test_jvms + CI_SPLIT: ["1/2", "2/2"] + +.test_matrix_4: &test_matrix_4 + - testJvm: *test_jvms + CI_SPLIT: ["1/4", "2/4", "3/4", "4/4"] + +.test_matrix_6: &test_matrix_6 + - testJvm: *test_jvms + CI_SPLIT: ["1/6", "2/6", "3/6", "4/6", "5/6", "6/6"] + +.test_matrix_8: &test_matrix_8 + - testJvm: *test_jvms + CI_SPLIT: ["1/8", "2/8", "3/8", "4/8", "5/8", "6/8", "7/8", "8/8"] + +.test_matrix_12: &test_matrix_12 + - testJvm: *test_jvms + CI_SPLIT: [ "1/12", "2/12", "3/12", "4/12", "5/12", "6/12", "7/12", "8/12", "9/12", "10/12", "11/12", "12/12" ] + +.master_only: &master_only + - if: $CI_COMMIT_BRANCH == "master" + when: on_success + default: tags: [ "arch:amd64" ] interruptible: true +.set_datadog_api_keys: &set_datadog_api_keys + - export DATADOG_API_KEY_PROD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.DATADOG_API_KEY_PROD --with-decryption --query "Parameter.Value" --out text) + +# CI_NODE_INDEX and CI_NODE_TOTAL are 1-indexed and not always set. These steps normalize the numbers for jobs .normalize_node_index: &normalize_node_index - if [ "$CI_NO_SPLIT" == "true" ] ; then CI_NODE_INDEX=1; CI_NODE_TOTAL=1; fi # A job uses parallel but doesn't intend to split by index - if [ -n "$CI_SPLIT" ]; then CI_NODE_INDEX="${CI_SPLIT%%/*}"; CI_NODE_TOTAL="${CI_SPLIT##*/}"; fi @@ -113,6 +184,21 @@ default: KUBERNETES_MEMORY_REQUEST: 16Gi KUBERNETES_MEMORY_LIMIT: 16Gi +.tier_l: + variables: &tier_l_variables + GRADLE_WORKERS: 6 + GRADLE_MEMORY_MIN: 1G + GRADLE_MEMORY_MAX: 6G + KUBERNETES_CPU_REQUEST: 10 + KUBERNETES_MEMORY_REQUEST: 20Gi + KUBERNETES_MEMORY_LIMIT: 20Gi + +.tier_xl: + variables: &tier_xl_variables + <<: *tier_l_variables + KUBERNETES_MEMORY_REQUEST: 32Gi + KUBERNETES_MEMORY_LIMIT: 32Gi + .gitlab_base_ref_params: &gitlab_base_ref_params - | export GIT_BASE_REF=$(.gitlab/find-gh-base-ref.sh) @@ -204,6 +290,326 @@ default: # Check and fail early if maven central credentials are incorrect. When a new token is generated # on the central publisher portal, it invalidates the old one. This check prevents going further. # See https://datadoghq.atlassian.net/wiki/x/Oog5OgE +maven-central-pre-release-check: + image: ${BUILDER_IMAGE_REPO}:${BUILDER_IMAGE_VERSION_PREFIX}base + stage: .pre + rules: + - if: '$CI_COMMIT_BRANCH == "master"' + when: on_success + allow_failure: false + - if: '$CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+$/' + when: on_success + allow_failure: false + script: + - | + MAVEN_CENTRAL_USERNAME=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.central_username --with-decryption --query "Parameter.Value" --out text) + MAVEN_CENTRAL_PASSWORD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.central_password --with-decryption --query "Parameter.Value" --out text) + # See https://central.sonatype.org/publish/publish-portal-ossrh-staging-api/ + # Use the staging API search endpoint to validate the tokens without relying on a specific deployment + AUTHORIZATION_HEADER="Authorization: Bearer $(printf '%s:%s' "$MAVEN_CENTRAL_USERNAME" "$MAVEN_CENTRAL_PASSWORD" | base64)" + if ! curl --silent --show-error --fail \ + "https://ossrh-staging-api.central.sonatype.com/manual/search/repositories?ip=any" \ + --header "$AUTHORIZATION_HEADER" \ + > /dev/null; then + echo "Failed to authenticate tokens against maven central staging API. Check credentials and see https://datadoghq.atlassian.net/wiki/x/Oog5OgE" + exit 1 + fi + +dd-octo-sts-pre-release-check: + image: registry.ddbuild.io/images/dd-octo-sts-ci-base:2025.06-1 + stage: .pre + tags: [ "arch:amd64" ] + id_tokens: + DDOCTOSTS_ID_TOKEN: + aud: dd-octo-sts + rules: + - if: '$POPULATE_CACHE' + when: never + - if: '$CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+$/' + when: on_success + allow_failure: false + before_script: + - dd-octo-sts version + - dd-octo-sts debug --scope DataDog/dd-trace-java --policy self.gitlab.release + - dd-octo-sts token --scope DataDog/dd-trace-java --policy self.gitlab.release > test-github-token.txt + script: + - gh auth login --with-token < test-github-token.txt + - gh auth status + after_script: + - dd-octo-sts revoke -t $(cat test-github-token.txt) + retry: + max: 2 + when: always + +build: + needs: + - job: maven-central-pre-release-check + optional: true + - job: dd-octo-sts-pre-release-check + optional: true + extends: .gradle_build + variables: + BUILD_CACHE_POLICY: push + CACHE_TYPE: "lib" + DEPENDENCY_CACHE_POLICY: pull + script: + - if [ $CI_PIPELINE_SOURCE == "schedule" ] ; then ./gradlew resolveAndLockAll --write-locks $GRADLE_ARGS; fi + - ./gradlew --version + - ./gradlew clean :dd-java-agent:shadowJar :dd-java-agent:check :dd-trace-api:jar :dd-trace-ot:shadowJar :products:feature-flagging:feature-flagging-api:jar -PskipTests -x spotlessCheck $GRADLE_ARGS + - echo UPSTREAM_TRACER_VERSION=$(java -jar workspace/dd-java-agent/build/libs/*.jar) >> upstream.env + - echo "BUILD_JOB_NAME=$CI_JOB_NAME" >> build.env + - echo "BUILD_JOB_ID=$CI_JOB_ID" >> build.env + artifacts: + when: always + paths: + - 'workspace/dd-java-agent/build/libs/*.jar' + - 'workspace/dd-trace-api/build/libs/*.jar' + - 'workspace/dd-trace-ot/build/libs/*.jar' + - 'workspace/products/feature-flagging/feature-flagging-api/build/libs/*.jar' + - 'upstream.env' + - '.gradle/daemon/*/*.out.log' + reports: + dotenv: build.env + +build_tests: + extends: .gradle_build + variables: + <<: *tier_xl_variables + BUILD_CACHE_POLICY: push + DEPENDENCY_CACHE_POLICY: pull + parallel: + matrix: + - GRADLE_TARGET: ":baseTest" + CACHE_TYPE: "base" + - GRADLE_TARGET: ":profilingTest" + CACHE_TYPE: "profiling" + - GRADLE_TARGET: ":instrumentationTest" + CACHE_TYPE: "inst" + - GRADLE_TARGET: ":instrumentationLatestDepTest" + CACHE_TYPE: "latestdep" + - GRADLE_TARGET: ":smokeTest" + CACHE_TYPE: "smoke" + MAVEN_OPTS: "-Xms256M -Xmx1024M" + script: + - *gitlab_base_ref_params + - ./gradlew --version + - ./gradlew clean $GRADLE_TARGET $GRADLE_PARAMS -PskipTests $GRADLE_ARGS + +populate_plugin_cache: + extends: .gradle_build + cache: + - &plugin_cache + key: dependency-plugins-v1 + paths: + - .gradle-plugin-cache/caches/modules-2 + policy: pull-push + when: always + unprotect: true + rules: + - if: '$POPULATE_CACHE' + when: on_success + script: + - mkdir -p .gradle-plugin-cache + - sudo chown -R 1001:1001 .gradle-plugin-cache + - export GRADLE_USER_HOME=$(pwd)/.gradle-plugin-cache + - | + gradle_status=0 + ./gradlew help -PskipTests $GRADLE_ARGS || gradle_status=$? + if ! find .gradle-plugin-cache/caches/modules-2/files-2.1 -type f -print -quit 2>/dev/null | grep -q .; then + echo "WARNING: No plugin artifacts were cached; skipping the cache upload." + rm -rf .gradle-plugin-cache/caches/modules-2 + fi + exit "$gradle_status" + # This cache warmer is an optimization; population jobs can continue with a cold plugin cache. + allow_failure: true + +populate_dep_cache: + extends: build_tests + # Wait for build-cache producers so their caches are uploaded before this job restores them. + needs: + - job: populate_plugin_cache + - job: build + artifacts: false + - job: build_tests + artifacts: false + variables: + BUILD_CACHE_POLICY: pull + DEPENDENCY_CACHE_POLICY: push + cache: + - <<: *plugin_cache + policy: pull + - *dependency_cache + - *build_cache + rules: + - if: '$POPULATE_CACHE' + when: on_success + before_script: + - !reference [.gradle_build, before_script] + # Seed the otherwise cold writable cache, keeping the cache uploaded below self-contained. + - mkdir -p .gradle/caches + - | + if [ -d .gradle-plugin-cache/caches/modules-2 ]; then + sudo chown -R 1001:1001 .gradle-plugin-cache + cp -a .gradle-plugin-cache/caches/modules-2 .gradle/caches/ + else + echo "Plugin cache is unavailable; continuing with a cold dependency cache." + fi + parallel: + matrix: + - GRADLE_TARGET: ":dd-java-agent:shadowJar :dd-trace-api:jar :dd-trace-ot:shadowJar" + CACHE_TYPE: "lib" + - GRADLE_TARGET: ":baseTest" + CACHE_TYPE: "base" + - GRADLE_TARGET: ":profilingTest" + CACHE_TYPE: "profiling" + - GRADLE_TARGET: ":instrumentationTest" + CACHE_TYPE: "inst" + - GRADLE_TARGET: ":instrumentationLatestDepTest" + CACHE_TYPE: "latestdep" + - GRADLE_TARGET: ":smokeTest" + CACHE_TYPE: "smoke" + - GRADLE_TARGET: "spotlessCheck" + CACHE_TYPE: "spotless" + GRADLE_MEMORY_MAX: "6G" + +publish-artifacts-to-s3: + image: registry.ddbuild.io/images/mirror/amazon/aws-cli:2.4.29 + stage: publish + needs: [ build ] + script: + - source upstream.env + - export VERSION="${UPSTREAM_TRACER_VERSION%~*}" # remove ~githash from the end of version + - aws s3 cp workspace/dd-java-agent/build/libs/dd-java-agent-${VERSION}.jar s3://dd-trace-java-builds/${CI_COMMIT_REF_NAME}/dd-java-agent.jar + - aws s3 cp workspace/dd-trace-api/build/libs/dd-trace-api-${VERSION}.jar s3://dd-trace-java-builds/${CI_COMMIT_REF_NAME}/dd-trace-api.jar + - aws s3 cp workspace/dd-trace-ot/build/libs/dd-trace-ot-${VERSION}.jar s3://dd-trace-java-builds/${CI_COMMIT_REF_NAME}/dd-trace-ot.jar + - aws s3 cp workspace/products/feature-flagging/feature-flagging-api/build/libs/dd-openfeature-${VERSION}.jar s3://dd-trace-java-builds/${CI_COMMIT_REF_NAME}/dd-openfeature.jar + - aws s3 cp workspace/dd-java-agent/build/libs/dd-java-agent-${VERSION}.jar s3://dd-trace-java-builds/${CI_PIPELINE_ID}/dd-java-agent.jar + - aws s3 cp workspace/dd-trace-api/build/libs/dd-trace-api-${VERSION}.jar s3://dd-trace-java-builds/${CI_PIPELINE_ID}/dd-trace-api.jar + - aws s3 cp workspace/dd-trace-ot/build/libs/dd-trace-ot-${VERSION}.jar s3://dd-trace-java-builds/${CI_PIPELINE_ID}/dd-trace-ot.jar + - aws s3 cp workspace/products/feature-flagging/feature-flagging-api/build/libs/dd-openfeature-${VERSION}.jar s3://dd-trace-java-builds/${CI_PIPELINE_ID}/dd-openfeature.jar + - | + cat << EOF > links.json + { + "S3 Links": [ + { + "external_link": { + "label": "Public Link to dd-java-agent.jar", + "url": "https://s3.us-east-1.amazonaws.com/dd-trace-java-builds/${CI_PIPELINE_ID}/dd-java-agent.jar" + } + }, + { + "external_link": { + "label": "Public Link to dd-openfeature.jar", + "url": "https://s3.us-east-1.amazonaws.com/dd-trace-java-builds/${CI_PIPELINE_ID}/dd-openfeature.jar" + } + } + ] + } + EOF + artifacts: + reports: + annotations: + - links.json + + +spotless: + extends: .gradle_build + stage: tests + needs: [] + variables: + GRADLE_MEMORY_MAX: 6G + CACHE_TYPE: "spotless" + script: + - ./gradlew --version + # test-published-dependencies's build file needs main version file + - ./gradlew spotlessCheck writeMainVersionFile $GRADLE_ARGS + - cd test-published-dependencies && ./gradlew spotlessCheck $GRADLE_ARGS + +check-instrumentation-naming: + extends: .gradle_build + stage: tests + needs: [ ] + script: + - ./gradlew --version + - ./gradlew checkInstrumentationNaming + +config-inversion-linter: + extends: .gradle_build + stage: tests + needs: [] + script: + - ./gradlew --version + - ./gradlew checkConfigurations + +test_published_artifacts: + extends: .gradle_build + image: ${BUILDER_IMAGE_REPO}:${BUILDER_IMAGE_VERSION_PREFIX}7 # Needs Java7 for some tests + stage: tests + needs: [ build ] + variables: + CACHE_TYPE: "lib" + script: + - mvn_local_repo=$(./mvnw help:evaluate -Dexpression=settings.localRepository -q -DforceStdout) + - rm -rf "${mvn_local_repo}/com/datadoghq" + - export GPG_PRIVATE_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.signing.gpg_private_key --with-decryption --query "Parameter.Value" --out text) + - export GPG_PASSWORD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.signing.gpg_passphrase --with-decryption --query "Parameter.Value" --out text) + - export GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xms2G -Xmx2G -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp'" + - ./gradlew publishToMavenLocal $GRADLE_ARGS + - cd test-published-dependencies + - printf '\norg.gradle.console=colored\n' >> gradle.properties + - export GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xms1G -Xmx1G -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp'" + - ./gradlew --version + - ./gradlew check --info $GRADLE_ARGS + after_script: + - *cgroup_info + - source .gitlab/gitlab-utils.sh + - gitlab_section_start "collect-reports" "Collecting reports" + - .gitlab/collect_reports.sh + - gitlab_section_end "collect-reports" + artifacts: + when: always + paths: + - ./check_reports + +validate_build: + extends: .gradle_build + stage: tests + needs: [ build ] + variables: + CACHE_TYPE: "lib" + script: + # Preserve the `build` job artifacts before Gradle rebuilds and overwrites them under + # workspace/**/build/libs, so jardiff can compare the rebuilt jars against them. + - mkdir -p reference-artifacts + - cp workspace/dd-java-agent/build/libs/*.jar reference-artifacts/ + - cp workspace/dd-trace-api/build/libs/*.jar reference-artifacts/ + - cp workspace/dd-trace-ot/build/libs/*.jar reference-artifacts/ + # Scheduled builds refresh dependency locks before creating the reference artifacts. + # Refresh them here as well so the candidate uses the same dependency resolution. + - if [ $CI_PIPELINE_SOURCE == "schedule" ] ; then ./gradlew resolveAndLockAll --write-locks $GRADLE_ARGS; fi + - ./gradlew --version + # This will run the shadowJar task and exercise the build cache, allowing to identify build-cache issues + # Keep both JAR sets for direct inspection when the comparison fails. + - | + if ! ./gradlew compareToReferenceJar -PjardiffReferenceDir="$CI_PROJECT_DIR/reference-artifacts" -PskipTests $GRADLE_ARGS; then + mkdir -p check_reports/jar-comparison-artifacts/reference check_reports/jar-comparison-artifacts/candidate + cp reference-artifacts/*.jar check_reports/jar-comparison-artifacts/reference/ + cp workspace/dd-java-agent/build/libs/*.jar check_reports/jar-comparison-artifacts/candidate/ + cp workspace/dd-trace-api/build/libs/*.jar check_reports/jar-comparison-artifacts/candidate/ + cp workspace/dd-trace-ot/build/libs/*.jar check_reports/jar-comparison-artifacts/candidate/ + exit 1 + fi + after_script: + - source .gitlab/gitlab-utils.sh + - gitlab_section_start "collect-reports" "Collecting reports" + - .gitlab/collect_reports.sh --destination ./check_reports + - gitlab_section_end "collect-reports" + artifacts: + when: always + paths: + - ./check_reports + - '.gradle/daemon/*/*.out.log' + .check_job: extends: .gradle_build needs: [ build ] @@ -244,8 +650,861 @@ check_build_src: extends: .check_job needs: [] variables: - GRADLE_TARGET: ":smoke-test:test" + GRADLE_TARGET: ":buildSrc:build :build-logic:check" + +check_base: + extends: .check_job + variables: + GRADLE_TARGET: ":baseCheck" + +check_inst: + extends: .check_job + parallel: 4 + variables: + GRADLE_TARGET: ":instrumentationCheck" + CACHE_TYPE: "inst" + +check_smoke: + extends: .check_job + parallel: 4 + variables: + GRADLE_TARGET: ":smokeCheck" + CACHE_TYPE: "smoke" + +check_profiling: + extends: .check_job + variables: + GRADLE_TARGET: ":profilingCheck" + +check_debugger: + extends: .check_job + variables: + GRADLE_TARGET: ":debuggerCheck" + +muzzle: + extends: .gradle_build + # needs:parallel:matrix limits this job to a specific build_tests combination. + # Keep matrix vars exact and in build_tests declaration order: + # https://docs.gitlab.com/ci/yaml/#needsparallelmatrix + needs: &needs_build_tests_inst + - job: build_tests + parallel: + matrix: + - GRADLE_TARGET: ":instrumentationTest" + CACHE_TYPE: "inst" + stage: tests + rules: + - if: '$CI_COMMIT_BRANCH =~ /^mq-working-branch-/' + when: never + - if: '$CI_COMMIT_BRANCH =~ /^gh-readonly-queue/' + when: never + - when: on_success + parallel: + matrix: + - CI_SPLIT: ["1/8", "2/8", "3/8", "4/8", "5/8", "6/8", "7/8", "8/8"] + variables: + CACHE_TYPE: "inst" + script: + - export SKIP_BUILDSCAN="true" + - ./gradlew --version + - ./gradlew :runMuzzle -Pslot=$CI_NODE_INDEX/$CI_NODE_TOTAL $GRADLE_ARGS + after_script: + - *container_info + - *cgroup_info + - *set_datadog_api_keys + - source .gitlab/gitlab-utils.sh + - gitlab_section_start "collect-reports" "Collecting reports" + - .gitlab/collect_reports.sh + - .gitlab/collect_results.sh + - .gitlab/upload_ciapp.sh $CACHE_TYPE + - gitlab_section_end "collect-reports" + artifacts: + when: always + paths: + - ./reports + - ./results + - '.gradle/daemon/*/*.out.log' + reports: + junit: results/*.xml + +muzzle-dep-report: + extends: .gradle_build + needs: *needs_build_tests_inst + stage: tests + rules: + - if: '$CI_COMMIT_BRANCH =~ /^mq-working-branch-/' + when: never + - if: '$CI_COMMIT_BRANCH =~ /^gh-readonly-queue/' + when: never + - when: on_success + variables: + CACHE_TYPE: "inst" + script: + - export SKIP_BUILDSCAN="true" + - ./gradlew --version + - ./gradlew generateMuzzleReport muzzleInstrumentationReport $GRADLE_ARGS + after_script: + - *container_info + - *cgroup_info + - .gitlab/collect_muzzle_deps.sh + artifacts: + when: always + paths: + - ./reports + - '.gradle/daemon/*/*.out.log' + +# In Gitlab, DD_* variables are set because the build runner is instrumented with Datadog telemetry +# To have a pristine environment for the tests, these variables are saved before the test run and restored afterwards +.prepare_test_env: &prepare_test_env + - export gitlabVariables=("DD_SERVICE" "DD_ENTITY_ID" "DD_SITE" "DD_ENV" "DD_DATACENTER" "DD_PARTITION" "DD_CLOUDPROVIDER") + - '[ ! -e pretest.env ] || rm pretest.env' + - | + for VARIABLE in "${gitlabVariables[@]}" + do + echo "export $VARIABLE=${!VARIABLE}" >> pretest.env + unset "$VARIABLE" + done + +.restore_pretest_env: &restore_pretest_env + - source pretest.env + +.test_job_common: + extends: .gradle_build + image: ${BUILDER_IMAGE_REPO}:${BUILDER_IMAGE_VERSION_PREFIX}$testJvm + variables: + <<: *tier_m_variables + GRADLE_PARAMS: "-PskipFlakyTests" + CONTINUE_ON_FAILURE: "false" + TEST_JVM_ARGS: "" + TESTCONTAINERS_CHECKS_DISABLE: "true" + TESTCONTAINERS_RYUK_DISABLED: "true" + TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX: "registry.ddbuild.io/images/mirror/" + JETTY_AVAILABLE_PROCESSORS: 4 # Jetty incorrectly calculates processor count in containers script: - *gitlab_base_ref_params - - JAVA_HOME="$JAVA_25_HOME" ./gradlew --version - - JAVA_HOME="$JAVA_25_HOME" ./gradlew -p build-logic :smoke-test:test -Dorg.gradle.java.installations.fromEnv=JAVA_21_HOME,JAVA_25_HOME --tests '*SmokeTestAppPluginTest' --tests '*SmokeTestAppEndToEndTest.nested build clears inherited Gradle launcher environment' --no-build-cache --rerun-tasks --stacktrace --parallel --max-workers=$GRADLE_WORKERS $GRADLE_PARAMS + - > + if [ "$PROFILE_TESTS" == "true" ] && [ "$testJvm" != "ibm8" ] && [ "$testJvm" != "oracle8" ]; + then + export PROFILER_COMMAND="-XX:StartFlightRecording=settings=profile,filename=/tmp/${CI_JOB_NAME_SLUG}.jfr,dumponexit=true"; + fi + - *prepare_test_env + - export GRADLE_OPTS="-Dorg.gradle.jvmargs='$TEST_JVM_ARGS -Xms$GRADLE_MEMORY_MIN -Xmx$GRADLE_MEMORY_MAX $PROFILER_COMMAND -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp -Djava.util.prefs.userRoot=/tmp/.java/.userPrefs-${CI_JOB_ID}' -Ddatadog.forkedMinHeapSize=128M -Ddatadog.forkedMaxHeapSize=1024M" + - ./gradlew --version + - ./gradlew $GRADLE_TARGET $GRADLE_PARAMS -PtestJvm=$testJvm -Pslot=$CI_NODE_INDEX/$CI_NODE_TOTAL $GRADLE_ARGS --continue || $CONTINUE_ON_FAILURE + after_script: + - *restore_pretest_env + - *set_datadog_api_keys + - *container_info + - *cgroup_info + - source .gitlab/gitlab-utils.sh + - gitlab_section_start "collect-reports" "Collecting reports" + - .gitlab/collect_reports.sh + - if [ "$PROFILE_TESTS" == "true" ]; then .gitlab/collect_profiles.sh; fi + - .gitlab/collect_results.sh + - .gitlab/upload_ciapp.sh $CACHE_TYPE $testJvm + - gitlab_section_end "collect-reports" + - .gitlab/count_tests.sh "$GRADLE_TARGET" "$testJvm" "./results" "./test_counts_${CI_JOB_ID}.json" + - export TEST_COUNTS_S3_PREFIX="test-counts/${CI_PIPELINE_ID}" + - export TEST_COUNTS_FILE="./test_counts_${CI_JOB_ID}.json" + # Use logical job identity in S3 so retried jobs overwrite stale attempts. + - export TEST_COUNTS_S3_FILE="test_counts_${CI_JOB_NAME_SLUG}_${CI_NODE_INDEX}-${CI_NODE_TOTAL}.json" + - export TEST_COUNTS_S3_URI="s3://${TEST_COUNTS_S3_BUCKET}/${TEST_COUNTS_S3_PREFIX}/${TEST_COUNTS_S3_FILE}" + - echo "Uploading ${TEST_COUNTS_FILE} to ${TEST_COUNTS_S3_URI}" + - aws s3 cp "$TEST_COUNTS_FILE" "$TEST_COUNTS_S3_URI" --only-show-errors + - URL_ENCODED_JOB_NAME=$(jq -rn --arg x "$CI_JOB_NAME" '$x|@uri') + - echo -e "${TEXT_BOLD}${TEXT_YELLOW}See test results in Datadog:${TEXT_CLEAR} https://app.datadoghq.com/ci/test/runs?query=test_level%3Atest%20%40test.service%3Add-trace-java%20%40ci.pipeline.id%3A${CI_PIPELINE_ID}%20%40ci.job.name%3A%22${URL_ENCODED_JOB_NAME}%22" + artifacts: + when: always + paths: + - ./reports.tar + - ./profiles.tar + - ./results + - './test_counts_*.json' + - '.gradle/daemon/*/*.out.log' + reports: + junit: results/*.xml + retry: + max: 2 + when: + - unknown_failure + - stuck_or_timeout_failure + - runner_system_failure + - unmet_prerequisites + - scheduler_failure + - data_integrity_failure + +.test_job_amd64: + extends: .test_job_common + tags: [ "docker-in-docker:amd64" ] # use docker-in-docker runner for testcontainers + needs: [ build_tests ] + stage: tests + rules: + # Protected branches (master/mq/gh-readonly): all JVMs run unconditionally + - if: '$CI_COMMIT_BRANCH == "master"' + when: on_success + - if: '$CI_COMMIT_BRANCH =~ /^mq-working-branch-/' + when: on_success + - if: '$CI_COMMIT_BRANCH =~ /^gh-readonly-queue/' + when: on_success + # Enable for default test JVMs or for NON_DEFAULT_JVMS + - if: '$NON_DEFAULT_JVMS == "true"' + when: on_success + - if: '$CI_COMMIT_MESSAGE =~ /\[ci: NON_DEFAULT_JVMS\]/' + when: on_success + - if: '$testJvm =~ $DEFAULT_TEST_JVMS' + when: on_success + +.test_job_arm64: + extends: .test_job_common + tags: [ "docker-in-docker:arm64" ] + stage: tests-arm64 + # Use the amd64 build only as a compilation gate. Do not download its platform-specific artifacts. + needs: + - job: build + artifacts: false + variables: + DEFAULT_TEST_JVMS: /^(11|17|21|25|27|tip)$/ # Java 27 TODO: remove 27 after GA (tip will move to 27) + TEST_JVM_ARGS: "-Xshare:off" + rules: + # Java 8 is excluded from the arm64 test matrix for all distributions. + # Java 8 JVMs are very slow on AARCH64. + - if: '$testJvm =~ /^(8|oracle8|zulu8|semeru8|ibm8)$/' + when: never + # arm64 tests are newly introduced to the merge queue and master. Keep them + # non-blocking (allow_failure) for now so we can collect stability stats and + # fix flaky/failing jobs without blocking the whole team. Remove allow_failure + # once the arm64 suite is proven stable. + - if: '$CI_COMMIT_BRANCH =~ /^mq-working-branch-/' + when: on_success + allow_failure: true + - if: '$CI_COMMIT_BRANCH == "master"' + when: on_success + allow_failure: true + # Enable non-default JVMs on demand. + - if: '$NON_DEFAULT_JVMS == "true"' + when: on_success + allow_failure: true + - if: '$CI_COMMIT_MESSAGE =~ /\[ci: NON_DEFAULT_JVMS\]/' + when: on_success + allow_failure: true + # Keep the default JVM subset available for manual runs on feature branches/PRs. + - if: '$testJvm =~ $DEFAULT_TEST_JVMS' + when: manual + allow_failure: true + cache: + - key: dependency-$CACHE_TYPE + paths: + - .gradle/wrapper + - .gradle/caches + - .gradle/notifications + - .mvn/caches + policy: pull + fallback_keys: + - dependency-base + - dependency-lib + unprotect: true + before_script: + - git config --global --add safe.directory "$CI_PROJECT_DIR" + - !reference [.gradle_build, before_script] + +.test_job_with_test_agent_common: + variables: + CI_USE_TEST_AGENT: "true" + CI_AGENT_HOST: local-agent + services: + - name: registry.ddbuild.io/images/mirror/dd-apm-test-agent/ddapm-test-agent:v1.64.1 + alias: local-agent + variables: + LOG_LEVEL: "DEBUG" + TRACE_LANGUAGE: "java" + DD_SUPPRESS_TRACE_PARSE_ERRORS: "true" + DD_POOL_TRACE_CHECK_FAILURES: "true" + DD_DISABLE_ERROR_RESPONSES: "true" + ENABLED_CHECKS: "trace_content_length,trace_stall,meta_tracer_version_header,trace_count_header,trace_peer_service,trace_dd_service" + script: + - !reference [.test_job_common, script] + - .gitlab/check_test_agent_results.sh + +.test_job_amd64_with_test_agent: + extends: + - .test_job_amd64 + - .test_job_with_test_agent_common + +.test_job_arm64_with_test_agent: + extends: + - .test_job_arm64 + - .test_job_with_test_agent_common + +agent_integration_tests: + extends: .test_job_amd64 + tags: [ "arch:amd64" ] + variables: + testJvm: "8" + CI_AGENT_HOST: local-agent + GRADLE_TARGET: "traceAgentTest" + CACHE_TYPE: "base" + services: + - name: registry.ddbuild.io/images/mirror/datadog/agent:7.40.1 + alias: local-agent + variables: + DD_APM_ENABLED: "true" + DD_BIND_HOST: "0.0.0.0" + DD_HOSTNAME: "local-agent" + DD_API_KEY: "invalid_key_but_this_is_fine" + +test_base: + extends: .test_job_amd64 + # needs:parallel:matrix limits this job to a specific build_tests combination. + # Keep matrix vars exact and in build_tests declaration order: + # https://docs.gitlab.com/ci/yaml/#needsparallelmatrix + needs: + - job: build_tests + parallel: + matrix: + - GRADLE_TARGET: ":baseTest" + CACHE_TYPE: "base" + variables: + GRADLE_TARGET: ":baseTest" + CACHE_TYPE: "base" + parallel: + matrix: *test_matrix_4 + script: + - if [ "$testJvm" == "8" ]; then export GRADLE_PARAMS="-PskipFlakyTests -PcheckCoverage"; fi + - !reference [.test_job_common, script] + +test_base_arm64: + extends: .test_job_arm64 + variables: + GRADLE_TARGET: ":baseTest" + CACHE_TYPE: "base" + parallel: + matrix: *test_matrix_4 + +test_inst: + extends: .test_job_amd64_with_test_agent + needs: *needs_build_tests_inst + variables: + <<: *tier_l_variables + GRADLE_TARGET: ":instrumentationTest" + CACHE_TYPE: "inst" + parallel: + matrix: *test_matrix_8 + +test_inst_arm64: + extends: .test_job_arm64_with_test_agent + variables: + <<: *tier_l_variables + GRADLE_TARGET: ":instrumentationTest" + CACHE_TYPE: "inst" + parallel: + matrix: *test_matrix_8 + +test_inst_latest: + extends: .test_job_amd64_with_test_agent + # needs:parallel:matrix limits this job to a specific build_tests combination. + # Keep matrix vars exact and in build_tests declaration order: + # https://docs.gitlab.com/ci/yaml/#needsparallelmatrix + needs: + - job: build_tests + parallel: + matrix: + - GRADLE_TARGET: ":instrumentationLatestDepTest" + CACHE_TYPE: "latestdep" + variables: + <<: *tier_l_variables + GRADLE_TARGET: ":instrumentationLatestDepTest" + CACHE_TYPE: "latestdep" + parallel: + matrix: + - testJvm: ["8", "17", "21", "25", "27", "tip"] # Java 27 TODO: remove 27 after GA (tip will move to 27) + # Gitlab doesn't support "parallel" and "parallel:matrix" at the same time + # This emulates "parallel" by including it in the matrix + CI_SPLIT: [ "1/6", "2/6", "3/6", "4/6", "5/6", "6/6"] + +test_inst_latest_arm64: + extends: .test_job_arm64_with_test_agent + variables: + <<: *tier_l_variables + GRADLE_TARGET: ":instrumentationLatestDepTest" + CACHE_TYPE: "latestdep" + parallel: + matrix: + - testJvm: ["17", "21", "25", "27", "tip"] # Java 27 TODO: remove 27 after GA (tip will move to 27) + # Gitlab doesn't support "parallel" and "parallel:matrix" at the same time + # This emulates "parallel" by including it in the matrix + CI_SPLIT: [ "1/6", "2/6", "3/6", "4/6", "5/6", "6/6"] + +test_flaky: + extends: .test_job_amd64_with_test_agent + variables: + GRADLE_PARAMS: "-PrunFlakyTests" + CACHE_TYPE: "smoke" + testJvm: "8" + CONTINUE_ON_FAILURE: "true" + rules: + - *master_only + - if: $RUN_FLAKY_TESTS == "true" + when: on_success + parallel: + matrix: + - GRADLE_TARGET: [":baseTest", ":smokeTest", ":debuggerTest"] + # Gitlab doesn't support "parallel" and "parallel:matrix" at the same time + # This emulates "parallel" by including it in the matrix + CI_SPLIT: [ "1/4", "2/4", "3/4", "4/4" ] + +test_flaky_inst: + extends: .test_job_amd64 + needs: *needs_build_tests_inst + variables: + GRADLE_TARGET: ":instrumentationTest" + GRADLE_PARAMS: "-PrunFlakyTests" + CACHE_TYPE: "inst" + testJvm: "8" + CONTINUE_ON_FAILURE: "true" + rules: + - *master_only + - if: $RUN_FLAKY_TESTS == "true" + when: on_success + parallel: 6 + +test_profiling: + extends: .test_job_amd64 + # needs:parallel:matrix limits this job to a specific build_tests combination. + # Keep matrix vars exact and in build_tests declaration order: + # https://docs.gitlab.com/ci/yaml/#needsparallelmatrix + needs: + - job: build_tests + parallel: + matrix: + - GRADLE_TARGET: ":profilingTest" + CACHE_TYPE: "profiling" + variables: + GRADLE_TARGET: ":profilingTest" + CACHE_TYPE: "profiling" + parallel: + matrix: *test_matrix + +test_profiling_arm64: + extends: .test_job_arm64 + variables: + GRADLE_TARGET: ":profilingTest" + CACHE_TYPE: "profiling" + parallel: + matrix: *test_matrix + +# specific jvms list for debugger project because J9-based JVMs have issues with local vars +# so need to test at least against one J9-based JVM +test_debugger: + extends: .test_job_amd64 + variables: + GRADLE_TARGET: ":debuggerTest" + CACHE_TYPE: "base" + DEFAULT_TEST_JVMS: /^(8|11|17|21|25|27|tip|semeru8)$/ # Java 27 TODO: remove 27 after GA (tip will move to 27) + parallel: + matrix: *test_matrix + # avoid running coverage for semeru8, semeru11, semeru17 and ibm8 as some tests are disabled and therefore cannot reach the + # exepected coverage + script: + - if [[ "$testJvm" != "semeru8" && "$testJvm" != "semeru11" && "$testJvm" != "semeru17" && "$testJvm" != "ibm8" ]]; then export GRADLE_PARAMS="$GRADLE_PARAMS -PcheckCoverage"; fi + - !reference [.test_job_common, script] + +test_debugger_arm64: + extends: .test_job_arm64 + variables: + GRADLE_TARGET: ":debuggerTest" + CACHE_TYPE: "base" + parallel: + matrix: *test_matrix + # avoid running coverage for semeru11 and semeru17 as some tests are disabled and therefore cannot reach the + # expected coverage + script: + - if [[ "$testJvm" != "semeru11" && "$testJvm" != "semeru17" ]]; then export GRADLE_PARAMS="$GRADLE_PARAMS -PcheckCoverage"; fi + - !reference [.test_job_common, script] + +test_smoke: + extends: .test_job_amd64_with_test_agent + # needs:parallel:matrix limits this job to a specific build_tests combination. + # Keep matrix vars exact and in build_tests declaration order: + # https://docs.gitlab.com/ci/yaml/#needsparallelmatrix + needs: &needs_build_tests_smoke + - job: build_tests + parallel: + matrix: + - GRADLE_TARGET: ":smokeTest" + CACHE_TYPE: "smoke" + MAVEN_OPTS: "-Xms256M -Xmx1024M" + variables: + <<: *tier_l_variables + GRADLE_TARGET: "stageMainDist :smokeTest" + GRADLE_PARAMS: "-PskipFlakyTests" + CACHE_TYPE: "smoke" + parallel: + matrix: *test_matrix_8 + +test_smoke_arm64: + extends: .test_job_arm64_with_test_agent + variables: + <<: *tier_l_variables + GRADLE_TARGET: "stageMainDist :smokeTest" + GRADLE_PARAMS: "-PskipFlakyTests" + CACHE_TYPE: "smoke" + parallel: + matrix: *test_matrix_8 + +test_ssi_smoke: + extends: .test_job_amd64 + needs: *needs_build_tests_smoke + rules: + - if: $CI_COMMIT_BRANCH == "master" + when: on_success + - if: '$CI_COMMIT_BRANCH =~ /^mq-working-branch-/' + when: on_success + - if: '$CI_COMMIT_BRANCH =~ /^gh-readonly-queue/' + when: on_success + variables: + <<: *tier_l_variables + GRADLE_TARGET: "stageMainDist :smokeTest" + CACHE_TYPE: "smoke" + DD_INJECT_FORCE: "true" + DD_INJECTION_ENABLED: "tracer" + parallel: + matrix: *test_matrix_8 + +test_ssi_smoke_arm64: + extends: .test_job_arm64 + variables: + <<: *tier_l_variables + GRADLE_TARGET: "stageMainDist :smokeTest" + CACHE_TYPE: "smoke" + DD_INJECT_FORCE: "true" + DD_INJECTION_ENABLED: "tracer" + parallel: + matrix: *test_matrix_8 + +test_smoke_graalvm: + extends: .test_job_amd64 + needs: *needs_build_tests_smoke + tags: [ "arch:amd64" ] + variables: + <<: *tier_l_variables + GRADLE_TARGET: "stageMainDist :dd-smoke-test:spring-boot-3.0-native:test :dd-smoke-test:quarkus-native:test" + CACHE_TYPE: "smoke" + CI_NO_SPLIT: "true" + NON_DEFAULT_JVMS: "true" + parallel: + matrix: + - testJvm: ["graalvm17", "graalvm21", "graalvm25"] + +test_smoke_graalvm_arm64: + extends: .test_job_arm64 + tags: [ "arch:arm64" ] + variables: + <<: *tier_l_variables + GRADLE_TARGET: "stageMainDist :dd-smoke-test:spring-boot-3.0-native:test :dd-smoke-test:quarkus-native:test" + CACHE_TYPE: "smoke" + CI_NO_SPLIT: "true" + NON_DEFAULT_JVMS: "true" + parallel: + matrix: + - testJvm: ["graalvm17", "graalvm21", "graalvm25"] + +test_smoke_semeru8_debugger: + extends: .test_job_amd64 + needs: *needs_build_tests_smoke + tags: [ "arch:amd64" ] + variables: + GRADLE_TARGET: "stageMainDist dd-smoke-tests:debugger-integration-tests:test" + CACHE_TYPE: "smoke" + NON_DEFAULT_JVMS: "true" + testJvm: "semeru8" + +aggregate_test_counts: + image: ${BUILDER_IMAGE_REPO}:${BUILDER_IMAGE_VERSION_PREFIX}base + stage: test-summary + # Keep stage ordering, but prevent GitLab from downloading all previous-stage artifacts. + dependencies: [] + rules: + - if: '$POPULATE_CACHE' + when: never + - if: '$CI_COMMIT_BRANCH == "master"' + when: on_success + - if: '$CI_COMMIT_BRANCH =~ /^mq-working-branch-/' + when: on_success + - if: '$CI_COMMIT_BRANCH =~ /^gh-readonly-queue/' + when: on_success + - if: '$CI_COMMIT_BRANCH' + when: on_success + script: + - *set_datadog_api_keys + - export TEST_COUNTS_S3_PREFIX="test-counts/${CI_PIPELINE_ID}" + - mkdir -p ./test_counts_aggregate + - echo "Downloading test count files from s3://${TEST_COUNTS_S3_BUCKET}/${TEST_COUNTS_S3_PREFIX}/" + - aws s3 cp "s3://${TEST_COUNTS_S3_BUCKET}/${TEST_COUNTS_S3_PREFIX}/" ./test_counts_aggregate/ --recursive --exclude "*" --include "test_counts_*.json" --only-show-errors + - find ./test_counts_aggregate -name 'test_counts_*.json' -type f -print | sort + - .gitlab/aggregate_test_counts.sh ./test_counts_aggregate + artifacts: + when: always + paths: + - test_counts_aggregate/test_counts_*.json + - test_counts_summary.json + - test_counts_report.md + +deploy_to_profiling_backend: + stage: publish + needs: [ build ] + rules: + - if: '$POPULATE_CACHE' + when: never + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' + when: on_success + - if: '$CI_COMMIT_TAG =~ /^v.*/' + when: on_success + - when: manual + allow_failure: true + trigger: + project: DataDog/profiling-backend + branch: dogfooding + variables: + UPSTREAM_PACKAGE_JOB: $BUILD_JOB_NAME + UPSTREAM_PACKAGE_JOB_ID: $BUILD_JOB_ID + UPSTREAM_PROJECT_ID: $CI_PROJECT_ID + UPSTREAM_PROJECT_NAME: $CI_PROJECT_NAME + UPSTREAM_PIPELINE_ID: $CI_PIPELINE_ID + UPSTREAM_BRANCH: $CI_COMMIT_BRANCH + UPSTREAM_TAG: $CI_COMMIT_TAG + +deploy_to_di_backend:manual: + stage: publish + needs: [ build ] + rules: + - if: '$POPULATE_CACHE' + when: never + - when: manual + allow_failure: true + trigger: + project: DataDog/debugger-demos + branch: main + variables: + UPSTREAM_PACKAGE_JOB: build + UPSTREAM_PROJECT_ID: $CI_PROJECT_ID + UPSTREAM_PROJECT_NAME: $CI_PROJECT_NAME + UPSTREAM_PIPELINE_ID: $CI_PIPELINE_ID + UPSTREAM_BRANCH: $CI_COMMIT_BRANCH + UPSTREAM_TAG: $CI_COMMIT_TAG + UPSTREAM_COMMIT_AUTHOR: $CI_COMMIT_AUTHOR + UPSTREAM_COMMIT_SHORT_SHA: $CI_COMMIT_SHORT_SHA + +deploy_to_reliability_env: + needs: [ build ] + +# If the deploy_to_maven_central job is re-run, re-trigger the deploy_artifacts_to_github job as well so that the artifacts match. +deploy_to_maven_central: + extends: .gradle_build + stage: publish-release-artifacts + needs: + - job: build + - job: system_tests + optional: true + variables: + CACHE_TYPE: "lib" + rules: + - if: '$POPULATE_CACHE' + when: never + - if: '$CI_COMMIT_BRANCH == "master"' + when: on_success + # Do not deploy release candidate versions + - if: '$CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+$/' + when: on_success + allow_failure: true + - when: manual + allow_failure: true + script: + - export MAVEN_CENTRAL_USERNAME=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.central_username --with-decryption --query "Parameter.Value" --out text) + - export MAVEN_CENTRAL_PASSWORD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.central_password --with-decryption --query "Parameter.Value" --out text) + - export GPG_PRIVATE_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.signing.gpg_private_key --with-decryption --query "Parameter.Value" --out text) + - export GPG_PASSWORD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.signing.gpg_passphrase --with-decryption --query "Parameter.Value" --out text) + - ./gradlew publishToSonatype closeSonatypeStagingRepository -PskipTests $GRADLE_ARGS + artifacts: + paths: + - 'workspace/dd-java-agent/build/libs/*.jar' + - 'workspace/dd-trace-api/build/libs/*.jar' + - 'workspace/dd-trace-ot/build/libs/*.jar' + +deploy_snapshot_with_ddprof_snapshot: + extends: .gradle_build + stage: publish + needs: [ build ] + variables: + CACHE_TYPE: "lib" + rules: + - if: '$POPULATE_CACHE' + when: never + # Manual trigger only - for testing with ddprof snapshot versions + - when: manual + allow_failure: true + script: + - export MAVEN_CENTRAL_USERNAME=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.central_username --with-decryption --query "Parameter.Value" --out text) + - export MAVEN_CENTRAL_PASSWORD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.central_password --with-decryption --query "Parameter.Value" --out text) + - export GPG_PRIVATE_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.signing.gpg_private_key --with-decryption --query "Parameter.Value" --out text) + - export GPG_PASSWORD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.signing.gpg_passphrase --with-decryption --query "Parameter.Value" --out text) + - echo "Publishing dd-trace-java snapshot with ddprof snapshot dependency" + - ./gradlew -PbuildInfo.build.number=$CI_JOB_ID -PddprofUseSnapshot publishToSonatype -PskipTests $GRADLE_ARGS + artifacts: + paths: + - 'workspace/dd-java-agent/build/libs/*.jar' + - 'workspace/dd-trace-api/build/libs/*.jar' + - 'workspace/dd-trace-ot/build/libs/*.jar' + +deploy_artifacts_to_github: + stage: publish-release-artifacts + image: registry.ddbuild.io/images/dd-octo-sts-ci-base:2025.06-1 + tags: [ "arch:amd64" ] + id_tokens: + DDOCTOSTS_ID_TOKEN: + aud: dd-octo-sts + rules: + - if: '$POPULATE_CACHE' + when: never + - if: '$CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+$/' + when: on_success + # Requires the deploy_to_maven_central job to have run first (the UP-TO-DATE gradle check across jobs is broken) + # This will deploy the artifacts built from the publishToSonatype task to the GitHub release + needs: + - job: deploy_to_maven_central + # The deploy_to_maven_central job is not run for release candidate versions + optional: true + before_script: + - dd-octo-sts version + - dd-octo-sts debug --scope DataDog/dd-trace-java --policy self.gitlab.release + - dd-octo-sts token --scope DataDog/dd-trace-java --policy self.gitlab.release > github-token.txt + script: + - gh auth login --with-token < github-token.txt + - gh auth status + - export VERSION=${CI_COMMIT_TAG##v} # remove "v" from front of tag to get version + - cp workspace/dd-java-agent/build/libs/dd-java-agent-${VERSION}.jar workspace/dd-java-agent/build/libs/dd-java-agent.jar # upload two filenames + - gh release upload --clobber --repo DataDog/dd-trace-java $CI_COMMIT_TAG workspace/dd-java-agent/build/libs/dd-java-agent.jar + - gh release upload --clobber --repo DataDog/dd-trace-java $CI_COMMIT_TAG workspace/dd-java-agent/build/libs/dd-java-agent-${VERSION}.jar + - gh release upload --clobber --repo DataDog/dd-trace-java $CI_COMMIT_TAG workspace/dd-trace-api/build/libs/dd-trace-api-${VERSION}.jar + - gh release upload --clobber --repo DataDog/dd-trace-java $CI_COMMIT_TAG workspace/dd-trace-ot/build/libs/dd-trace-ot-${VERSION}.jar + after_script: + - dd-octo-sts revoke -t $(cat github-token.txt) + retry: + max: 2 + when: always + +requirements_json_test: + rules: + - when: on_success + variables: + REQUIREMENTS_BLOCK_JSON_PATH: "metadata/requirements-block.json" + REQUIREMENTS_ALLOW_JSON_PATH: "metadata/requirements-allow.json" + +package-oci: + needs: [ build ] + +override_verify_maven_central: + image: registry.ddbuild.io/images/base/gbi-ubuntu_2204:release + stage: publish + needs: [ ] + rules: + - if: '$POPULATE_CACHE' + when: never + - when: manual + allow_failure: true + script: + - touch OVERRIDE_MAVEN_VERIFY + cache: # Cache is used to signal between the override_verify_maven_central and verify_maven_central_deployment jobs + - key: $CI_PIPELINE_ID-OVERRIDE_SIGNAL + paths: + - OVERRIDE_MAVEN_VERIFY + policy: push + unprotect: true + +# Verify Maven Central deployment is publicly available before publishing OCI images +verify_maven_central_deployment: + image: registry.ddbuild.io/images/base/gbi-ubuntu_2204:release + stage: publish-release-artifacts + needs: [ deploy_to_maven_central ] + rules: + - if: '$POPULATE_CACHE' + when: never + - if: '$CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+$/' + when: on_success + cache: # Cache is used to signal between the override_verify_maven_central and verify_maven_central_deployment jobs + - key: $CI_PIPELINE_ID-OVERRIDE_SIGNAL + paths: + - OVERRIDE_MAVEN_VERIFY + policy: pull + unprotect: true + script: + - if [ -f OVERRIDE_MAVEN_VERIFY ]; then echo "SKIPPING MAVEN VERIFICATION"; exit 0; fi + - | + export VERSION=${CI_COMMIT_TAG##v} + ARTIFACT_URLS=( + "https://repo1.maven.org/maven2/com/datadoghq/dd-java-agent/${VERSION}/dd-java-agent-${VERSION}.jar" + "https://repo1.maven.org/maven2/com/datadoghq/dd-trace-api/${VERSION}/dd-trace-api-${VERSION}.jar" + "https://repo1.maven.org/maven2/com/datadoghq/dd-trace-ot/${VERSION}/dd-trace-ot-${VERSION}.jar" + ) + # Try once immediately (fast path on job retry), then every 5 mins for 45 mins. + TRY=0 + MAX_TRIES=10 + RETRY_DELAY=300 + while [ $TRY -lt $MAX_TRIES ]; do + ARTIFACTS_AVAILABLE=true + for URL in "${ARTIFACT_URLS[@]}"; do + if ! curl --location --fail --silent --show-error -I "$URL"; then + ARTIFACTS_AVAILABLE=false + break + fi + done + if [ "$ARTIFACTS_AVAILABLE" = true ]; then + break + fi + TRY=$((TRY + 1)) + if [ $TRY -eq $MAX_TRIES ]; then + echo "The release was not available after 45 mins. Manually re-run the job to try again." + exit 1 + fi + sleep $RETRY_DELAY + done + +publishing-gate: + stage: publish-release-artifacts + needs: + - job: verify_maven_central_deployment + optional: true # Required for releases only + +configure_system_tests: + variables: + SYSTEM_TESTS_REF: "main" # system tests are pinned on release branches only + SYSTEM_TESTS_SCENARIOS_GROUPS: "simple_onboarding,simple_onboarding_profiling,simple_onboarding_appsec,docker-ssi,lib-injection" + +create_key: + stage: generate-signing-key + when: manual + needs: [ ] + variables: + PROJECT_NAME: "dd-trace-java" + EXPORT_TO_KEYSERVER: "true" + image: $REGISTRY/ci/agent-key-management-tools/gpg:1 + script: + - /create.sh + artifacts: + expire_in: 13 mos + paths: + - pubkeys + +validate_supported_configurations_v2_local_file: + extends: .validate_supported_configurations_v2_local_file + variables: + LOCAL_JSON_PATH: "metadata/supported-configurations.json" + BACKFILLED: "false" + +update_central_configurations_version_range_v2: + extends: .update_central_configurations_version_range_v2 + variables: + LOCAL_REPO_NAME: "dd-trace-java" + LOCAL_JSON_PATH: "metadata/supported-configurations.json" + LANGUAGE_NAME: "java" + MULTIPLE_RELEASE_LINES: "false" + SEND_ALIASES: "true"