From 292349ffb6d93e0d8283dfde942096f59799023b Mon Sep 17 00:00:00 2001 From: Sarah Chen Date: Thu, 29 Oct 2026 14:21:17 -0400 Subject: [PATCH 1/6] Configure shared and release S3 build cache namespaces --- .gitlab-ci.yml | 13 ++++++++++++- docs/how_to_work_with_gradle.md | 9 +++++++++ settings.gradle.kts | 11 ++++++----- 3 files changed, 27 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 361e7d978aa..4ac1bbaca3e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -240,6 +240,16 @@ default: before_script: - source .gitlab/gitlab-utils.sh - *container_info + - | + if [ -z "${S3_BUILD_CACHE_PREFIX+x}" ]; then + s3_cache_arch=$(uname -m) + s3_cache_os=$(uname -s | tr '[:upper:]' '[:lower:]') + if [[ "${CI_COMMIT_TAG:-}" =~ [^[:space:]] ]]; then + export S3_BUILD_CACHE_PREFIX="v1/release/${CI_PIPELINE_ID:?}/$s3_cache_arch/$s3_cache_os/" + else + export S3_BUILD_CACHE_PREFIX="v1/$s3_cache_arch/$s3_cache_os/" + fi + fi # Akka token added to SSM from https://account.akka.io/token - export ORG_GRADLE_PROJECT_akkaRepositoryToken=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.akka_repo_token --with-decryption --query "Parameter.Value" --out text) - export ORG_GRADLE_PROJECT_mavenRepositoryProxy=$MAVEN_REPOSITORY_PROXY @@ -575,6 +585,8 @@ validate_build: variables: CACHE_TYPE: "lib" script: + # Rebuild independently of the cache used to produce the reference artifacts. + - export GRADLE_ARGS="$GRADLE_ARGS --no-build-cache --rerun-tasks" # Preserve the `build` job artifacts before Gradle rebuilds and overwrites them under # workspace/**/build/libs, so jardiff can compare the rebuilt jars against them. - mkdir -p reference-artifacts @@ -586,7 +598,6 @@ validate_build: # Refresh them here as well so the candidate uses the same dependency resolution. - if [ $CI_PIPELINE_SOURCE == "schedule" ] ; then ./gradlew resolveAndLockAll --write-locks $GRADLE_ARGS; fi - ./gradlew --version - # This will run the shadowJar task and exercise the build cache, allowing to identify build-cache issues # Keep both JAR sets for direct inspection when the comparison fails. - | if ! ./gradlew compareToReferenceJar -PjardiffReferenceDir="$CI_PROJECT_DIR/reference-artifacts" -PskipTests $GRADLE_ARGS; then diff --git a/docs/how_to_work_with_gradle.md b/docs/how_to_work_with_gradle.md index 6d772d4db04..698854edac7 100644 --- a/docs/how_to_work_with_gradle.md +++ b/docs/how_to_work_with_gradle.md @@ -1799,6 +1799,15 @@ When Gradle builds fail or behave unexpectedly, several tools and techniques can > [!TIP] > In general following the [Gradle best practices](https://docs.gradle.org/current/userguide/best_practices.html) is safe bet to avoid isses. +### S3 Build Cache in GitLab CI + +`S3_BUILD_CACHE_PREFIX` overrides the full prefix; Gradle appends the task cache key. +Defaults are `v1///` for non-tag builds (including master) and +`v1/release////` for tags. Include the pipeline ID +in release overrides to preserve isolation. + +`validate_build` compares build artifacts against an uncached rebuild. + ### Forcing Fresh CI Task Execution Set the GitLab CI variable `SKIP_BUILD_CACHE=true` when starting a pipeline. diff --git a/settings.gradle.kts b/settings.gradle.kts index 9f6c33f828c..f73bad84f8a 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -40,8 +40,6 @@ plugins { val isCI = providers.environmentVariable("CI") val isGitLabCI = providers.environmentVariable("GITLAB_CI") -val isGitLabTag = providers.environmentVariable("CI_COMMIT_TAG") -val gitLabRefSlug = providers.environmentVariable("CI_COMMIT_REF_SLUG") val skipBuildscan = providers.environmentVariable("SKIP_BUILDSCAN").map { it.toBoolean() }.orElse(false) develocity { @@ -58,12 +56,15 @@ if (isCI.isPresent) { local { directory = File(rootDir, "workspace/build-cache") } - if (isGitLabCI.isPresent && !isGitLabTag.isPresent) { + if (isGitLabCI.isPresent) { + val cachePrefix = providers.environmentVariable("S3_BUILD_CACHE_PREFIX").orNull?.trim()?.trim('/') + require(!cachePrefix.isNullOrBlank()) { + "S3_BUILD_CACHE_PREFIX must be set to a non-empty path in GitLab CI" + } remote { region = providers.environmentVariable("S3_BUILD_CACHE_REGION").get() bucket = providers.environmentVariable("S3_BUILD_CACHE_BUCKET").get() - prefix = - "trial/v1/${gitLabRefSlug.get()}/${System.getProperty("os.arch")}/" + prefix = "$cachePrefix/" isPush = true lookupDefaultAwsCredentials = true isReducedRedundancy = false From 2feecba92f800a5cf88133a70d697f1912787e07 Mon Sep 17 00:00:00 2001 From: Sarah Chen Date: Thu, 29 Oct 2026 18:19:57 -0400 Subject: [PATCH 2/6] Prevent stale version stamps in cached agent JARs --- dd-java-agent/build.gradle | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/dd-java-agent/build.gradle b/dd-java-agent/build.gradle index 3fe773b73a4..2b9ef700227 100644 --- a/dd-java-agent/build.gradle +++ b/dd-java-agent/build.gradle @@ -309,6 +309,15 @@ def traceShadowJar = tasks.register('traceShadowJar', ShadowJar) { } includeShadowJar(traceShadowJar, 'trace', includedJarFileTree) +[sharedShadowJar, traceShadowJar].each { shadowJarTask -> + shadowJarTask.configure { + // Track packaged bytes, including version files ignored by classpath normalization. + inputs.files(includedDependencies) + .withPropertyName('packagedDependencies') + .withPathSensitivity(PathSensitivity.NONE) + } +} + tasks.named("shadowJar", ShadowJar) { // Include AgentPreCheck compiled with Java 6. from sourceSets.main_java6.output From ebb43c0b6c488ec556d10ce80a341707e575e29e Mon Sep 17 00:00:00 2001 From: Sarah Chen Date: Fri, 30 Oct 2026 13:36:31 -0400 Subject: [PATCH 3/6] Run CI with warm cache From ad94906ce6608f8a0fede96060fa0c63a9881de8 Mon Sep 17 00:00:00 2001 From: Sarah Chen Date: Mon, 2 Nov 2026 12:49:58 -0500 Subject: [PATCH 4/6] Address review comments --- .gitlab-ci.yml | 10 ---------- dd-java-agent/build.gradle | 4 ++-- docs/how_to_work_with_gradle.md | 17 +++++++++++------ settings.gradle.kts | 21 +++++++++++++++++++-- 4 files changed, 32 insertions(+), 20 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4ac1bbaca3e..c324f5198b3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -240,16 +240,6 @@ default: before_script: - source .gitlab/gitlab-utils.sh - *container_info - - | - if [ -z "${S3_BUILD_CACHE_PREFIX+x}" ]; then - s3_cache_arch=$(uname -m) - s3_cache_os=$(uname -s | tr '[:upper:]' '[:lower:]') - if [[ "${CI_COMMIT_TAG:-}" =~ [^[:space:]] ]]; then - export S3_BUILD_CACHE_PREFIX="v1/release/${CI_PIPELINE_ID:?}/$s3_cache_arch/$s3_cache_os/" - else - export S3_BUILD_CACHE_PREFIX="v1/$s3_cache_arch/$s3_cache_os/" - fi - fi # Akka token added to SSM from https://account.akka.io/token - export ORG_GRADLE_PROJECT_akkaRepositoryToken=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.akka_repo_token --with-decryption --query "Parameter.Value" --out text) - export ORG_GRADLE_PROJECT_mavenRepositoryProxy=$MAVEN_REPOSITORY_PROXY diff --git a/dd-java-agent/build.gradle b/dd-java-agent/build.gradle index 2b9ef700227..24ece636e9b 100644 --- a/dd-java-agent/build.gradle +++ b/dd-java-agent/build.gradle @@ -309,8 +309,8 @@ def traceShadowJar = tasks.register('traceShadowJar', ShadowJar) { } includeShadowJar(traceShadowJar, 'trace', includedJarFileTree) -[sharedShadowJar, traceShadowJar].each { shadowJarTask -> - shadowJarTask.configure { +['sharedShadowJar', 'traceShadowJar'].each { taskName -> + tasks.named(taskName, ShadowJar).configure { // Track packaged bytes, including version files ignored by classpath normalization. inputs.files(includedDependencies) .withPropertyName('packagedDependencies') diff --git a/docs/how_to_work_with_gradle.md b/docs/how_to_work_with_gradle.md index 698854edac7..13ef6921636 100644 --- a/docs/how_to_work_with_gradle.md +++ b/docs/how_to_work_with_gradle.md @@ -1799,14 +1799,19 @@ When Gradle builds fail or behave unexpectedly, several tools and techniques can > [!TIP] > In general following the [Gradle best practices](https://docs.gradle.org/current/userguide/best_practices.html) is safe bet to avoid isses. -### S3 Build Cache in GitLab CI +### S3 Build Cache -`S3_BUILD_CACHE_PREFIX` overrides the full prefix; Gradle appends the task cache key. -Defaults are `v1///` for non-tag builds (including master) and -`v1/release////` for tags. Include the pipeline ID -in release overrides to preserve isolation. +Gradle's build cache reuses task outputs when their inputs match. S3 shares those outputs +across CI jobs and pipelines. Cacheable tasks must declare all inputs that affect their outputs. +Otherwise, Gradle may reuse stale outputs when an untracked input changes. -`validate_build` compares build artifacts against an uncached rebuild. +| Pipeline | Default prefix | +| --- | --- | +| Non-tag (including PR, nightly, and master) | `v1///` | +| Release tag | `v1/release////` | + +`S3_BUILD_CACHE_PREFIX` overrides the entire prefix; Gradle appends the task cache key. +Include the pipeline ID in release overrides to preserve isolation. ### Forcing Fresh CI Task Execution diff --git a/settings.gradle.kts b/settings.gradle.kts index f73bad84f8a..6b79e09ae64 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -57,8 +57,25 @@ if (isCI.isPresent) { directory = File(rootDir, "workspace/build-cache") } if (isGitLabCI.isPresent) { - val cachePrefix = providers.environmentVariable("S3_BUILD_CACHE_PREFIX").orNull?.trim()?.trim('/') - require(!cachePrefix.isNullOrBlank()) { + val cachePrefix = providers.environmentVariable("S3_BUILD_CACHE_PREFIX").orNull?.trim()?.trim('/') ?: run { + val osName = providers.systemProperty("os.name").get() + val os = if (osName == "Mac OS X") "darwin" else osName.lowercase() + val arch = when (val archName = providers.systemProperty("os.arch").get()) { + "amd64" -> "x86_64" + "aarch64", "arm64" -> if (os == "darwin") "arm64" else "aarch64" + else -> archName + } + if (providers.environmentVariable("CI_COMMIT_TAG").orNull.isNullOrBlank()) { + "v1/$arch/$os" + } else { + val pipelineId = providers.environmentVariable("CI_PIPELINE_ID").orNull + require(!pipelineId.isNullOrBlank()) { + "CI_PIPELINE_ID must be set for release cache isolation" + } + "v1/release/$pipelineId/$arch/$os" + } + } + require(cachePrefix.isNotBlank()) { "S3_BUILD_CACHE_PREFIX must be set to a non-empty path in GitLab CI" } remote { From d74e07e2953e0cbc1b1f3003d71534a3292786ed Mon Sep 17 00:00:00 2001 From: Sarah Chen Date: Mon, 2 Nov 2026 16:23:28 -0500 Subject: [PATCH 5/6] Add build cache matrix for validate_build job --- .gitlab-ci.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c324f5198b3..b41a766edde 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -574,9 +574,12 @@ validate_build: needs: [ build ] variables: CACHE_TYPE: "lib" + parallel: + matrix: + - BUILD_CACHE: ["true", "false"] script: # Rebuild independently of the cache used to produce the reference artifacts. - - export GRADLE_ARGS="$GRADLE_ARGS --no-build-cache --rerun-tasks" + - if [ "$BUILD_CACHE" = "false" ]; then export GRADLE_ARGS="$GRADLE_ARGS --no-build-cache --rerun-tasks"; fi # Preserve the `build` job artifacts before Gradle rebuilds and overwrites them under # workspace/**/build/libs, so jardiff can compare the rebuilt jars against them. - mkdir -p reference-artifacts @@ -590,7 +593,7 @@ validate_build: - ./gradlew --version # Keep both JAR sets for direct inspection when the comparison fails. - | - if ! ./gradlew compareToReferenceJar -PjardiffReferenceDir="$CI_PROJECT_DIR/reference-artifacts" -PskipTests $GRADLE_ARGS; then + if ! ./gradlew clean compareToReferenceJar -PjardiffReferenceDir="$CI_PROJECT_DIR/reference-artifacts" -PskipTests $GRADLE_ARGS; then mkdir -p check_reports/jar-comparison-artifacts/reference check_reports/jar-comparison-artifacts/candidate cp reference-artifacts/*.jar check_reports/jar-comparison-artifacts/reference/ cp workspace/dd-java-agent/build/libs/*.jar check_reports/jar-comparison-artifacts/candidate/ From a23385200c18d6a3c98c8640fa57518cd87df06b Mon Sep 17 00:00:00 2001 From: Sarah Chen Date: Tue, 3 Nov 2026 09:47:01 -0500 Subject: [PATCH 6/6] Apply review suggestions --- settings.gradle.kts | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/settings.gradle.kts b/settings.gradle.kts index 6b79e09ae64..46daf1b651c 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -57,7 +57,7 @@ if (isCI.isPresent) { directory = File(rootDir, "workspace/build-cache") } if (isGitLabCI.isPresent) { - val cachePrefix = providers.environmentVariable("S3_BUILD_CACHE_PREFIX").orNull?.trim()?.trim('/') ?: run { + val defaultCachePrefix = providers.provider { val osName = providers.systemProperty("os.name").get() val os = if (osName == "Mac OS X") "darwin" else osName.lowercase() val arch = when (val archName = providers.systemProperty("os.arch").get()) { @@ -65,6 +65,12 @@ if (isCI.isPresent) { "aarch64", "arm64" -> if (os == "darwin") "arm64" else "aarch64" else -> archName } + // Non-tag pipelines share a cache per architecture/OS so that all jobs contribute + // outputs and reuse them across jobs and pipelines. Release prefixes can use: + // - Pipeline ID (default): jobs share within one pipeline; new pipelines start fresh. + // - Tag: pipelines for the same tag share cached outputs. + // - Commit hash: pipelines for the same commit share cached outputs. + // Pipeline IDs keep separate release runs independent, even for the same tag or commit. if (providers.environmentVariable("CI_COMMIT_TAG").orNull.isNullOrBlank()) { "v1/$arch/$os" } else { @@ -75,6 +81,10 @@ if (isCI.isPresent) { "v1/release/$pipelineId/$arch/$os" } } + val cachePrefix = providers.environmentVariable("S3_BUILD_CACHE_PREFIX") + .map { it.trim().trim('/') } + .orElse(defaultCachePrefix) + .get() require(cachePrefix.isNotBlank()) { "S3_BUILD_CACHE_PREFIX must be set to a non-empty path in GitLab CI" }