Configure shared and release S3 build cache namespaces

.gitlab-ci.yml

@@ -574,7 +574,12 @@ validate_build:
   needs: [ build ]
   variables:
     CACHE_TYPE: "lib"
+  parallel:
+    matrix:
+      - BUILD_CACHE: ["true", "false"]
   script:
+    # Rebuild independently of the cache used to produce the reference artifacts.
+    - if [ "$BUILD_CACHE" = "false" ]; then export GRADLE_ARGS="$GRADLE_ARGS --no-build-cache --rerun-tasks"; fi
     # Preserve the `build` job artifacts before Gradle rebuilds and overwrites them under
     # workspace/**/build/libs, so jardiff can compare the rebuilt jars against them.
     - mkdir -p reference-artifacts
@@ -586,10 +591,9 @@ validate_build:
     # Refresh them here as well so the candidate uses the same dependency resolution.
     - if [ $CI_PIPELINE_SOURCE == "schedule" ] ; then ./gradlew resolveAndLockAll --write-locks $GRADLE_ARGS; fi
     - ./gradlew --version
-    # This will run the shadowJar task and exercise the build cache, allowing to identify build-cache issues
     # Keep both JAR sets for direct inspection when the comparison fails.
     - |
-      if ! ./gradlew compareToReferenceJar -PjardiffReferenceDir="$CI_PROJECT_DIR/reference-artifacts" -PskipTests $GRADLE_ARGS; then
+      if ! ./gradlew clean compareToReferenceJar -PjardiffReferenceDir="$CI_PROJECT_DIR/reference-artifacts" -PskipTests $GRADLE_ARGS; then
         mkdir -p check_reports/jar-comparison-artifacts/reference check_reports/jar-comparison-artifacts/candidate
         cp reference-artifacts/*.jar check_reports/jar-comparison-artifacts/reference/
         cp workspace/dd-java-agent/build/libs/*.jar check_reports/jar-comparison-artifacts/candidate/

dd-java-agent/build.gradle

@@ -309,6 +309,15 @@ def traceShadowJar = tasks.register('traceShadowJar', ShadowJar) {
 }
 includeShadowJar(traceShadowJar, 'trace', includedJarFileTree)
 
+['sharedShadowJar', 'traceShadowJar'].each { taskName ->
+  tasks.named(taskName, ShadowJar).configure {
+    // Track packaged bytes, including version files ignored by classpath normalization.
+    inputs.files(includedDependencies)
+      .withPropertyName('packagedDependencies')
+      .withPathSensitivity(PathSensitivity.NONE)
+  }
+}
+
 tasks.named("shadowJar", ShadowJar) {
   // Include AgentPreCheck compiled with Java 6.
   from sourceSets.main_java6.output

docs/how_to_work_with_gradle.md

@@ -1799,6 +1799,20 @@ When Gradle builds fail or behave unexpectedly, several tools and techniques can
 > [!TIP]
 > In general following the [Gradle best practices](https://docs.gradle.org/current/userguide/best_practices.html) is safe bet to avoid isses.
 
+### S3 Build Cache
+
+Gradle's build cache reuses task outputs when their inputs match. S3 shares those outputs
+across CI jobs and pipelines. Cacheable tasks must declare all inputs that affect their outputs.
+Otherwise, Gradle may reuse stale outputs when an untracked input changes.
+
+| Pipeline | Default prefix |
+| --- | --- |
+| Non-tag (including PR, nightly, and master) | `v1/<architecture>/<os>/` |
+| Release tag | `v1/release/<CI_PIPELINE_ID>/<architecture>/<os>/` |
+
+`S3_BUILD_CACHE_PREFIX` overrides the entire prefix; Gradle appends the task cache key.
+Include the pipeline ID in release overrides to preserve isolation.
+
 ### Forcing Fresh CI Task Execution
 
 Set the GitLab CI variable `SKIP_BUILD_CACHE=true` when starting a pipeline.

settings.gradle.kts

@@ -40,8 +40,6 @@ plugins {
 
 val isCI = providers.environmentVariable("CI")
 val isGitLabCI = providers.environmentVariable("GITLAB_CI")
-val isGitLabTag = providers.environmentVariable("CI_COMMIT_TAG")
-val gitLabRefSlug = providers.environmentVariable("CI_COMMIT_REF_SLUG")
 val skipBuildscan = providers.environmentVariable("SKIP_BUILDSCAN").map { it.toBoolean() }.orElse(false)
 
 develocity {
@@ -58,12 +56,42 @@ if (isCI.isPresent) {
     local {
       directory = File(rootDir, "workspace/build-cache")
     }
-    if (isGitLabCI.isPresent && !isGitLabTag.isPresent) {
+    if (isGitLabCI.isPresent) {
+      val defaultCachePrefix = providers.provider {
+        val osName = providers.systemProperty("os.name").get()
+        val os = if (osName == "Mac OS X") "darwin" else osName.lowercase()
+        val arch = when (val archName = providers.systemProperty("os.arch").get()) {
+          "amd64" -> "x86_64"
+          "aarch64", "arm64" -> if (os == "darwin") "arm64" else "aarch64"
+          else -> archName
+        }
+        // Non-tag pipelines share a cache per architecture/OS so that all jobs contribute
+        // outputs and reuse them across jobs and pipelines. Release prefixes can use:
+        // - Pipeline ID (default): jobs share within one pipeline; new pipelines start fresh.
+        // - Tag: pipelines for the same tag share cached outputs.
+        // - Commit hash: pipelines for the same commit share cached outputs.
+        // Pipeline IDs keep separate release runs independent, even for the same tag or commit.
+        if (providers.environmentVariable("CI_COMMIT_TAG").orNull.isNullOrBlank()) {
+          "v1/$arch/$os"
+        } else {
+          val pipelineId = providers.environmentVariable("CI_PIPELINE_ID").orNull
+          require(!pipelineId.isNullOrBlank()) {
+            "CI_PIPELINE_ID must be set for release cache isolation"
+          }
+          "v1/release/$pipelineId/$arch/$os"
+        }
+      }
+      val cachePrefix = providers.environmentVariable("S3_BUILD_CACHE_PREFIX")
+        .map { it.trim().trim('/') }
+        .orElse(defaultCachePrefix)
+        .get()
+      require(cachePrefix.isNotBlank()) {
+        "S3_BUILD_CACHE_PREFIX must be set to a non-empty path in GitLab CI"
+      }
       remote<com.github.burrunan.s3cache.AwsS3BuildCache> {
         region = providers.environmentVariable("S3_BUILD_CACHE_REGION").get()
         bucket = providers.environmentVariable("S3_BUILD_CACHE_BUCKET").get()
-        prefix =
-          "trial/v1/${gitLabRefSlug.get()}/${System.getProperty("os.arch")}/"
+        prefix = "$cachePrefix/"
         isPush = true
         lookupDefaultAwsCredentials = true
         isReducedRedundancy = false