diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 361e7d978aa..b41a766edde 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -574,7 +574,12 @@ validate_build: needs: [ build ] variables: CACHE_TYPE: "lib" + parallel: + matrix: + - BUILD_CACHE: ["true", "false"] script: + # Rebuild independently of the cache used to produce the reference artifacts. + - if [ "$BUILD_CACHE" = "false" ]; then export GRADLE_ARGS="$GRADLE_ARGS --no-build-cache --rerun-tasks"; fi # Preserve the `build` job artifacts before Gradle rebuilds and overwrites them under # workspace/**/build/libs, so jardiff can compare the rebuilt jars against them. - mkdir -p reference-artifacts @@ -586,10 +591,9 @@ validate_build: # Refresh them here as well so the candidate uses the same dependency resolution. - if [ $CI_PIPELINE_SOURCE == "schedule" ] ; then ./gradlew resolveAndLockAll --write-locks $GRADLE_ARGS; fi - ./gradlew --version - # This will run the shadowJar task and exercise the build cache, allowing to identify build-cache issues # Keep both JAR sets for direct inspection when the comparison fails. - | - if ! ./gradlew compareToReferenceJar -PjardiffReferenceDir="$CI_PROJECT_DIR/reference-artifacts" -PskipTests $GRADLE_ARGS; then + if ! ./gradlew clean compareToReferenceJar -PjardiffReferenceDir="$CI_PROJECT_DIR/reference-artifacts" -PskipTests $GRADLE_ARGS; then mkdir -p check_reports/jar-comparison-artifacts/reference check_reports/jar-comparison-artifacts/candidate cp reference-artifacts/*.jar check_reports/jar-comparison-artifacts/reference/ cp workspace/dd-java-agent/build/libs/*.jar check_reports/jar-comparison-artifacts/candidate/ diff --git a/dd-java-agent/build.gradle b/dd-java-agent/build.gradle index 3fe773b73a4..24ece636e9b 100644 --- a/dd-java-agent/build.gradle +++ b/dd-java-agent/build.gradle @@ -309,6 +309,15 @@ def traceShadowJar = tasks.register('traceShadowJar', ShadowJar) { } includeShadowJar(traceShadowJar, 'trace', includedJarFileTree) +['sharedShadowJar', 'traceShadowJar'].each { taskName -> + tasks.named(taskName, ShadowJar).configure { + // Track packaged bytes, including version files ignored by classpath normalization. + inputs.files(includedDependencies) + .withPropertyName('packagedDependencies') + .withPathSensitivity(PathSensitivity.NONE) + } +} + tasks.named("shadowJar", ShadowJar) { // Include AgentPreCheck compiled with Java 6. from sourceSets.main_java6.output diff --git a/docs/how_to_work_with_gradle.md b/docs/how_to_work_with_gradle.md index 6d772d4db04..13ef6921636 100644 --- a/docs/how_to_work_with_gradle.md +++ b/docs/how_to_work_with_gradle.md @@ -1799,6 +1799,20 @@ When Gradle builds fail or behave unexpectedly, several tools and techniques can > [!TIP] > In general following the [Gradle best practices](https://docs.gradle.org/current/userguide/best_practices.html) is safe bet to avoid isses. +### S3 Build Cache + +Gradle's build cache reuses task outputs when their inputs match. S3 shares those outputs +across CI jobs and pipelines. Cacheable tasks must declare all inputs that affect their outputs. +Otherwise, Gradle may reuse stale outputs when an untracked input changes. + +| Pipeline | Default prefix | +| --- | --- | +| Non-tag (including PR, nightly, and master) | `v1///` | +| Release tag | `v1/release////` | + +`S3_BUILD_CACHE_PREFIX` overrides the entire prefix; Gradle appends the task cache key. +Include the pipeline ID in release overrides to preserve isolation. + ### Forcing Fresh CI Task Execution Set the GitLab CI variable `SKIP_BUILD_CACHE=true` when starting a pipeline. diff --git a/settings.gradle.kts b/settings.gradle.kts index 9f6c33f828c..46daf1b651c 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -40,8 +40,6 @@ plugins { val isCI = providers.environmentVariable("CI") val isGitLabCI = providers.environmentVariable("GITLAB_CI") -val isGitLabTag = providers.environmentVariable("CI_COMMIT_TAG") -val gitLabRefSlug = providers.environmentVariable("CI_COMMIT_REF_SLUG") val skipBuildscan = providers.environmentVariable("SKIP_BUILDSCAN").map { it.toBoolean() }.orElse(false) develocity { @@ -58,12 +56,42 @@ if (isCI.isPresent) { local { directory = File(rootDir, "workspace/build-cache") } - if (isGitLabCI.isPresent && !isGitLabTag.isPresent) { + if (isGitLabCI.isPresent) { + val defaultCachePrefix = providers.provider { + val osName = providers.systemProperty("os.name").get() + val os = if (osName == "Mac OS X") "darwin" else osName.lowercase() + val arch = when (val archName = providers.systemProperty("os.arch").get()) { + "amd64" -> "x86_64" + "aarch64", "arm64" -> if (os == "darwin") "arm64" else "aarch64" + else -> archName + } + // Non-tag pipelines share a cache per architecture/OS so that all jobs contribute + // outputs and reuse them across jobs and pipelines. Release prefixes can use: + // - Pipeline ID (default): jobs share within one pipeline; new pipelines start fresh. + // - Tag: pipelines for the same tag share cached outputs. + // - Commit hash: pipelines for the same commit share cached outputs. + // Pipeline IDs keep separate release runs independent, even for the same tag or commit. + if (providers.environmentVariable("CI_COMMIT_TAG").orNull.isNullOrBlank()) { + "v1/$arch/$os" + } else { + val pipelineId = providers.environmentVariable("CI_PIPELINE_ID").orNull + require(!pipelineId.isNullOrBlank()) { + "CI_PIPELINE_ID must be set for release cache isolation" + } + "v1/release/$pipelineId/$arch/$os" + } + } + val cachePrefix = providers.environmentVariable("S3_BUILD_CACHE_PREFIX") + .map { it.trim().trim('/') } + .orElse(defaultCachePrefix) + .get() + require(cachePrefix.isNotBlank()) { + "S3_BUILD_CACHE_PREFIX must be set to a non-empty path in GitLab CI" + } remote { region = providers.environmentVariable("S3_BUILD_CACHE_REGION").get() bucket = providers.environmentVariable("S3_BUILD_CACHE_BUCKET").get() - prefix = - "trial/v1/${gitLabRefSlug.get()}/${System.getProperty("os.arch")}/" + prefix = "$cachePrefix/" isPush = true lookupDefaultAwsCredentials = true isReducedRedundancy = false