From b227ce42fb04a366a08504a1723c1973efd1a2d1 Mon Sep 17 00:00:00 2001 From: Shunping Huang Date: Tue, 15 Sep 2026 11:51:55 -0400 Subject: [PATCH] Track ReplaceTokens values as processResources inputs Gradle treats the `tokens:` map passed to the ReplaceTokens content filter as a filter argument, not a task input, so it affects neither the up-to-date check nor the build cache key. The filtered sources contain only unsubstituted placeholders and are byte identical regardless of what the tokens expand to, so nothing invalidates processResources when a token value changes. In a shared checkout where build/ survives branch switches, the task stays UP-TO-DATE and the previous branch's substituted resource is left behind. Switching master -> a former release branch left dataflow.properties holding master's container version, so pipelines ran against a wrong version of beam-master SDK harness. The same staleness also poisons the build cache, since both branches compute the same key for different correct outputs, so `clean` is not a reliable workaround. This can be fixed by declaring the token map as an input property. --- runners/google-cloud-dataflow-java/arm/build.gradle | 5 ++++- runners/google-cloud-dataflow-java/build.gradle | 5 ++++- sdks/java/extensions/sql/jdbc/build.gradle | 5 ++++- sdks/java/maven-archetypes/examples/build.gradle | 5 ++++- sdks/java/maven-archetypes/gcp-bom-examples/build.gradle | 5 ++++- sdks/java/maven-archetypes/starter/build.gradle | 5 ++++- 6 files changed, 24 insertions(+), 6 deletions(-) diff --git a/runners/google-cloud-dataflow-java/arm/build.gradle b/runners/google-cloud-dataflow-java/arm/build.gradle index 531300bb63ea..b1e1ae59e9b3 100644 --- a/runners/google-cloud-dataflow-java/arm/build.gradle +++ b/runners/google-cloud-dataflow-java/arm/build.gradle @@ -52,11 +52,14 @@ evaluationDependsOn(":runners:google-cloud-dataflow-java:worker") evaluationDependsOn(":sdks:java:container:java11") processResources { - filter org.apache.tools.ant.filters.ReplaceTokens, tokens: [ + // Filter arguments are not tracked by Gradle, so declare the tokens as an input property. + def resourceTokens = [ 'dataflow.fnapi_environment_major_version' : project(':runners:google-cloud-dataflow-java').ext.dataflowFnapiEnvironmentMajorVersion, 'dataflow.fnapi_container_version' : project(':runners:google-cloud-dataflow-java').ext.dataflowFnapiContainerVersion, 'dataflow.container_base_repository' : project(':runners:google-cloud-dataflow-java').ext.dataflowContainerBaseRepository, ] + inputs.property('resourceTokens', resourceTokens) + filter org.apache.tools.ant.filters.ReplaceTokens, tokens: resourceTokens } // Exclude tests that need a runner diff --git a/runners/google-cloud-dataflow-java/build.gradle b/runners/google-cloud-dataflow-java/build.gradle index 9b32c482afac..012308fa5604 100644 --- a/runners/google-cloud-dataflow-java/build.gradle +++ b/runners/google-cloud-dataflow-java/build.gradle @@ -57,13 +57,16 @@ ext.dataflowFnapiContainerVersion = 'beam-master-20260731' ext.dataflowContainerBaseRepository = 'gcr.io/cloud-dataflow/v1beta3' processResources { - filter org.apache.tools.ant.filters.ReplaceTokens, tokens: [ + // Filter arguments are not tracked by Gradle, so declare the tokens as an input property. + def resourceTokens = [ 'dataflow.legacy_environment_major_version' : project.ext.dataflowFnapiEnvironmentMajorVersion, 'dataflow.fnapi_environment_major_version' : project.ext.dataflowFnapiEnvironmentMajorVersion, 'dataflow.legacy_container_version' : project.ext.dataflowLegacyContainerVersion, 'dataflow.fnapi_container_version' : project.ext.dataflowFnapiContainerVersion, 'dataflow.container_base_repository' : project.ext.dataflowContainerBaseRepository, ] + inputs.property('resourceTokens', resourceTokens) + filter org.apache.tools.ant.filters.ReplaceTokens, tokens: resourceTokens } // Exclude tests that need a runner diff --git a/sdks/java/extensions/sql/jdbc/build.gradle b/sdks/java/extensions/sql/jdbc/build.gradle index 91c48b635d50..334da3293084 100644 --- a/sdks/java/extensions/sql/jdbc/build.gradle +++ b/sdks/java/extensions/sql/jdbc/build.gradle @@ -63,9 +63,12 @@ dependencies { } processResources { - filter org.apache.tools.ant.filters.ReplaceTokens, tokens: [ + // Filter arguments are not tracked by Gradle, so declare the tokens as an input property. + def resourceTokens = [ 'beam.version': version, ] + inputs.property('resourceTokens', resourceTokens) + filter org.apache.tools.ant.filters.ReplaceTokens, tokens: resourceTokens } shadowJar { diff --git a/sdks/java/maven-archetypes/examples/build.gradle b/sdks/java/maven-archetypes/examples/build.gradle index 1edb55a10f95..c3c203410c88 100644 --- a/sdks/java/maven-archetypes/examples/build.gradle +++ b/sdks/java/maven-archetypes/examples/build.gradle @@ -25,7 +25,8 @@ example pipelines from the Apache Beam Java SDK.""" processResources { dependsOn 'generateSources' - filter org.apache.tools.ant.filters.ReplaceTokens, tokens: [ + // Filter arguments are not tracked by Gradle, so declare the tokens as an input property. + def resourceTokens = [ 'project.version': version, 'bigquery.version': dependencies.create(project.library.java.google_api_services_bigquery).getVersion(), 'google-api-client.version': dependencies.create(project.library.java.google_api_client).getVersion(), @@ -49,6 +50,8 @@ processResources { 'maven-surefire-plugin.version': dependencies.create(project.library.maven.maven_surefire_plugin).getVersion(), 'flink.artifact.name': 'beam-runners-flink-'.concat(project(":runners:flink:${project.ext.latestFlinkVersion}").getName()), ] + inputs.property('resourceTokens', resourceTokens) + filter org.apache.tools.ant.filters.ReplaceTokens, tokens: resourceTokens } /* diff --git a/sdks/java/maven-archetypes/gcp-bom-examples/build.gradle b/sdks/java/maven-archetypes/gcp-bom-examples/build.gradle index f9fabcfe19b0..845cf5aa5c98 100644 --- a/sdks/java/maven-archetypes/gcp-bom-examples/build.gradle +++ b/sdks/java/maven-archetypes/gcp-bom-examples/build.gradle @@ -26,7 +26,8 @@ using the Beam Google Cloud Platform BOM""" processResources { dependsOn 'generateSources' - filter org.apache.tools.ant.filters.ReplaceTokens, tokens: [ + // Filter arguments are not tracked by Gradle, so declare the tokens as an input property. + def resourceTokens = [ 'project.version': version, 'bigquery.version': dependencies.create(project.library.java.google_api_services_bigquery).getVersion(), 'google-api-client.version': dependencies.create(project.library.java.google_api_client).getVersion(), @@ -48,6 +49,8 @@ processResources { 'maven-surefire-plugin.version': dependencies.create(project.library.maven.maven_surefire_plugin).getVersion(), 'flink.artifact.name': 'beam-runners-flink-'.concat(project(":runners:flink:${project.ext.latestFlinkVersion}").getName()), ] + inputs.property('resourceTokens', resourceTokens) + filter org.apache.tools.ant.filters.ReplaceTokens, tokens: resourceTokens } /* diff --git a/sdks/java/maven-archetypes/starter/build.gradle b/sdks/java/maven-archetypes/starter/build.gradle index 7c8c02bd5593..3bb4a311fca1 100644 --- a/sdks/java/maven-archetypes/starter/build.gradle +++ b/sdks/java/maven-archetypes/starter/build.gradle @@ -24,10 +24,13 @@ ext.summary = """A Maven archetype to create a simple starter pipeline to get started using the Apache Beam Java SDK.""" processResources { - filter org.apache.tools.ant.filters.ReplaceTokens, tokens: [ + // Filter arguments are not tracked by Gradle, so declare the tokens as an input property. + def resourceTokens = [ 'project.version': version, 'slf4j.version': dependencies.create(project.library.java.slf4j_api).getVersion(), 'maven-compiler-plugin.version': dependencies.create(project.library.maven.maven_compiler_plugin).getVersion(), 'maven-exec-plugin.version': dependencies.create(project.library.maven.maven_exec_plugin).getVersion(), ] + inputs.property('resourceTokens', resourceTokens) + filter org.apache.tools.ant.filters.ReplaceTokens, tokens: resourceTokens }