diff --git a/.test-infra/metrics/build.gradle b/.test-infra/metrics/build.gradle index 384a7b465102..cf153690d812 100644 --- a/.test-infra/metrics/build.gradle +++ b/.test-infra/metrics/build.gradle @@ -27,7 +27,7 @@ ext { cluster = 'metrics' zone='us-central1-a' projectName='apache-beam-testing' - kubeConfigPath = project.properties['KUBE_CONFIG_PATH'] ?: '/home/jenkins/.kube/config' + kubeConfigPath = project.findProperty('KUBE_CONFIG_PATH') ?: '/home/jenkins/.kube/config' } applyGroovyNature() diff --git a/build.gradle.kts b/build.gradle.kts index 1cefe1ac5b93..cdc46ef2bb0a 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -869,10 +869,9 @@ tasks.register("pushAllRunnersDockerImages") { doLast { if (project.hasProperty("prune-images")) { - exec { - executable("docker") - args("system", "prune", "-a", "--force") - } + providers.exec { + commandLine("docker", "system", "prune", "-a", "--force") + }.result.get() } } } @@ -893,10 +892,9 @@ tasks.register("pushAllSdkDockerImages") { doLast { if (project.hasProperty("prune-images")) { - exec { - executable("docker") - args("system", "prune", "-a", "--force") - } + providers.exec { + commandLine("docker", "system", "prune", "-a", "--force") + }.result.get() } } } @@ -914,10 +912,9 @@ tasks.register("pushAllXlangDockerImages") { doLast { if (project.hasProperty("prune-images")) { - exec { - executable("docker") - args("system", "prune", "-a", "--force") - } + providers.exec { + commandLine("docker", "system", "prune", "-a", "--force") + }.result.get() } } } diff --git a/buildSrc/build.gradle.kts b/buildSrc/build.gradle.kts index c8bcc22c9021..6d14ebc559a4 100644 --- a/buildSrc/build.gradle.kts +++ b/buildSrc/build.gradle.kts @@ -64,9 +64,9 @@ dependencies { exclude(group="org.codehaus.plexus", module="plexus-xml") // plexus-xml 4.x requires Java17 } runtimeOnly("org.codehaus.plexus:plexus-xml:3.0.2") // Do not bump to plexus-xml 4.x which requires Java17 - implementation("com.github.spotbugs.snom:spotbugs-gradle-plugin:5.0.14") + implementation("com.github.spotbugs.snom:spotbugs-gradle-plugin:6.0.3") - runtimeOnly("com.google.protobuf:protobuf-gradle-plugin:0.8.13") // Enable proto code generation + runtimeOnly("com.google.protobuf:protobuf-gradle-plugin:0.9.6") // Enable proto code generation runtimeOnly("com.github.davidmc24.gradle.plugin:gradle-avro-plugin:1.9.1") // Enable Avro code generation. Version 1.1.0 is the last supporting avro 1.10.2 runtimeOnly("com.diffplug.spotless:spotless-plugin-gradle:7.2.1") // Enable a code formatting plugin runtimeOnly("gradle.plugin.com.dorongold.plugins:task-tree:1.5") // Adds a 'taskTree' task to print task dependency tree @@ -77,7 +77,7 @@ dependencies { runtimeOnly("com.avast.gradle:gradle-docker-compose-plugin:0.16.12") // Enable docker compose tasks runtimeOnly("ca.cutterslade.gradle:gradle-dependency-analyze:1.8.3") // Enable dep analysis runtimeOnly("gradle.plugin.net.ossindex:ossindex-gradle-plugin:0.4.11") // Enable dep vulnerability analysis - runtimeOnly("org.checkerframework:checkerframework-gradle-plugin:0.6.56") // Enable enhanced static checking plugin + implementation("org.checkerframework:org.checkerframework.gradle.plugin:1.0.2") // Enable enhanced static checking plugin } // Because buildSrc is built and tested automatically _before_ gradle diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy index 1e1c9108de76..206e32cca827 100644 --- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy +++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy @@ -29,6 +29,8 @@ import org.gradle.api.GradleException import org.gradle.api.JavaVersion import org.gradle.api.Plugin import org.gradle.api.Project +import org.gradle.api.plugins.BasePluginExtension +import org.gradle.api.plugins.JavaPluginExtension import org.gradle.api.artifacts.Configuration import org.gradle.api.artifacts.ProjectDependency import org.gradle.api.file.FileCollection @@ -522,7 +524,7 @@ class BeamModulePlugin implements Plugin { // Default to dash-separated directories for artifact base name, // which will also be the default artifactId for maven publications project.apply plugin: 'base' - project.archivesBaseName = defaultArchivesBaseName(project) + project.extensions.getByType(BasePluginExtension).archivesName.set(defaultArchivesBaseName(project)) // Register all Beam repositories and configuration tweaks Repositories.register(project) @@ -615,11 +617,11 @@ class BeamModulePlugin implements Plugin { def aws_java_sdk2_version = "2.20.162" def cassandra_driver_version = "3.10.2" def cdap_version = "6.11.4" - def checkerframework_version = "3.42.0" + def checkerframework_version = "3.49.5" def classgraph_version = "4.8.192" def delta_lake_version = "4.2.0" def dbcp2_version = "2.9.0" - def errorprone_version = "2.31.0" + def errorprone_version = "2.36.0" // [bomupgrader] determined by: com.google.api:gax, consistent with: google_cloud_platform_libraries_bom def gax_version = "2.84.0" def google_ads_version = "33.0.0" @@ -1034,9 +1036,9 @@ class BeamModulePlugin implements Plugin { if (ver == '25') { // TODO: Upgrade errorprone and checkerFramework when Beam move to Java17+ and enable it in Java 25+ options.errorprone.errorproneArgs.add("-XepDisableAllChecks") - project.checkerFramework { - skipCheckerFramework = true - } + project.extensions.getByType( + org.checkerframework.plugin.gradle.CheckerFrameworkExtension + ).skipCheckerFramework.set(true) options.compilerArgs += [ '-Xlint:-dangling-doc-comments' ] @@ -1053,10 +1055,10 @@ class BeamModulePlugin implements Plugin { url "file://${project.rootProject.projectDir}/testPublication/" } maven { - url(project.properties['distMgmtSnapshotsUrl'] ?: isRelease(project) + url = project.findProperty('distMgmtSnapshotsUrl') ?: (isRelease(project) ? 'https://repository.apache.org/service/local/staging/deploy/maven2' : 'https://repository.apache.org/content/repositories/snapshots') - name(project.properties['distMgmtServerId'] ?: isRelease(project) + name = project.findProperty('distMgmtServerId') ?: (isRelease(project) ? 'apache.releases.https' : 'apache.snapshots.https') // The maven settings plugin will load credentials from ~/.m2/settings.xml file that a user // has configured with the Apache release and snapshot staging credentials. @@ -1135,7 +1137,7 @@ class BeamModulePlugin implements Plugin { } if (configuration.archivesBaseName) { - project.archivesBaseName = configuration.archivesBaseName + project.extensions.getByType(BasePluginExtension).archivesName.set(configuration.archivesBaseName) } project.apply plugin: "java" @@ -1200,8 +1202,9 @@ class BeamModulePlugin implements Plugin { } } - project.sourceCompatibility = project.javaVersion - project.targetCompatibility = project.javaVersion + def javaExtension = project.extensions.getByType(JavaPluginExtension) + javaExtension.sourceCompatibility = JavaVersion.toVersion(project.javaVersion) + javaExtension.targetCompatibility = JavaVersion.toVersion(project.javaVersion) def defaultLintSuppressions = [ 'options', @@ -1295,35 +1298,34 @@ class BeamModulePlugin implements Plugin { String skipUsesCombinedRegex = skipUsesRegexes.collect({ regex -> "(${regex})"}).join("|") project.apply plugin: 'org.checkerframework' - project.checkerFramework { - checkers = [ - 'org.checkerframework.checker.nullness.NullnessChecker' - ] - - // Only skip checkerframework if explicitly requested - skipCheckerFramework = project.hasProperty('enableCheckerFramework') && - !parseBooleanProperty(project, 'enableCheckerFramework') - - // Always exclude checkerframework on tests. It's slow, and it often - // raises erroneous error because we don't have checker annotations for - // test libraries like junit and hamcrest. See BEAM-11436. - // Consider re-enabling if we can get annotations for the test libraries - // we use. - excludeTests = true - - extraJavacArgs = [ - "-AskipDefs=${skipDefCombinedRegex}", - "-AskipUses=${skipUsesCombinedRegex}", - "-AnoWarnMemoryConstraints", - "-AsuppressWarnings=annotation.not.completed,keyfor", - ] + def checkerFrameworkExtension = + project.extensions.getByType(org.checkerframework.plugin.gradle.CheckerFrameworkExtension) + checkerFrameworkExtension.version.set(checkerframework_version) + checkerFrameworkExtension.checkers.set([ + 'org.checkerframework.checker.nullness.NullnessChecker' + ]) + + // Only skip checkerframework if explicitly requested + checkerFrameworkExtension.skipCheckerFramework.set( + project.hasProperty('enableCheckerFramework') && + !parseBooleanProperty(project, 'enableCheckerFramework')) + + // Always exclude checkerframework on tests. It's slow, and it often + // raises erroneous error because we don't have checker annotations for + // test libraries like junit and hamcrest. See BEAM-11436. + // Consider re-enabling if we can get annotations for the test libraries + // we use. + checkerFrameworkExtension.excludeTests.set(true) + + checkerFrameworkExtension.extraJavacArgs.set([ + "-AskipDefs=${skipDefCombinedRegex}", + "-AskipUses=${skipUsesCombinedRegex}", + "-AnoWarnMemoryConstraints", + "-AsuppressWarnings=annotation.not.completed,keyfor", + ]) - project.dependencies { - checkerFramework("org.checkerframework:checker:$checkerframework_version") - } - project.configurations.all { - it.exclude(group:"org.checkerframework", module:"jdk8") - } + project.configurations.all { + it.exclude(group:"org.checkerframework", module:"jdk8") } // Ban these dependencies from all configurations @@ -1533,7 +1535,7 @@ class BeamModulePlugin implements Plugin { enforceCheck !disableSpotlessCheck java { licenseHeader javaLicenseHeader - googleJavaFormat('1.17.0') + googleJavaFormat('1.35.0') target project.fileTree(project.projectDir) { include 'src/*/java/**/*.java' exclude '**/DefaultPackageTest.java' @@ -1681,6 +1683,23 @@ class BeamModulePlugin implements Plugin { '-Werror' ] + (defaultLintSuppressions + configuration.disableLintWarnings).collect { "-Xlint:-${it}" }) + + // JDK 25 reports existing dangling Javadoc comments as warnings. + if (JavaVersion.current().isCompatibleWith(JavaVersion.VERSION_25)) { + options.compilerArgs += ['-Xlint:-dangling-doc-comments'] + + if (!forkJavaVersion) { + options.errorprone.errorproneArgs.add("-XepDisableAllChecks") + } + + // Skip Checker Framework when compiling directly with JDK 25. + // The forked-JDK path configures this in setJavaVerOptions(). + if (!forkJavaVersion) { + project.extensions.getByType( + org.checkerframework.plugin.gradle.CheckerFrameworkExtension + ).skipCheckerFramework.set(true) + } + } } if (forkJavaVersion) { @@ -1811,7 +1830,7 @@ class BeamModulePlugin implements Plugin { if (configuration.validateShadowJar) { def validateShadedJarDoesntLeakNonProjectClasses = project.tasks.register('validateShadedJarDoesntLeakNonProjectClasses') { dependsOn 'shadowJar' - ext.outFile = project.file("${project.reportsDir}/${name}.out") + ext.outFile = project.layout.buildDirectory.file("reports/${name}.out").get().asFile inputs.files(project.configurations.shadow.artifacts.files) .withPropertyName("shadowArtifactsFiles") .withPathSensitivity(PathSensitivity.RELATIVE) @@ -1858,15 +1877,15 @@ class BeamModulePlugin implements Plugin { implementation project.library.java.jmh_core } - project.compileJava { + project.tasks.named("compileJava", JavaCompile).configure { // Always exclude checkerframework on JMH generated code. It's slow, // and it often raises erroneous error because we don't have checker // annotations for generated code and test libraries. // // Consider re-enabling if we can get annotations for the generated // code and test libraries we use. - checkerFramework { - skipCheckerFramework = true + options.extensions.configure("checkerFrameworkCompile") { + enabled = false } } @@ -1967,7 +1986,7 @@ class BeamModulePlugin implements Plugin { new File("${pomPropertiesFile}").text = """version=${project.version} groupId=${project.mavenGroupId} - artifactId=${project.archivesBaseName}""" + artifactId=${project.extensions.getByType(BasePluginExtension).archivesName.get()}""" } } @@ -1987,13 +2006,13 @@ class BeamModulePlugin implements Plugin { } dependsOn 'generatePomFileForMavenJavaPublication' - into("META-INF/maven/${project.mavenGroupId}/${project.archivesBaseName}") { + into("META-INF/maven/${project.mavenGroupId}/${project.extensions.getByType(BasePluginExtension).archivesName.get()}") { from "${pomFile}" rename('.*', 'pom.xml') } dependsOn project.generatePomPropertiesFileForMavenJavaPublication - into("META-INF/maven/${project.mavenGroupId}/${project.archivesBaseName}") { from "${pomPropertiesFile}" } + into("META-INF/maven/${project.mavenGroupId}/${project.extensions.getByType(BasePluginExtension).archivesName.get()}") { from "${pomPropertiesFile}" } } // Only build artifacts for archives if we are publishing @@ -2050,7 +2069,7 @@ class BeamModulePlugin implements Plugin { } artifact project.javadocJar - artifactId = project.archivesBaseName + artifactId = project.extensions.getByType(BasePluginExtension).archivesName.get() groupId = project.mavenGroupId pom { @@ -2147,7 +2166,7 @@ class BeamModulePlugin implements Plugin { if (it instanceof ProjectDependency) { dependencyNode.appendNode('groupId', it.getDependencyProject().mavenGroupId) - dependencyNode.appendNode('artifactId', it.getDependencyProject().archivesBaseName) + dependencyNode.appendNode('artifactId', it.getDependencyProject().extensions.getByType(BasePluginExtension).archivesName.get()) dependencyNode.appendNode('version', it.version) dependencyNode.appendNode('scope', param.scope) appendClassifier(it) @@ -2288,7 +2307,7 @@ class BeamModulePlugin implements Plugin { } else { project.evaluationDependsOn(":runners:google-cloud-dataflow-java:worker") def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?: - project.project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath + project.project(":runners:google-cloud-dataflow-java:worker").shadowJar.archiveFile.get().asFile allOptionsList.addAll([ // Keep as legacy flag to ensure via test this flag works for // legacy pipeline. @@ -2554,13 +2573,13 @@ class BeamModulePlugin implements Plugin { sourceDirs += project.file(generatedProtoMainJavaDir) generatedSourceDirs += project.file(generatedProtoMainJavaDir) - testSourceDirs += project.file(generatedProtoTestJavaDir) + testSources.from(project.file(generatedProtoTestJavaDir)) generatedSourceDirs += project.file(generatedProtoTestJavaDir) sourceDirs += project.file(generatedGrpcMainJavaDir) generatedSourceDirs += project.file(generatedGrpcMainJavaDir) - testSourceDirs += project.file(generatedGrpcTestJavaDir) + testSources.from(project.file(generatedGrpcTestJavaDir)) generatedSourceDirs += project.file(generatedGrpcTestJavaDir) } } @@ -2574,7 +2593,7 @@ class BeamModulePlugin implements Plugin { PortabilityNatureConfiguration configuration = it ? it as PortabilityNatureConfiguration : new PortabilityNatureConfiguration() if (configuration.archivesBaseName) { - project.archivesBaseName = configuration.archivesBaseName + project.extensions.getByType(BasePluginExtension).archivesName.set(configuration.archivesBaseName) } project.ext.applyJavaNature( @@ -2853,7 +2872,7 @@ class BeamModulePlugin implements Plugin { def pythonDir = project.project(":sdks:python").projectDir def javaPort = getRandomPort() def pythonPort = getRandomPort() - def expansionJar = project.project(':sdks:java:testing:expansion-service').buildTestExpansionServiceJar.archivePath + def expansionJar = project.project(':sdks:java:testing:expansion-service').tasks.named('buildTestExpansionServiceJar', com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar).get().archiveFile.get().asFile def javaClassLookupAllowlistFile = project.project(":sdks:java:testing:expansion-service").projectDir.getPath() + "/src/test/resources/test_expansion_service_allowlist.yaml" def expansionServiceOpts = [ "group_id": project.name, @@ -3047,7 +3066,7 @@ class BeamModulePlugin implements Plugin { def envDir = project.project(":sdks:python").envdir def pythonDir = project.project(":sdks:python").projectDir def externalPort = getRandomPort() - def launcherJar = project.project(':sdks:java:transform-service:app').shadowJar.archivePath + def launcherJar = project.project(':sdks:java:transform-service:app').shadowJar.archiveFile.get().asFile def groupId = project.name + randomUUID().toString() def transformServiceOpts = [ "transform_service_launcher_jar": launcherJar, @@ -3386,8 +3405,8 @@ class BeamModulePlugin implements Plugin { "--runner=${runner}", "--parallelism=2", "--sdk_worker_parallelism=1", - "--flink_job_server_jar=${project.project(flinkJobServerProject).shadowJar.archivePath}", - "--spark_job_server_jar=${project.project(':runners:spark:3:job-server').shadowJar.archivePath}", + "--flink_job_server_jar=${project.project(flinkJobServerProject).shadowJar.archiveFile.get().asFile}", + "--spark_job_server_jar=${project.project(':runners:spark:3:job-server').shadowJar.archiveFile.get().asFile}", ] if (isStreaming) options += [ diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/VendorJavaPlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/VendorJavaPlugin.groovy index f6a581b4fc68..f9db280aa940 100644 --- a/buildSrc/src/main/groovy/org/apache/beam/gradle/VendorJavaPlugin.groovy +++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/VendorJavaPlugin.groovy @@ -21,6 +21,7 @@ package org.apache.beam.gradle import org.gradle.api.GradleException import org.gradle.api.Plugin import org.gradle.api.Project +import org.gradle.api.plugins.BasePluginExtension import org.gradle.api.artifacts.ProjectDependency import org.gradle.api.file.FileTree import org.gradle.api.publish.maven.MavenPublication @@ -74,7 +75,7 @@ class VendorJavaPlugin implements Plugin { VendorJavaPluginConfig config = it ? it as VendorJavaPluginConfig : new VendorJavaPluginConfig() project.apply plugin: 'base' - project.archivesBaseName = config.artifactId + project.extensions.getByType(BasePluginExtension).archivesName.set(config.artifactId) if (!isRelease(project)) { config.version += '-SNAPSHOT' @@ -303,7 +304,7 @@ artifactId=${project.name} if (it instanceof ProjectDependency) { dependencyNode.appendNode('groupId', it.getDependencyProject().mavenGroupId) - dependencyNode.appendNode('artifactId', it.getDependencyProject().archivesBaseName) + dependencyNode.appendNode('artifactId', it.getDependencyProject().extensions.getByType(BasePluginExtension).archivesName.get()) dependencyNode.appendNode('version', it.version) dependencyNode.appendNode('scope', param.scope) appendClassifier(it) diff --git a/examples/kotlin/build.gradle b/examples/kotlin/build.gradle index 60ddc1c52d7f..5a93ab9b655c 100644 --- a/examples/kotlin/build.gradle +++ b/examples/kotlin/build.gradle @@ -19,7 +19,7 @@ import groovy.json.JsonOutput plugins { id 'org.apache.beam.module' - id 'org.jetbrains.kotlin.jvm' version '1.6.10' + id 'org.jetbrains.kotlin.jvm' version '2.4.21' } applyJavaNature(exportJavadoc: false, automaticModuleName: 'org.apache.beam.examples.kotlin') @@ -46,7 +46,7 @@ configurations.sparkRunnerPreCommit { exclude group: "org.slf4j", module: "slf4j-jdk14" } -def kotlin_version = "1.6.10" +def kotlin_version = "2.4.21" dependencies { implementation enforcedPlatform(library.java.google_cloud_platform_libraries_bom) diff --git a/examples/multi-language/build.gradle b/examples/multi-language/build.gradle index 71e2ba543a19..612b1ab5c37a 100644 --- a/examples/multi-language/build.gradle +++ b/examples/multi-language/build.gradle @@ -18,7 +18,9 @@ apply plugin: 'org.apache.beam.module' apply plugin: 'application' -mainClassName = "org.apache.beam.sdk.expansion.service.ExpansionService" +application { + mainClass = "org.apache.beam.sdk.expansion.service.ExpansionService" +} applyJavaNature( automaticModuleName: 'org.apache.beam.examples.multilanguage', diff --git a/learning/tour-of-beam/frontend/build.gradle.kts b/learning/tour-of-beam/frontend/build.gradle.kts index 2a70778bf08a..11adb87e931a 100644 --- a/learning/tour-of-beam/frontend/build.gradle.kts +++ b/learning/tour-of-beam/frontend/build.gradle.kts @@ -20,138 +20,95 @@ evaluationDependsOn(":playground:frontend:playground_components") tasks.register("generate") { dependsOn(":playground:frontend:playground_components:generate") - dependsOn("generateCode") - group = "build" description = "Generates all generated files." } -tasks.register("analyze") { +tasks.register("analyze") { dependsOn(":playground:frontend:playground_components:generateCode") dependsOn("generateCode") - group = "verification" description = "Analyze dart code" - - doLast { - exec { - executable("dart") - args("analyze", "lib", "test") - } - } + commandLine("dart", "analyze", "lib", "test") } -tasks.register("pubGet") { +tasks.register("pubGet") { group = "build" description = "Get packages for the frontend project" - doLast { - exec { - executable("flutter") - args("pub", "get") - } - } + commandLine("flutter", "pub", "get") } -tasks.register("format") { +tasks.register("format") { group = "build" description = "Idiomatically format Dart source code" - doLast { - exec { - executable("dart") - args("format", "lib", "test") - } - } + commandLine("dart", "format", "lib", "test") } -tasks.register("run") { +tasks.register("run") { group = "application" description = "Run application on Google Chrome" - - doLast { - exec { - executable("flutter") - args("run", "-d", "chrome") - } - } + commandLine("flutter", "run", "-d", "chrome") } -tasks.register("test") { +tasks.register("test") { dependsOn(":playground:frontend:playground_components:generateCode") dependsOn("generateCode") - group = "verification" description = "flutter test" - - doLast { - exec { - executable("flutter") - args("test") - } - } + commandLine("flutter", "test") } tasks.register("precommit") { dependsOn(":playground:frontend:playground_components:precommit") - dependsOn("analyze") dependsOn("test") } -tasks.register("generateCode") { - dependsOn(":playground:frontend:playground_components:generateCode") - +tasks.register("generateCode") { dependsOn("cleanFlutter") dependsOn("pubGet") - group = "build" description = "Generate code" - - doLast { - exec { - executable("flutter") - args("pub", "run", "build_runner", "build", "--delete-conflicting-outputs") - } - } + commandLine( + "flutter", "pub", "run", "build_runner", "build", + "--delete-conflicting-outputs" + ) } -tasks.register("cleanFlutter") { +tasks.register("cleanFlutter") { group = "build" description = "Remove build artifacts" + commandLine("flutter", "clean") +} - doLast { - exec { - executable("flutter") - args("clean") +val deleteFilesByRegExp: (String) -> Unit = { re -> + val regex = Regex(re) + val matchedFiles = listOf("assets", "lib", "test").flatMap { directory -> + val base = file(directory) + if (base.exists()) { + fileTree(base).files.filter { + regex.matches(it.relativeTo(base).invariantSeparatorsPath) + } + } else { + emptyList() } - } + }.distinct() + + matchedFiles.forEach { println(it) } + delete(matchedFiles) } tasks.register("cleanGenerated") { dependsOn(":playground:frontend:playground_components:cleanGenerated") - group = "build" description = "Remove build artifacts" doLast { println("Deleting:") - - deleteFilesByRegExp(".*\\.g\\.dart\$") - deleteFilesByRegExp(".*\\.gen\\.dart\$") - deleteFilesByRegExp(".*\\.mocks\\.dart\$") - } -} - -val deleteFilesByRegExp: (String) -> Unit = { re -> - // Prints file names. - exec { - executable("find") - args("assets", "lib", "test", "-regex", re) - } - // Actually deletes them. - exec { - executable("find") - args("assets", "lib", "test", "-regex", re, "-delete") + deleteFilesByRegExp(".*\\.g\\.dart$") + deleteFilesByRegExp(".*\\.gen\\.dart$") + deleteFilesByRegExp(".*\\.mocks\\.dart$") } } @@ -160,30 +117,24 @@ tasks.register("integrationTest") { dependsOn("integrationTest_welcome_page") } -tasks.register("integrationTest_tour_page") { - doLast { - runIntegrationTest("tour_page", "/") - } -} +val integrationDeviceId = providers.gradleProperty("deviceId").getOrElse("chrome") -tasks.register("integrationTest_welcome_page") { - doLast { - runIntegrationTest("welcome_page", "/") - } +tasks.register("integrationTest_tour_page") { + commandLine( + "flutter", "drive", + "--driver=test_driver/integration_test.dart", + "--target=integration_test/tour_page_test.dart", + "--web-launch-url='/'", + "--device-id=$integrationDeviceId" + ) } -fun runIntegrationTest(path: String, url: String) { - // Run with -PdeviceId=web-server for headless tests. - val deviceId: String = if (project.hasProperty("deviceId")) project.property("deviceId") as String else "chrome" - - exec { - executable = "flutter" - args( - "drive", - "--driver=test_driver/integration_test.dart", - "--target=integration_test/${path}_test.dart", - "--web-launch-url='$url'", - "--device-id=$deviceId", - ) - } +tasks.register("integrationTest_welcome_page") { + commandLine( + "flutter", "drive", + "--driver=test_driver/integration_test.dart", + "--target=integration_test/welcome_page_test.dart", + "--web-launch-url='/'", + "--device-id=$integrationDeviceId" + ) } diff --git a/learning/tour-of-beam/terraform/build.gradle.kts b/learning/tour-of-beam/terraform/build.gradle.kts index 3080f74a5149..979628b12b25 100644 --- a/learning/tour-of-beam/terraform/build.gradle.kts +++ b/learning/tour-of-beam/terraform/build.gradle.kts @@ -86,39 +86,37 @@ tasks.register("terraformDestroy") { ) } -tasks.register("getRouterHost") { +tasks.register("getRouterHost") { group = "backend-deploy" + commandLine( + "kubectl", + "get", + "svc", + "-l", + "app=backend-router-grpc", + "-o", + "jsonpath='{.items[0].status.loadBalancer.ingress[0].ip}:{.items[0].spec.ports[0].port}'" + ) + standardOutput = ByteArrayOutputStream() + doLast { - val result = ByteArrayOutputStream() - exec { - commandLine( - "kubectl", - "get", - "svc", - "-l", - "app=backend-router-grpc", - "-o", - "jsonpath='{.items[0].status.loadBalancer.ingress[0].ip}:{.items[0].spec.ports[0].port}'" - ) - standardOutput = result - } + val result = standardOutput as ByteArrayOutputStream val pgRouterHost = result.toString().trim().replace("'", "") project.extensions.extraProperties["pg_router_host"] = pgRouterHost } } -tasks.register("indexcreate") { +tasks.register("indexcreate") { group = "backend-deploy" dependsOn("getRouterHost") - doLast { - - val indexpath = "../backend/internal/storage/index.yaml" - exec { - executable("gcloud") - args("datastore", "indexes", "create", indexpath) - } - } + commandLine( + "gcloud", + "datastore", + "indexes", + "create", + "../backend/internal/storage/index.yaml" + ) } tasks.register("firebaseProjectCreate") { @@ -126,21 +124,18 @@ tasks.register("firebaseProjectCreate") { dependsOn("prepareFirebasercConfig") doLast { - val result = ByteArrayOutputStream() val projectId = project.property("project_id") as String - exec { - executable("firebase") - args("projects:list") - standardOutput = result - } - val output = result.toString().trim() + val output = providers.exec { + commandLine("firebase", "projects:list") + }.standardOutput.asText.get().trim() + if (output.contains(projectId)) { println("Firebase is already added to project $projectId.") } else { - exec { - executable("firebase") - args("projects:addfirebase", projectId) - }.assertNormalExitValue() + providers.exec { + commandLine("firebase", "projects:addfirebase", projectId) + }.result.get().assertNormalExitValue() + println("Firebase has been added to project $projectId.") } } @@ -153,34 +148,31 @@ tasks.register("firebaseHostingCreate") { doLast { val projectId = project.property("project_id") as String val webapp_id = project.property("webapp_id") as String - val listSitesResult = ByteArrayOutputStream() - exec { - executable("firebase") - args("hosting:sites:list", "--project", projectId) - workingDir("../frontend") - standardOutput = listSitesResult - } + val frontendDir = project.file("../frontend") + + val listSitesOutput = providers.exec { + commandLine("firebase", "hosting:sites:list", "--project", projectId) + workingDir(frontendDir) + }.standardOutput.asText.get() + + println(listSitesOutput) - println(listSitesResult) - val output = listSitesResult.toString() val regex = "\\b$webapp_id\\b".toRegex() - if (regex.containsMatchIn(output)) { - println("Firebase is already added to project $projectId.") + if (regex.containsMatchIn(listSitesOutput)) { + println("Firebase hosting site is already added to project $projectId.") } else { - exec { - executable("firebase") - args("hosting:sites:create", webapp_id) - workingDir("../frontend") - }.assertNormalExitValue() + providers.exec { + commandLine("firebase", "hosting:sites:create", webapp_id) + workingDir(frontendDir) + }.result.get().assertNormalExitValue() + println("Firebase hosting site has been added to project $projectId.") } - exec { - executable("firebase") - args("target:apply", "hosting", webapp_id , webapp_id) - workingDir("../frontend") - - }.assertNormalExitValue() + providers.exec { + commandLine("firebase", "target:apply", "hosting", webapp_id, webapp_id) + workingDir(frontendDir) + }.result.get().assertNormalExitValue() val file = project.file("../frontend/firebase.json") val content = file.readText() @@ -199,16 +191,15 @@ tasks.register("firebaseWebAppCreate") { dependsOn("firebaseProjectCreate") doLast { - val result = ByteArrayOutputStream() val projectId = project.property("project_id") as String val webappId = project.property("webapp_id") as String - exec { - executable("firebase") - args("apps:list", "--project", projectId) - standardOutput = result - } - println(result) - val output = result.toString() + + val output = providers.exec { + commandLine("firebase", "apps:list", "--project", projectId) + }.standardOutput.asText.get() + + println(output) + if (output.contains(webappId)) { println("Webapp id $webappId is already created on the project: $projectId.") val regex = Regex("""$webappId[\W]+([\d:a-zA-Z]+)[\W]+WEB""") @@ -216,14 +207,14 @@ tasks.register("firebaseWebAppCreate") { println("Firebase app ID for existing Firebase Web App: $firebaseAppId") project.extensions.extraProperties["firebaseAppId"] = firebaseAppId } else { - val result2 = ByteArrayOutputStream() - exec { - executable("firebase") - args("apps:create", "WEB", webappId, "--project", projectId) - standardOutput = result2 - }.assertNormalExitValue() + val result = providers.exec { + commandLine("firebase", "apps:create", "WEB", webappId, "--project", projectId) + } + val creationOutput = result.standardOutput.asText.get() + result.result.get().assertNormalExitValue() + val firebaseAppId = - result2.toString().lines().find { it.startsWith(" - App ID:") }?.substringAfter(":")?.trim() + creationOutput.lines().find { it.startsWith(" - App ID:") }?.substringAfter(":")?.trim() project.extensions.extraProperties["firebaseAppId"] = firebaseAppId println("Firebase app ID for newly created Firebase Web App: $firebaseAppId") } @@ -237,13 +228,11 @@ tasks.register("getSdkConfigWebApp") { doLast { val firebaseAppId = project.extensions.extraProperties["firebaseAppId"] as String - val result = ByteArrayOutputStream() - exec { - executable("firebase") - args("apps:sdkconfig", "WEB", firebaseAppId) - standardOutput = result + val result = providers.exec { + commandLine("firebase", "apps:sdkconfig", "WEB", firebaseAppId) } - val output = result.toString().trim() + val output = result.standardOutput.asText.get().trim() + result.result.get().assertNormalExitValue() val pattern = Pattern.compile("\\{[^{]*\"locationId\":\\s*\".*?\"[^}]*\\}", Pattern.DOTALL) val matcher = pattern.matcher(output) if (matcher.find()) { @@ -277,11 +266,10 @@ tasks.register("prepareFirebaseOptionsDart") { tasks.register("flutterPubGetPG") { dependsOn("prepareFirebaseOptionsDart") doLast { - exec { - executable("flutter") - args("pub", "get") - workingDir("../../../playground/frontend/playground_components") - } + providers.exec { + commandLine("flutter", "pub", "get") + workingDir(project.file("../../../playground/frontend/playground_components")) + }.result.get().assertNormalExitValue() } } @@ -289,11 +277,10 @@ tasks.register("flutterPubRunPG") { dependsOn("flutterPubGetPG") doLast { - exec { - executable("flutter") - args("pub", "run", "build_runner", "build", "--delete-conflicting-outputs") - workingDir("../../../playground/frontend/playground_components") - } + providers.exec { + commandLine("flutter", "pub", "run", "build_runner", "build", "--delete-conflicting-outputs") + workingDir(project.file("../../../playground/frontend/playground_components")) + }.result.get().assertNormalExitValue() } } @@ -302,11 +289,10 @@ tasks.register("flutterPubGetTob") { dependsOn("getSdkConfigWebApp") doLast { - exec { - executable("flutter") - args("pub", "get") - workingDir("../frontend") - } + providers.exec { + commandLine("flutter", "pub", "get") + workingDir(project.file("../frontend")) + }.result.get().assertNormalExitValue() } } @@ -314,11 +300,10 @@ tasks.register("flutterPubRunTob") { dependsOn("flutterPubGetTob") doLast { - exec { - executable("flutter") - args("pub", "run", "build_runner", "build", "--delete-conflicting-outputs") - workingDir("../frontend") - } + providers.exec { + commandLine("flutter", "pub", "run", "build_runner", "build", "--delete-conflicting-outputs") + workingDir(project.file("../frontend")) + }.result.get().assertNormalExitValue() } } @@ -326,11 +311,10 @@ tasks.register("flutterBuildWeb") { dependsOn("flutterPubRunTob") doLast { - exec { - executable("flutter") - args("build", "web", "-v", "--release") - workingDir("../frontend") - } + providers.exec { + commandLine("flutter", "build", "web", "-v", "--release") + workingDir(project.file("../frontend")) + }.result.get().assertNormalExitValue() } } @@ -340,10 +324,10 @@ tasks.register("firebaseDeploy") { doLast { val projectId = project.property("project_id") as String val webapp_id = project.property("webapp_id") as String - exec { - commandLine("firebase", "deploy", "--only", "hosting:$webapp_id", "--project", projectId) - workingDir("../frontend") - } + providers.exec { + commandLine("firebase", "deploy", "--only", "hosting:$webapp_id", "--project", projectId) + workingDir(project.file("../frontend")) + }.result.get().assertNormalExitValue() } } @@ -400,13 +384,13 @@ tasks.register("uploadLearningMaterials") { doLast { val projectId = project.property("project_id") as String - exec { + providers.exec { commandLine("go", "run", "cmd/ci_cd/ci_cd.go") environment("DATASTORE_PROJECT_ID", projectId) environment("GOOGLE_PROJECT_ID", projectId) environment("TOB_LEARNING_ROOT", "../learning-content/") - workingDir("../backend") - } + workingDir(project.file("../backend")) + }.result.get().assertNormalExitValue() } } diff --git a/playground/backend/build.gradle.kts b/playground/backend/build.gradle.kts index 03060bd61dd0..a95befd4e132 100644 --- a/playground/backend/build.gradle.kts +++ b/playground/backend/build.gradle.kts @@ -4,13 +4,13 @@ * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the - * License); you may not use this file except in compliance + * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an AS IS BASIS, + * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. @@ -18,144 +18,121 @@ description = "Apache Beam :: Playground :: Backend" -task("format") { - group = "build" - description = "Format backend go code" - doLast { - exec { - executable("gofmt") - args("-w", ".") - } - } +tasks.register("format") { + group = "build" + description = "Format backend go code" + executable = "gofmt" + args("-w", ".") } -task("tidy") { - group = "build" - description = "Run go mod tidy" - doLast { - exec { - executable("go") - args("mod", "tidy") - } - } +tasks.register("tidy") { + group = "build" + description = "Run go mod tidy" + executable = "go" + args("mod", "tidy") } -val test by tasks.registering { +val test by tasks.registering(org.gradle.api.tasks.Exec::class) { group = "verification" description = "Test the backend" - doLast { - exec { - executable("go") - args("test", "./...") - } - } + executable = "go" + args("test", "./...") +} + +val cleanTestCache by tasks.registering(org.gradle.api.tasks.Exec::class) { + executable = "go" + args("clean", "-testcache") } -val testWithoutCache by tasks.registering { +val testWithoutCache by tasks.registering(org.gradle.api.tasks.Exec::class) { group = "verification" description = "Test the backend" - doFirst { - exec { - executable("go") - args("clean", "-testcache") - } - } - doLast { - exec { - executable("go") - args("test", "./...") - } - } + dependsOn(cleanTestCache) + executable = "go" + args("test", "./...") } -task("removeUnusedSnippet") { - doLast { - exec { - executable("go") - args("run", "cmd/remove_unused_snippets/remove_unused_snippets.go", "cleanup", - "-day_diff", System.getProperty("dayDiff"), "-project_id", System.getProperty("projectId"), - "-namespace", System.getProperty("namespace")) - } - } +tasks.register("removeUnusedSnippet") { + executable = "go" + args( + "run", + "cmd/remove_unused_snippets/remove_unused_snippets.go", + "cleanup", + "-day_diff", System.getProperty("dayDiff"), + "-project_id", System.getProperty("projectId"), + "-namespace", System.getProperty("namespace") + ) } -task("removeSnippet") { - doLast { - exec { - executable("go") - args("run", "cmd/remove_unused_snippets/remove_unused_snippets.go", "remove", - "-snippet_id", System.getProperty("snippetId"), "-project_id", System.getProperty("projectId"), - "-namespace", System.getProperty("namespace")) - } - } +tasks.register("removeSnippet") { + executable = "go" + args( + "run", + "cmd/remove_unused_snippets/remove_unused_snippets.go", + "remove", + "-snippet_id", System.getProperty("snippetId"), + "-project_id", System.getProperty("projectId"), + "-namespace", System.getProperty("namespace") + ) } -task("benchmarkPrecompiledObjects") { - group = "verification" - description = "Run benchmarks for precompiled objects" - doLast { - exec { - executable("go") - args("test", "-bench", ".", "-benchmem", "./internal/cloud_bucket/...") - } - } +tasks.register("benchmarkPrecompiledObjects") { + group = "verification" + description = "Run benchmarks for precompiled objects" + executable = "go" + args("test", "-bench", ".", "-benchmem", "./internal/cloud_bucket/...") } -task("benchmarkCodeProcessing") { - group = "verification" - description = "Run benchmarks for code processing" - doLast { - exec { - executable("go") - args("test", "-run=^$", "-bench", ".", "-benchmem", "./internal/code_processing/...") - } - } +tasks.register("benchmarkCodeProcessing") { + group = "verification" + description = "Run benchmarks for code processing" + executable = "go" + args( + "test", + "-run=^$", + "-bench", + ".", + "-benchmem", + "./internal/code_processing/..." + ) } -task("benchmark") { - dependsOn(":playground:backend:benchmarkCodeProcessing") +tasks.register("benchmark") { + dependsOn(":playground:backend:benchmarkCodeProcessing") } -task("checkFormat") { - doLast { - val stdout = java.io.ByteArrayOutputStream() +val formatOutput = java.io.ByteArrayOutputStream() - exec { - executable("gofmt") - args("-l", "-e", "-d", ".") - standardOutput = stdout - } - if (stdout.size() > 0) { - println(stdout.toString()) - throw GradleException("gofmt check indicates that there are unformatted files") - } - } -} +tasks.register("checkFormat") { + executable = "gofmt" + args("-l", "-e", "-d", ".") + standardOutput = formatOutput -task("installLinter") { - doLast { - exec { - executable("sh") - args("env_setup.sh") + doLast { + if (formatOutput.size() > 0) { + println(formatOutput.toString()) + throw GradleException( + "gofmt check indicates that there are unformatted files" + ) + } } - } } -task("runLint") { - dependsOn(":playground:backend:installLinter") - doLast { - exec { - executable("golangci-lint") - args("run", "cmd/server/...") - } - } +tasks.register("installLinter") { + executable = "sh" + args("env_setup.sh") } -task("precommit") { - dependsOn(":playground:backend:runLint") - dependsOn(":playground:backend:checkFormat") - dependsOn(":playground:backend:tidy") - dependsOn(":playground:backend:test") - dependsOn(":playground:backend:benchmark") +tasks.register("runLint") { + dependsOn(":playground:backend:installLinter") + executable = "golangci-lint" + args("run", "cmd/server/...") } +tasks.register("precommit") { + dependsOn(":playground:backend:runLint") + dependsOn(":playground:backend:checkFormat") + dependsOn(":playground:backend:tidy") + dependsOn(":playground:backend:test") + dependsOn(":playground:backend:benchmark") +} diff --git a/playground/backend/containers/git-functions.gradle b/playground/backend/containers/git-functions.gradle index 9f5eb39eb0a8..98b13421b2c8 100644 --- a/playground/backend/containers/git-functions.gradle +++ b/playground/backend/containers/git-functions.gradle @@ -17,21 +17,17 @@ */ ext.getGitCommitHash = () -> { - def stdout = new ByteArrayOutputStream() - exec { - executable('git') - args('rev-parse', 'HEAD') - standardOutput = stdout - } - return stdout.toString().trim() + def output = providers.exec { + commandLine('git', 'rev-parse', 'HEAD') + }.standardOutput.asText.get().trim() + + return output } as Closure ext.getGitCommitTimestamp = () -> { - def stdout = new ByteArrayOutputStream() - exec { - executable('git') - args('show', '-s', '--format=%ct', 'HEAD') - standardOutput = stdout - } - return stdout.toString().trim() + def output = providers.exec { + commandLine('git', 'show', '-s', '--format=%ct', 'HEAD') + }.standardOutput.asText.get().trim() + + return output } as Closure diff --git a/playground/build.gradle.kts b/playground/build.gradle.kts index 250d7cf132d0..fc37eca4a57a 100644 --- a/playground/build.gradle.kts +++ b/playground/build.gradle.kts @@ -18,28 +18,20 @@ description = "Apache Beam :: Playground" -// generate protobuf client and server stubs -task("generateProto") { - group = "build" - doLast { - exec { - executable("buf") - args("generate") - } - } +// Generate protobuf client and server stubs +tasks.register("generateProto") { + group = "build" + workingDir = projectDir + commandLine("buf", "generate") } -// lint proto files -task("lintProto") { - group = "build" - doLast { - exec { - executable("buf") - args("lint", "--path", "api/") - } - } + +tasks.register("lintProto") { + group = "build" + workingDir = projectDir + commandLine("buf", "lint", "--path", "api/") } -task("dockerComposeLocalUp") { +tasks.register("dockerComposeLocalUp") { dependsOn(":playground:backend:containers:router:docker") dependsOn(":playground:backend:containers:go:docker") dependsOn(":playground:backend:containers:java:docker") @@ -49,23 +41,13 @@ task("dockerComposeLocalUp") { group = "build" description = "Start Playground backend and frontend locally" - - doLast { - exec { - executable("docker-compose") - args("-f", "docker-compose.local.yaml", "up", "-d") - } - } + workingDir = projectDir + commandLine("docker-compose", "-f", "docker-compose.local.yaml", "up", "-d") } -task("dockerComposeLocalDown") { +tasks.register("dockerComposeLocalDown") { group = "build" description = "Stop Playground backend and frontend locally" - - doLast { - exec { - executable("docker-compose") - args("-f", "docker-compose.local.yaml", "down") - } - } + workingDir = projectDir + commandLine("docker-compose", "-f", "docker-compose.local.yaml", "down") } diff --git a/playground/frontend/playground_components/build.gradle.kts b/playground/frontend/playground_components/build.gradle.kts index 1af9d6053765..5bbc33f7fd9a 100644 --- a/playground/frontend/playground_components/build.gradle.kts +++ b/playground/frontend/playground_components/build.gradle.kts @@ -16,14 +16,13 @@ * limitations under the License. */ -import java.io.FileOutputStream +import java.io.ByteArrayOutputStream description = "Apache Beam :: Playground :: playground_components Flutter Package" tasks.register("generate") { dependsOn("generateCode") dependsOn("extractBeamSymbols") - group = "build" description = "Generates all generated files." } @@ -31,61 +30,51 @@ tasks.register("generate") { tasks.register("precommit") { dependsOn("analyze") dependsOn("test") - group = "verification" description = "All possible checks before a commit." } -tasks.register("analyze") { +tasks.register("analyze") { dependsOn("generateCode") - group = "verification" description = "Run dart analyzer" - - doLast { - exec { - executable("dart") - args("analyze", ".") - } - } + commandLine("dart", "analyze", ".") } -tasks.register("test") { +tasks.register("test") { dependsOn("generateCode") - group = "verification" description = "Run tests" - - doLast { - exec { - executable("flutter") - args("test") - } - } + commandLine("flutter", "test") } -tasks.register("cleanFlutter") { +tasks.register("cleanFlutter") { group = "build" description = "Remove build artifacts" - - doLast { - exec { - executable("flutter") - args("clean") - } - } + commandLine("flutter", "clean") } -tasks.register("pubGet") { +tasks.register("pubGet") { group = "build" description = "Install dependencies" + commandLine("flutter", "pub", "get") +} - doLast { - exec { - executable("flutter") - args("pub", "get") +val deleteFilesByRegExp: (String) -> Unit = { re -> + val regex = Regex(re) + val matchedFiles = listOf("assets", "lib", "test").flatMap { directory -> + val base = file(directory) + if (base.exists()) { + fileTree(base).files.filter { + regex.matches(it.relativeTo(base).invariantSeparatorsPath) + } + } else { + emptyList() } - } + }.distinct() + + matchedFiles.forEach { println(it) } + delete(matchedFiles) } tasks.register("cleanGenerated") { @@ -94,44 +83,23 @@ tasks.register("cleanGenerated") { doLast { println("Deleting:") - - deleteFilesByRegExp(".*\\.g\\.dart\$") - deleteFilesByRegExp(".*\\.g\\.yaml\$") - deleteFilesByRegExp(".*\\.gen\\.dart\$") - deleteFilesByRegExp(".*\\.mocks\\.dart\$") + deleteFilesByRegExp(".*\\.g\\.dart$") + deleteFilesByRegExp(".*\\.g\\.yaml$") + deleteFilesByRegExp(".*\\.gen\\.dart$") + deleteFilesByRegExp(".*\\.mocks\\.dart$") } } -val deleteFilesByRegExp by extra( - fun(re: String) { - // Prints file names. - exec { - executable("find") - args("assets", "lib", "test", "-regex", re) - } - - // Actually deletes them. - exec { - executable("find") - args("assets", "lib", "test", "-regex", re, "-delete") - } - } -) - -tasks.register("generateCode") { +tasks.register("generateCode") { dependsOn("cleanFlutter") dependsOn("pubGet") mustRunAfter("extractBeamSymbols") - group = "build" description = "Generate code" - - doLast { - exec { - executable("flutter") - args("pub", "run", "build_runner", "build", "--delete-conflicting-outputs") - } - } + commandLine( + "flutter", "pub", "run", "build_runner", "build", + "--delete-conflicting-outputs" + ) } tasks.register("extractBeamSymbols") { @@ -139,32 +107,27 @@ tasks.register("extractBeamSymbols") { dependsOn("extractBeamSymbolsGo") dependsOn("extractBeamSymbolsJava") dependsOn("extractBeamSymbolsPython") - group = "build" description = "Build Beam symbols dictionaries" } -tasks.register("ensureSymbolsDirectoryExists") { - doLast { - exec { - executable("mkdir") - args("-p", "assets/symbols") - } - } +tasks.register("ensureSymbolsDirectoryExists") { + commandLine("mkdir", "-p", "assets/symbols") } -tasks.register("extractBeamSymbolsGo") { +tasks.register("extractBeamSymbolsGo") { + val capturedOutput = ByteArrayOutputStream() + + workingDir("tools/extract_symbols_go") + commandLine( + "go", "run", "extract_symbols_go.go", + "../../../../../sdks/go/pkg/beam" + ) + standardOutput = capturedOutput + doLast { - exec { - workingDir("tools/extract_symbols_go") - executable("go") - args( - "run", - "extract_symbols_go.go", - "../../../../../sdks/go/pkg/beam", - ) - standardOutput = FileOutputStream("playground/frontend/playground_components/assets/symbols/go.g.yaml") - } + layout.projectDirectory.file("assets/symbols/go.g.yaml") + .asFile.writeBytes(capturedOutput.toByteArray()) } } @@ -172,15 +135,18 @@ tasks.register("extractBeamSymbolsJava") { dependsOn("tools:extract_symbols_java:buildJava") } -tasks.register("extractBeamSymbolsPython") { +tasks.register("extractBeamSymbolsPython") { + val capturedOutput = ByteArrayOutputStream() + + commandLine( + "python3", + "tools/extract_symbols_python/extract_symbols_python.py", + "../../../sdks/python/apache_beam" + ) + standardOutput = capturedOutput + doLast { - exec { - executable("python3") - args( - "tools/extract_symbols_python/extract_symbols_python.py", - "../../../sdks/python/apache_beam", - ) - standardOutput = FileOutputStream("playground/frontend/playground_components/assets/symbols/python.g.yaml") - } + layout.projectDirectory.file("assets/symbols/python.g.yaml") + .asFile.writeBytes(capturedOutput.toByteArray()) } } diff --git a/playground/terraform/build.gradle.kts b/playground/terraform/build.gradle.kts index ea10fa0e94ac..2f850cb28688 100644 --- a/playground/terraform/build.gradle.kts +++ b/playground/terraform/build.gradle.kts @@ -199,15 +199,10 @@ tasks.register("InitInfrastructure") { apply.mustRunAfter(init) } -tasks.register("indexcreate") { +tasks.register("indexcreate") { group = "deploy" - val indexpath = "../index.yaml" - doLast { - exec { - executable("gcloud") - args("app", "deploy", indexpath) - } - } + workingDir(project.projectDir) + commandLine("gcloud", "app", "deploy", "../index.yaml") } tasks.register("setPlaygroundStaticIpAddress") { @@ -384,33 +379,39 @@ func_view: ${pgfuncview} } } -task("applyMigrations") { - doLast { - val namespace = if (project.hasProperty("datastore-namespace")) (project.property("datastore-namespace") as String) else "" - val projectId = project.rootProject.extra["playground_gke_project"] - val modulePath = project(":playground").projectDir.absolutePath - val sdkConfig = "$modulePath/sdks.yaml" - exec { - workingDir("$modulePath/backend") - executable("go") - args("run", "cmd/migration_tool/migration_tool.go", - "-project-id", projectId, - "-sdk-config", sdkConfig, - "-namespace", namespace) - } +tasks.register("applyMigrations") { + group = "deploy" + + val modulePath = project(":playground").projectDir.absolutePath + val sdkConfig = "$modulePath/sdks.yaml" + + workingDir("$modulePath/backend") + + val namespace = if (project.hasProperty("datastore-namespace")) { + project.property("datastore-namespace") as String + } else { + "" } + + val projectId = project.rootProject.extra["playground_gke_project"] + + commandLine( + "go", + "run", + "cmd/migration_tool/migration_tool.go", + "-project-id", projectId.toString(), + "-sdk-config", sdkConfig, + "-namespace", namespace + ) } -tasks.register("helmRelease") { +tasks.register("helmRelease") { group = "deploy" + val modulePath = project(":playground").projectDir.absolutePath val helmdir = File("$modulePath/infrastructure/helm-playground/") - doLast{ - exec { - executable("helm") - args("upgrade", "--install", "playground", "$helmdir") - } - } + + commandLine("helm", "upgrade", "--install", "playground", helmdir.absolutePath) } tasks.register("gkebackend") { @@ -435,4 +436,4 @@ tasks.register("gkebackend") { indexcreateTask.mustRunAfter(pushFrontTask) applyMigrations.mustRunAfter(indexcreateTask) helmTask.mustRunAfter(applyMigrations) -} \ No newline at end of file +} diff --git a/runners/flink/job-server/flink_job_server.gradle b/runners/flink/job-server/flink_job_server.gradle index edf55807503f..ff03947f0f47 100644 --- a/runners/flink/job-server/flink_job_server.gradle +++ b/runners/flink/job-server/flink_job_server.gradle @@ -27,7 +27,9 @@ import org.apache.beam.gradle.BeamModulePlugin apply plugin: 'org.apache.beam.module' apply plugin: 'application' // we need to set mainClassName before applying shadow plugin -mainClassName = "org.apache.beam.runners.flink.FlinkJobServerDriver" +application { + mainClass = "org.apache.beam.runners.flink.FlinkJobServerDriver" +} // Resolve the Flink project name (and version) the job-server is based on def flinkRunnerProject = "${project.path.replace(":job-server", "")}" @@ -268,7 +270,7 @@ def artifactPort = BeamModulePlugin.getRandomPort() def setupTask = project.tasks.register("flinkJobServerSetup", Exec) { dependsOn shadowJar def pythonDir = project.project(":sdks:python").projectDir - def flinkJobServerJar = shadowJar.archivePath + def flinkJobServerJar = shadowJar.archiveFile.get().asFile def flinkDir = project.project(":runners:flink").projectDir def additionalArgs = "" @@ -341,11 +343,11 @@ shadowJar { // miniCluster jar starts an embedded Flink cluster intended for use in testing. tasks.register("miniCluster", Jar) { dependsOn shadowJar - archiveBaseName = "${project.archivesBaseName}-mini-cluster" + archiveBaseName = "${project.extensions.getByType(org.gradle.api.plugins.BasePluginExtension).archivesName.get()}-mini-cluster" dependencies { runtimeOnly project(path: flinkRunnerProject, configuration: "miniCluster") } - from zipTree(shadowJar.archivePath).matching { + from zipTree(shadowJar.archiveFile.get().asFile).matching { // If these classes aren't excluded from the mini cluster jar, they will be loaded instead of // the corresponding classes in the submitted job jar, preventing pipeline resources from // loading successfully. diff --git a/runners/google-cloud-dataflow-java/build.gradle b/runners/google-cloud-dataflow-java/build.gradle index d0aa2dfc0b10..ff8fc6b18719 100644 --- a/runners/google-cloud-dataflow-java/build.gradle +++ b/runners/google-cloud-dataflow-java/build.gradle @@ -165,7 +165,7 @@ def dataflowPostCommitTempRoot = project.findProperty('gcpTempRoot') ?: 'gs://te def dataflowPostCommitTempRootKms = project.findProperty('gcpTempRootKms') ?: 'gs://temp-storage-for-end-to-end-tests-cmek' def dataflowUploadTemp = project.findProperty('gcpTempRoot') ?: 'gs://temp-storage-for-upload-tests' def testFilesToStage = project.findProperty('filesToStage') ?: 'test.txt' -def dataflowLegacyWorkerJar = project.findProperty('dataflowWorkerJar') ?: project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath +def dataflowLegacyWorkerJar = project.findProperty('dataflowWorkerJar') ?: project(":runners:google-cloud-dataflow-java:worker").shadowJar.archiveFile.get().asFile def dataflowKmsKey = project.findProperty('dataflowKmsKey') ?: "projects/apache-beam-testing/locations/global/keyRings/beam-it/cryptoKeys/test" def firestoreDb = project.findProperty('firestoreDb') ?: 'firestoredb' diff --git a/runners/google-cloud-dataflow-java/examples-streaming/build.gradle b/runners/google-cloud-dataflow-java/examples-streaming/build.gradle index 6c009b3441af..2eb70ba8a19b 100644 --- a/runners/google-cloud-dataflow-java/examples-streaming/build.gradle +++ b/runners/google-cloud-dataflow-java/examples-streaming/build.gradle @@ -37,7 +37,7 @@ def gcsTempRoot = project.findProperty('gcsTempRoot') ?: 'gs://temp-storage-for- task windmillPreCommit(type: Test) { dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar" - def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?: project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath + def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?: project(":runners:google-cloud-dataflow-java:worker").shadowJar.archiveFile.get().asFile // Set workerHarnessContainerImage to empty to make Dataflow pick up the // non-versioned container image, which handles a staged worker jar. @@ -60,7 +60,7 @@ task windmillPreCommit(type: Test) { task appliancePreCommit(type: Test) { dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar" - def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?: project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath + def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?: project(":runners:google-cloud-dataflow-java:worker").shadowJar.archiveFile.get().asFile // Set workerHarnessContainerImage to empty to make Dataflow pick up the non-versioned container // image, which handles a staged worker jar. diff --git a/runners/google-cloud-dataflow-java/examples/build.gradle b/runners/google-cloud-dataflow-java/examples/build.gradle index 1b170de56750..85cc62cc3e3d 100644 --- a/runners/google-cloud-dataflow-java/examples/build.gradle +++ b/runners/google-cloud-dataflow-java/examples/build.gradle @@ -107,13 +107,13 @@ def commonConfig = { Map args -> tasks.register('preCommitLegacyWorker', Test) { dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar" - def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?: project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath + def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?: project(":runners:google-cloud-dataflow-java:worker").shadowJar.archiveFile.get().asFile with commonConfig(dataflowWorkerJar: dataflowWorkerJar) } tasks.register('preCommitLegacyWorkerImpersonate', Test) { dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar" - def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?: project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath + def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?: project(":runners:google-cloud-dataflow-java:worker").shadowJar.archiveFile.get().asFile with commonConfig( dataflowWorkerJar: dataflowWorkerJar, gcsTempRoot: impersonationTempRoot, @@ -126,7 +126,7 @@ tasks.register('preCommitLegacyWorkerImpersonate', Test) { tasks.register('verifyFnApiWorker', Test) { dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar" dependsOn ":runners:google-cloud-dataflow-java:buildAndPushDockerContainer" - def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?: project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath + def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?: project(":runners:google-cloud-dataflow-java:worker").shadowJar.archiveFile.get().asFile with commonConfig( dataflowWorkerJar: dataflowWorkerJar, workerHarnessContainerImage: dockerJavaImageName, @@ -139,7 +139,7 @@ tasks.register('verifyFnApiWorker', Test) { tasks.register('postCommitLegacyWorker', Test) { dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar" - def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?: project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath + def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?: project(":runners:google-cloud-dataflow-java:worker").shadowJar.archiveFile.get().asFile with commonConfig(dataflowWorkerJar: dataflowWorkerJar, runWordCount: 'exclude') } @@ -149,7 +149,7 @@ tasks.register('javaPostCommit') { tasks.register('postCommitLegacyWorkerJava8', Test) { dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar" - def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?: project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath + def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?: project(":runners:google-cloud-dataflow-java:worker").shadowJar.archiveFile.get().asFile systemProperty "java.specification.version", "8" with commonConfig(dataflowWorkerJar: dataflowWorkerJar, runWordCount: 'only') } @@ -160,7 +160,7 @@ tasks.register('java8PostCommit') { tasks.register('postCommitLegacyWorkerJava17', Test) { dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar" - def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?: project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath + def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?: project(":runners:google-cloud-dataflow-java:worker").shadowJar.archiveFile.get().asFile systemProperty "java.specification.version", "17" with commonConfig(dataflowWorkerJar: dataflowWorkerJar, runWordCount: 'only') } @@ -171,7 +171,7 @@ tasks.register('java17PostCommit') { tasks.register('postCommitLegacyWorkerJava21', Test) { dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar" - def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?: project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath + def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?: project(":runners:google-cloud-dataflow-java:worker").shadowJar.archiveFile.get().asFile systemProperty "java.specification.version", "21" with commonConfig(dataflowWorkerJar: dataflowWorkerJar, runWordCount: 'exclude') } @@ -182,7 +182,7 @@ tasks.register('java21PostCommit') { tasks.register('postCommitLegacyWorkerJava25', Test) { dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar" - def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?: project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath + def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?: project(":runners:google-cloud-dataflow-java:worker").shadowJar.archiveFile.get().asFile systemProperty "java.specification.version", "25" with commonConfig(dataflowWorkerJar: dataflowWorkerJar, runWordCount: 'exclude') } diff --git a/runners/google-cloud-dataflow-java/worker/build.gradle b/runners/google-cloud-dataflow-java/worker/build.gradle index 232fb683866b..dc3b34f9746f 100644 --- a/runners/google-cloud-dataflow-java/worker/build.gradle +++ b/runners/google-cloud-dataflow-java/worker/build.gradle @@ -256,7 +256,7 @@ dependencies { } project.task('validateShadedJarContainsSlf4jJdk14', dependsOn: 'shadowJar') { - ext.outFile = project.file("${project.reportsDir}/${name}.out") + ext.outFile = project.layout.buildDirectory.file("reports/${name}.out").get().asFile inputs.files(project.configurations.shadow.artifacts.files) .withPropertyName("shadowArtifactsFiles") .withPathSensitivity(PathSensitivity.RELATIVE) diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/streaming/ComputationState.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/streaming/ComputationState.java index e2333e23a716..83bb51d1745b 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/streaming/ComputationState.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/streaming/ComputationState.java @@ -106,7 +106,7 @@ public Optional acquireComputationWorkExecutor() { public boolean activateWork(ExecutableWork executableWork) { switch (activeWorkState.activateWorkForKey(executableWork)) { case DUPLICATE: - // Fall through intentionally. Work was not and will not be activated in these cases. + // Fall through intentionally. Work was not and will not be activated in these cases. case STALE: return false; case QUEUED: diff --git a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/artifact/ArtifactStagingService.java b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/artifact/ArtifactStagingService.java index 512d435db327..2a2718f57c46 100644 --- a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/artifact/ArtifactStagingService.java +++ b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/artifact/ArtifactStagingService.java @@ -418,7 +418,7 @@ public synchronized void onNext(ArtifactApi.ArtifactResponseWrapper responseWrap responseObserver.onError(exn); } state = State.GETCHUNK; - // fall through + // fall through case GETCHUNK: try { diff --git a/runners/kafka-streams/job-server/build.gradle b/runners/kafka-streams/job-server/build.gradle index aef2f271b7b1..6e0e8ac3fee2 100644 --- a/runners/kafka-streams/job-server/build.gradle +++ b/runners/kafka-streams/job-server/build.gradle @@ -27,7 +27,9 @@ apply plugin: 'org.apache.beam.module' apply plugin: 'application' // Must be set before the shadow plugin is applied. -mainClassName = "org.apache.beam.runners.kafka.streams.KafkaStreamsJobServerDriver" +application { + mainClass = "org.apache.beam.runners.kafka.streams.KafkaStreamsJobServerDriver" +} applyJavaNature( automaticModuleName: 'org.apache.beam.runners.kafka.streams.jobserver', diff --git a/runners/kafka-streams/measurement/build.gradle b/runners/kafka-streams/measurement/build.gradle index 5e7992a0916e..21bd5b615386 100644 --- a/runners/kafka-streams/measurement/build.gradle +++ b/runners/kafka-streams/measurement/build.gradle @@ -25,7 +25,9 @@ apply plugin: 'org.apache.beam.module' apply plugin: 'application' -mainClassName = "org.apache.beam.runners.kafka.streams.measurement.RescalingMeasurement" +application { + mainClass = "org.apache.beam.runners.kafka.streams.measurement.RescalingMeasurement" +} applyJavaNature( automaticModuleName: 'org.apache.beam.runners.kafka.streams.measurement', diff --git a/runners/spark/job-server/spark_job_server.gradle b/runners/spark/job-server/spark_job_server.gradle index 7e5e65aa4c40..864d984acc87 100644 --- a/runners/spark/job-server/spark_job_server.gradle +++ b/runners/spark/job-server/spark_job_server.gradle @@ -26,7 +26,9 @@ import org.apache.beam.gradle.BeamModulePlugin apply plugin: 'org.apache.beam.module' apply plugin: 'application' // we need to set mainClassName before applying shadow plugin -mainClassName = "org.apache.beam.runners.spark.SparkJobServerDriver" +application { + mainClass = "org.apache.beam.runners.spark.SparkJobServerDriver" +} def sparkVersion = project.findProperty('spark_version') ?: '' @@ -304,7 +306,7 @@ def sparkJobServerJvmArgs() { def setupTask = project.tasks.register("sparkJobServerSetup", Exec) { dependsOn shadowJar def pythonDir = project.project(":sdks:python").projectDir - def sparkJobServerJar = shadowJar.archivePath + def sparkJobServerJar = shadowJar.archiveFile.get().asFile def jvmArgs = sparkJobServerJvmArgs().join(' ') def jvmArgsOpt = jvmArgs ? "--jvm_args \"${jvmArgs}\"" : "" diff --git a/runners/spark/spark_runner.gradle b/runners/spark/spark_runner.gradle index c40eb788b61c..e529930eaf8c 100644 --- a/runners/spark/spark_runner.gradle +++ b/runners/spark/spark_runner.gradle @@ -306,7 +306,7 @@ dependencies { validatesRunner project(path: ":sdks:java:core", configuration: "shadowTest") validatesRunner project(path: ":runners:core-java", configuration: "testRuntimeMigration") validatesRunner project(":sdks:java:io:hadoop-format") - validatesRunner project(":sdks:java:io:hadoop-format").sourceSets.test.output + validatesRunner files(rootProject.project(":sdks:java:io:hadoop-format").extensions.getByType(org.gradle.api.tasks.SourceSetContainer).getByName("test").output) validatesRunner project(path: ":examples:java", configuration: "testRuntimeMigration") validatesRunner project(path: project.path, configuration: "testRuntimeMigration") hadoopVersions.each { kv -> diff --git a/sdks/go/test/build.gradle b/sdks/go/test/build.gradle index 8fcb09166146..564c9144c251 100644 --- a/sdks/go/test/build.gradle +++ b/sdks/go/test/build.gradle @@ -101,11 +101,11 @@ task flinkValidatesRunner { } doLast { def pipelineOptions = [ // Pipeline options piped directly to Go SDK flags. - "--expansion_jar=test:${project(":sdks:java:testing:expansion-service").buildTestExpansionServiceJar.archivePath}", + "--expansion_jar=test:${project(":sdks:java:testing:expansion-service").tasks.named("buildTestExpansionServiceJar", com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar).get().archiveFile.get().asFile}", ] def options = [ "--runner flink", - "--flink_job_server_jar ${project(":runners:flink:${flinkVersion}:job-server").shadowJar.archivePath}", + "--flink_job_server_jar ${project(":runners:flink:${flinkVersion}:job-server").shadowJar.archiveFile.get().asFile}", "--pipeline_opts \"${pipelineOptions.join(' ')}\"", "--prebuild_go_docker_tag ${project(":sdks:go:container").containerImageTags().first()}", ] @@ -131,11 +131,11 @@ task sparkValidatesRunner { dependsOn ":sdks:java:testing:expansion-service:buildTestExpansionServiceJar" doLast { def pipelineOptions = [ // Pipeline options piped directly to Go SDK flags. - "--expansion_jar=test:${project(":sdks:java:testing:expansion-service").buildTestExpansionServiceJar.archivePath}", + "--expansion_jar=test:${project(":sdks:java:testing:expansion-service").tasks.named("buildTestExpansionServiceJar", com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar).get().archiveFile.get().asFile}", ] def options = [ "--runner spark", - "--spark_job_server_jar ${project(":runners:spark:3:job-server").shadowJar.archivePath}", + "--spark_job_server_jar ${project(":runners:spark:3:job-server").shadowJar.archiveFile.get().asFile}", "--pipeline_opts \"${pipelineOptions.join(' ')}\"", "--prebuild_go_docker_tag ${project(":sdks:go:container").containerImageTags().first()}", ] @@ -167,7 +167,7 @@ tasks.register("ulrValidatesRunner") { dependsOn ":sdks:java:testing:expansion-service:buildTestExpansionServiceJar" doLast { def pipelineOptions = [ // Pipeline options piped directly to Go SDK flags. - "--expansion_jar=test:${project(":sdks:java:testing:expansion-service").buildTestExpansionServiceJar.archivePath}", + "--expansion_jar=test:${project(":sdks:java:testing:expansion-service").tasks.named("buildTestExpansionServiceJar", com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar).get().archiveFile.get().asFile}", ] def options = [ "--runner portable", @@ -200,7 +200,7 @@ task prismValidatesRunner { dependsOn ":sdks:java:testing:expansion-service:buildTestExpansionServiceJar" doLast { def pipelineOptions = [ // Pipeline options piped directly to Go SDK flags. - "--expansion_jar=test:${project(":sdks:java:testing:expansion-service").buildTestExpansionServiceJar.archivePath}", + "--expansion_jar=test:${project(":sdks:java:testing:expansion-service").tasks.named("buildTestExpansionServiceJar", com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar).get().archiveFile.get().asFile}", ] def options = [ "--runner prism", @@ -232,11 +232,11 @@ ext.goIoValidatesRunnerTask = { proj, name, scriptOpts, pipelineOpts -> dependsOn ":sdks:java:testing:kafka-service:buildTestKafkaServiceJar" doLast { - def ioExpJar = project(":sdks:java:io:expansion-service").shadowJar.archivePath - def schemaIoExpJar = project(":sdks:java:extensions:schemaio-expansion-service").shadowJar.archivePath - def debeziumIoExpJar = project(":sdks:java:io:debezium:expansion-service").shadowJar.archivePath - def gcpIoExpJar = project(":sdks:java:io:google-cloud-platform:expansion-service").shadowJar.archivePath - def kafkaJar = project(":sdks:java:testing:kafka-service:").buildTestKafkaServiceJar.archivePath + def ioExpJar = project(":sdks:java:io:expansion-service").shadowJar.archiveFile.get().asFile + def schemaIoExpJar = project(":sdks:java:extensions:schemaio-expansion-service").shadowJar.archiveFile.get().asFile + def debeziumIoExpJar = project(":sdks:java:io:debezium:expansion-service").shadowJar.archiveFile.get().asFile + def gcpIoExpJar = project(":sdks:java:io:google-cloud-platform:expansion-service").shadowJar.archiveFile.get().asFile + def kafkaJar = project(":sdks:java:testing:kafka-service:").buildTestKafkaServiceJar.archiveFile.get().asFile def pipelineOptions = [ // Pipeline options piped directly to Go SDK flags. "--kafka_jar=${kafkaJar}", "--expansion_jar=io:${ioExpJar}", diff --git a/sdks/java/bom/build.gradle b/sdks/java/bom/build.gradle index da2434684205..cb2ed4b30c2d 100644 --- a/sdks/java/bom/build.gradle +++ b/sdks/java/bom/build.gradle @@ -49,7 +49,7 @@ for (p in rootProject.subprojects) { def subproject = p // project(':' + p.name) if (subproject.ext.properties.containsKey('includeInJavaBom') && subproject.ext.properties.includeInJavaBom) { - bomModuleNames.add(subproject.archivesBaseName) + bomModuleNames.add(subproject.extensions.getByType(org.gradle.api.plugins.BasePluginExtension).archivesName.get()) } } @@ -122,7 +122,7 @@ afterEvaluate { publications { mavenJava(MavenPublication) { groupId = project.mavenGroupId - artifactId = archivesBaseName + artifactId = project.extensions.getByType(org.gradle.api.plugins.BasePluginExtension).archivesName.get() version = project.version // Ensures the published components are included in the maven metadata. diff --git a/sdks/java/bom/common.gradle b/sdks/java/bom/common.gradle index 74e5d423f809..6b076380c51a 100644 --- a/sdks/java/bom/common.gradle +++ b/sdks/java/bom/common.gradle @@ -35,7 +35,7 @@ publishing { publications { mavenJava(MavenPublication) { groupId = project.mavenGroupId - artifactId = archivesBaseName + artifactId = project.extensions.getByType(org.gradle.api.plugins.BasePluginExtension).archivesName.get() version = project.version pom { name = project.description diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/WriteFiles.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/WriteFiles.java index 2fd77a1ad0c6..29898391819c 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/WriteFiles.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/WriteFiles.java @@ -186,8 +186,8 @@ public static WriteFiles getSink(); - public abstract @Nullable - PTransform, PCollectionView> getComputeNumShards(); + public abstract @Nullable PTransform, PCollectionView> + getComputeNumShards(); // We don't use a side input for static sharding, as we want this value to be updatable // when a pipeline is updated. diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/SchemaTranslation.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/SchemaTranslation.java index 8122704e6436..a79d43fdd86b 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/SchemaTranslation.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/SchemaTranslation.java @@ -247,8 +247,8 @@ static SchemaApi.FieldType fieldTypeToProto( } builder.setLogicalType(logicalTypeBuilder.build()); break; - // Special-case for DATETIME and DECIMAL which are logical types in portable representation, - // but not yet in Java. (https://github.com/apache/beam/issues/19817) + // Special-case for DATETIME and DECIMAL which are logical types in portable representation, + // but not yet in Java. (https://github.com/apache/beam/issues/19817) case DATETIME: builder.setLogicalType( SchemaApi.LogicalType.newBuilder() diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/SchemaUtils.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/SchemaUtils.java index d45454be90fa..08264309267e 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/SchemaUtils.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/SchemaUtils.java @@ -102,7 +102,7 @@ static FieldType widenNullableTypes(FieldType fieldType1, FieldType fieldType2) + ".v.s" + fieldType2.getLogicalType().getIdentifier()); } - // fall through + // fall through default: result = fieldType1; } diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/utils/JsonUtils.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/utils/JsonUtils.java index 18f5813c6cc9..a08014a63fd1 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/utils/JsonUtils.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/utils/JsonUtils.java @@ -186,7 +186,7 @@ private static org.everit.json.schema.Schema jsonPropertyFromBeamType(FieldType propertySchema = jsonSchemaBuilderFromBeamSchema(rowSchema); break; - // add more Beam to JSON types + // add more Beam to JSON types default: throw new IllegalArgumentException("Unsupported Beam to JSON type: " + beamType); } diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/ByteBuddyDoFnInvokerFactory.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/ByteBuddyDoFnInvokerFactory.java index 9d307a4e0a98..e4bc3814dde2 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/ByteBuddyDoFnInvokerFactory.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/ByteBuddyDoFnInvokerFactory.java @@ -507,8 +507,8 @@ Coder invokeGetWatermarkEstimatorStateCoder( public static class DefaultGetInitialWatermarkEstimatorState { /** The default watermark estimator state is {@code null}. */ @SuppressWarnings("unused") - public static @Nullable - WatermarkEstimator invokeNewWatermarkEstimator( + public static + @Nullable WatermarkEstimator invokeNewWatermarkEstimator( DoFnInvoker.ArgumentProvider argumentProvider) { return null; } diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignature.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignature.java index 47c3e59eca0d..4df0c34999fc 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignature.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignature.java @@ -137,8 +137,8 @@ public abstract class DoFnSignature { /** Details about this {@link DoFn}'s {@link DoFn.GetInitialWatermarkEstimatorState} method. */ @Pure - public abstract @Nullable - GetInitialWatermarkEstimatorStateMethod getInitialWatermarkEstimatorState(); + public abstract @Nullable GetInitialWatermarkEstimatorStateMethod + getInitialWatermarkEstimatorState(); /** Details about this {@link DoFn}'s {@link DoFn.NewWatermarkEstimator} method. */ @Pure diff --git a/sdks/java/expansion-service/app/build.gradle b/sdks/java/expansion-service/app/build.gradle index 36fa0891dd34..add512602056 100644 --- a/sdks/java/expansion-service/app/build.gradle +++ b/sdks/java/expansion-service/app/build.gradle @@ -23,7 +23,9 @@ applyJavaNature( validateShadowJar: false, shadowClosure: {}, ) -mainClassName = "org.apache.beam.sdk.expansion.service.ExpansionService" +application { + mainClass = "org.apache.beam.sdk.expansion.service.ExpansionService" +} description = "Apache Beam :: SDKs :: Java :: Expansion Service :: App" diff --git a/sdks/java/extensions/avro/build.gradle b/sdks/java/extensions/avro/build.gradle index 6c21dbb7c08b..faeee1f28984 100644 --- a/sdks/java/extensions/avro/build.gradle +++ b/sdks/java/extensions/avro/build.gradle @@ -151,7 +151,7 @@ avroVersions.each { k, v -> group = "build" description = "Generate Avro classes for Avro version $v" classpath = configurations."avroVersion${k}Generate" - main = "org.apache.avro.tool.Main" + mainClass = "org.apache.avro.tool.Main" args = [ "compile", "-bigDecimal", // Use BigDecimal for logical type decimal, similarly to what gradle-avro-plugin does diff --git a/sdks/java/extensions/avro/src/main/java/org/apache/beam/sdk/extensions/avro/io/AvroIO.java b/sdks/java/extensions/avro/src/main/java/org/apache/beam/sdk/extensions/avro/io/AvroIO.java index fcb47ba74aba..632e8dbe9198 100644 --- a/sdks/java/extensions/avro/src/main/java/org/apache/beam/sdk/extensions/avro/io/AvroIO.java +++ b/sdks/java/extensions/avro/src/main/java/org/apache/beam/sdk/extensions/avro/io/AvroIO.java @@ -1443,8 +1443,8 @@ public abstract static class TypedWrite abstract @Nullable FilenamePolicy getFilenamePolicy(); - abstract @Nullable - DynamicAvroDestinations getDynamicDestinations(); + abstract @Nullable DynamicAvroDestinations + getDynamicDestinations(); abstract AvroSink.@Nullable DatumWriterFactory getDatumWriterFactory(); diff --git a/sdks/java/extensions/python/build.gradle b/sdks/java/extensions/python/build.gradle index 741508c4ae6f..694bd81b803d 100644 --- a/sdks/java/extensions/python/build.gradle +++ b/sdks/java/extensions/python/build.gradle @@ -32,7 +32,7 @@ dependencies { implementation project(path: ":sdks:java:transform-service:launcher") testImplementation library.java.junit testImplementation library.java.hamcrest - testImplementation project(":sdks:java:core").sourceSets.test.output + testImplementation files(rootProject.project(":sdks:java:core").sourceSets.test.output) testImplementation project(path: ":runners:direct-java", configuration: "shadowTest") testRuntimeOnly library.java.slf4j_simple } diff --git a/sdks/java/extensions/sql/build.gradle b/sdks/java/extensions/sql/build.gradle index 49633a0a47a1..5b984cca98c0 100644 --- a/sdks/java/extensions/sql/build.gradle +++ b/sdks/java/extensions/sql/build.gradle @@ -265,7 +265,7 @@ task postCommit { } task emptyJar(type: Jar) { - archiveBaseName = "${project.archivesBaseName}-empty-jar" + archiveBaseName = "${project.extensions.getByType(org.gradle.api.plugins.BasePluginExtension).archivesName.get()}-empty-jar" from fileTree(dir: getTemporaryDir().createNewFile().toString()) } @@ -286,7 +286,7 @@ hadoopVersions.each { kv -> // Pass jars used by Java UDF tests via system properties. evaluationDependsOn(":sdks:java:extensions:sql:udf-test-provider") // Needed to resolve jarPath. systemProperty "beam.sql.udf.test.jar_path", project(":sdks:java:extensions:sql:udf-test-provider").jarPath - systemProperty "beam.sql.udf.test.empty_jar_path", emptyJar.archivePath + systemProperty "beam.sql.udf.test.empty_jar_path", emptyJar.archiveFile.get().asFile } } @@ -295,5 +295,5 @@ test { // Pass jars used by Java UDF tests via system properties. evaluationDependsOn(":sdks:java:extensions:sql:udf-test-provider") // Needed to resolve jarPath. systemProperty "beam.sql.udf.test.jar_path", project(":sdks:java:extensions:sql:udf-test-provider").jarPath - systemProperty "beam.sql.udf.test.empty_jar_path", emptyJar.archivePath + systemProperty "beam.sql.udf.test.empty_jar_path", emptyJar.archiveFile.get().asFile } diff --git a/sdks/java/extensions/sql/hcatalog/build.gradle b/sdks/java/extensions/sql/hcatalog/build.gradle index 3fe36b7bb81a..494ec6c7142a 100644 --- a/sdks/java/extensions/sql/hcatalog/build.gradle +++ b/sdks/java/extensions/sql/hcatalog/build.gradle @@ -43,7 +43,12 @@ dependencies { implementation project(":sdks:java:core") implementation library.java.vendored_guava_32_1_2_jre testImplementation library.java.junit - testImplementation project(":sdks:java:io:hcatalog").sourceSets.test.output + testImplementation files( + rootProject.project(":sdks:java:io:hcatalog") + .extensions + .getByType(org.gradle.api.tasks.SourceSetContainer) + .getByName("test").output + ) // Needed for HCatalogTableProvider tests, // they use HCat* types testImplementation "io.netty:netty-all:$netty_version" diff --git a/sdks/java/extensions/sql/jdbc/build.gradle b/sdks/java/extensions/sql/jdbc/build.gradle index 334da3293084..1bb32b649702 100644 --- a/sdks/java/extensions/sql/jdbc/build.gradle +++ b/sdks/java/extensions/sql/jdbc/build.gradle @@ -148,6 +148,6 @@ processResources { * `-Pbeam.sql.shell.bundled="sdks:java:extensions:sql:hcatalog"` */ run { - mainClassName = "org.apache.beam.sdk.extensions.sql.jdbc.BeamSqlLine" + mainClass = "org.apache.beam.sdk.extensions.sql.jdbc.BeamSqlLine" standardInput System.in } diff --git a/sdks/java/extensions/sql/udf-test-provider/build.gradle b/sdks/java/extensions/sql/udf-test-provider/build.gradle index 7531e48d34b5..7132a6e71957 100644 --- a/sdks/java/extensions/sql/udf-test-provider/build.gradle +++ b/sdks/java/extensions/sql/udf-test-provider/build.gradle @@ -28,7 +28,7 @@ applyJavaNature( description = "Apache Beam :: SDKs :: Java :: Extensions :: SQL :: UDF test provider" ext.summary = "Java UDFs for testing. This project must be built separately from its parent so the UDF provider is not included in the context classloader for tests." -project.ext.jarPath = jar.archivePath +project.ext.jarPath = jar.archiveFile.get().asFile dependencies { // No dependency (direct or transitive) on :sdks:java:core. diff --git a/sdks/java/extensions/yaml/build.gradle b/sdks/java/extensions/yaml/build.gradle index 5b86d3239226..614341293e93 100644 --- a/sdks/java/extensions/yaml/build.gradle +++ b/sdks/java/extensions/yaml/build.gradle @@ -29,7 +29,7 @@ dependencies { implementation project(path: ":sdks:java:extensions:python") testImplementation library.java.junit testImplementation library.java.hamcrest - testImplementation project(":sdks:java:core").sourceSets.test.output + testImplementation files(rootProject.project(":sdks:java:core").sourceSets.test.output) testImplementation project(path: ":runners:portability:java") testRuntimeOnly library.java.slf4j_simple } diff --git a/sdks/java/extensions/zetasketch/build.gradle b/sdks/java/extensions/zetasketch/build.gradle index bb532ad08aa1..2cab7cfc5c6a 100644 --- a/sdks/java/extensions/zetasketch/build.gradle +++ b/sdks/java/extensions/zetasketch/build.gradle @@ -55,7 +55,7 @@ task integrationTest(type: Test) { group = "Verification" dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar" - def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?: project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath + def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?: project(":runners:google-cloud-dataflow-java:worker").shadowJar.archiveFile.get().asFile def gcpProject = project.findProperty('gcpProject') ?: 'apache-beam-testing' def gcpRegion = project.findProperty('gcpRegion') ?: 'us-central1' diff --git a/sdks/java/harness/jmh/build.gradle b/sdks/java/harness/jmh/build.gradle index a3f795b7bc20..ccba182da827 100644 --- a/sdks/java/harness/jmh/build.gradle +++ b/sdks/java/harness/jmh/build.gradle @@ -47,19 +47,19 @@ dependencies { } jmh { - configurations.jammAgent.resolvedConfiguration.files.each { + configurations.jammAgent.files.each { jvmArgs '-javaagent:' + it } } jmhTest { - configurations.jammAgent.resolvedConfiguration.files.each { + configurations.jammAgent.files.each { jvmArgs '-javaagent:' + it } } test { - configurations.jammAgent.resolvedConfiguration.files.each { + configurations.jammAgent.files.each { jvmArgs '-javaagent:' + it } } diff --git a/sdks/java/io/amazon-web-services2/expansion-service/build.gradle b/sdks/java/io/amazon-web-services2/expansion-service/build.gradle index fd712737f53c..8d7905bdec47 100644 --- a/sdks/java/io/amazon-web-services2/expansion-service/build.gradle +++ b/sdks/java/io/amazon-web-services2/expansion-service/build.gradle @@ -18,7 +18,9 @@ apply plugin: 'org.apache.beam.module' apply plugin: 'application' -mainClassName = "org.apache.beam.sdk.expansion.service.ExpansionService" +application { + mainClass = "org.apache.beam.sdk.expansion.service.ExpansionService" +} applyJavaNature( automaticModuleName: 'org.apache.beam.sdk.io.amazon-web-services2.expansion.service', diff --git a/sdks/java/io/clickhouse/build.gradle b/sdks/java/io/clickhouse/build.gradle index 08eaa69f9113..a7e55f537544 100644 --- a/sdks/java/io/clickhouse/build.gradle +++ b/sdks/java/io/clickhouse/build.gradle @@ -73,5 +73,5 @@ dependencies { processTestResources { // BEAM-12390: config.d/zookeeper_default.xml should have xx4 permission. - fileMode 0644 + filePermissions { unix(0644) } } diff --git a/sdks/java/io/debezium/expansion-service/build.gradle b/sdks/java/io/debezium/expansion-service/build.gradle index fe820b62c138..61036d4eb1d2 100644 --- a/sdks/java/io/debezium/expansion-service/build.gradle +++ b/sdks/java/io/debezium/expansion-service/build.gradle @@ -18,7 +18,9 @@ apply plugin: 'org.apache.beam.module' apply plugin: 'application' -mainClassName = "org.apache.beam.sdk.expansion.service.ExpansionService" +application { + mainClass = "org.apache.beam.sdk.expansion.service.ExpansionService" +} applyJavaNature( automaticModuleName: 'org.apache.beam.sdk.io.debezium.expansion.service', diff --git a/sdks/java/io/expansion-service/build.gradle b/sdks/java/io/expansion-service/build.gradle index 28b4060a2b58..1633045cc214 100644 --- a/sdks/java/io/expansion-service/build.gradle +++ b/sdks/java/io/expansion-service/build.gradle @@ -18,7 +18,9 @@ apply plugin: 'org.apache.beam.module' apply plugin: 'application' -mainClassName = "org.apache.beam.sdk.expansion.service.ExpansionService" +application { + mainClass = "org.apache.beam.sdk.expansion.service.ExpansionService" +} applyJavaNature( automaticModuleName: 'org.apache.beam.sdk.io.expansion.service', diff --git a/sdks/java/io/google-cloud-platform/expansion-service/build.gradle b/sdks/java/io/google-cloud-platform/expansion-service/build.gradle index 824a2659b84b..eddde8fd1a16 100644 --- a/sdks/java/io/google-cloud-platform/expansion-service/build.gradle +++ b/sdks/java/io/google-cloud-platform/expansion-service/build.gradle @@ -17,7 +17,9 @@ */ apply plugin: 'org.apache.beam.module' apply plugin: 'application' -mainClassName = "org.apache.beam.sdk.expansion.service.ExpansionService" +application { + mainClass = "org.apache.beam.sdk.expansion.service.ExpansionService" +} applyJavaNature( automaticModuleName: 'org.apache.beam.sdk.io.gcp.expansion.service', diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java index 07fa1a320e92..facd028317a8 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java @@ -1512,8 +1512,8 @@ abstract Builder setBadRecordErrorHandler( abstract @Nullable SerializableFunction getParseFn(); - abstract @Nullable - SerializableFunction> getDatumReaderFactory(); + abstract @Nullable SerializableFunction> + getDatumReaderFactory(); abstract @Nullable QueryPriority getQueryPriority(); @@ -2964,8 +2964,8 @@ public enum Method { abstract @Nullable ValueProvider getJsonTableRef(); - abstract @Nullable - SerializableFunction<@Nullable ValueInSingleWindow, TableDestination> getTableFunction(); + abstract @Nullable SerializableFunction<@Nullable ValueInSingleWindow, TableDestination> + getTableFunction(); abstract @Nullable TableRowFormatFunction getFormatFunction(); @@ -2973,8 +2973,8 @@ public enum Method { abstract RowWriterFactory.@Nullable AvroRowWriterFactory getAvroRowWriterFactory(); - abstract @Nullable - SerializableFunction<@Nullable TableSchema, org.apache.avro.Schema> getAvroSchemaFactory(); + abstract @Nullable SerializableFunction<@Nullable TableSchema, org.apache.avro.Schema> + getAvroSchemaFactory(); abstract boolean getUseAvroLogicalTypes(); @@ -3066,8 +3066,8 @@ public enum Method { abstract @Nullable String getWriteTempDataset(); - abstract @Nullable - SerializableFunction getRowMutationInformationFn(); + abstract @Nullable SerializableFunction + getRowMutationInformationFn(); abstract ErrorHandler getBadRecordErrorHandler(); diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiWritesShardedRecords.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiWritesShardedRecords.java index 336311e3421e..af0f0623ee8a 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiWritesShardedRecords.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiWritesShardedRecords.java @@ -293,8 +293,8 @@ public String toString() { @AutoValue abstract static class CreateRetryManagerResult { - abstract @Nullable - RetryManager> getRetryManager(); + abstract @Nullable RetryManager> + getRetryManager(); abstract boolean getSchemaMismatchSeen(); diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowToStorageApiProto.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowToStorageApiProto.java index d06c3d764c4f..29c8f007f0cd 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowToStorageApiProto.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowToStorageApiProto.java @@ -1963,8 +1963,8 @@ public static Object jsonValueFromMessageValue( return fieldValue.toString(); case JSON: case GEOGRAPHY: - // The above types have native representations in JSON for all their - // possible values. + // The above types have native representations in JSON for all their + // possible values. case STRING: return fieldValue.toString(); case INT64: diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableConfig.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableConfig.java index 6a35ae1507c9..3f65600d017e 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableConfig.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableConfig.java @@ -62,9 +62,8 @@ public abstract class BigtableConfig implements Serializable { abstract @Nullable BigtableOptions getBigtableOptions(); /** Configurator of the effective Bigtable Options. */ - abstract @Nullable - SerializableFunction - getBigtableOptionsConfigurator(); + abstract @Nullable SerializableFunction + getBigtableOptionsConfigurator(); /** Weather validate that table exists before writing. */ abstract boolean getValidate(); diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/MutationUtils.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/MutationUtils.java index f30407a86dcc..0c1fdf99eda8 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/MutationUtils.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/MutationUtils.java @@ -153,7 +153,7 @@ private static void setBeamValueToKey( case DECIMAL: keyBuilder.append(row.getDecimal(columnName)); break; - // TODO: Implement logical date and datetime + // TODO: Implement logical date and datetime case LOGICAL_TYPE: Schema.LogicalType logicalType = checkNotNull(field.getLogicalType()); String identifier = logicalType.getIdentifier(); diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/StructUtils.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/StructUtils.java index ac8f4becbd0c..c95956a030d3 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/StructUtils.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/StructUtils.java @@ -177,7 +177,7 @@ public static Struct beamRowToStruct(Row row) { structBuilder.set(column).to(decimal); } break; - // TODO: implement logical type date and timestamp + // TODO: implement logical type date and timestamp case DATETIME: @Nullable ReadableDateTime dateTime = row.getDateTime(column); if (dateTime == null) { @@ -259,7 +259,7 @@ private static Type simpleBeamTypeToSpannerType(Schema.FieldType beamType) { return Type.string(); case BOOLEAN: return Type.bool(); - // TODO: implement logical type date and timestamp + // TODO: implement logical type date and timestamp case DATETIME: return Type.timestamp(); default: @@ -332,7 +332,7 @@ private static void addIterableToStructBuilder( case STRING: structBuilder.set(column).toStringArray((Iterable) ((Object) iterable)); break; - // TODO: implement logical date and datetime + // TODO: implement logical date and datetime case DATETIME: if (iterable == null) { structBuilder.set(column).toTimestampArray(null); @@ -369,7 +369,7 @@ private static java.time.Instant fromSpannerTimestamp(Timestamp spannerTimestamp return struct.getBoolean(column); case BYTES: return struct.getBytes(column).toByteArray(); - // TODO: implement logical datetime + // TODO: implement logical datetime case TIMESTAMP: Timestamp spannerTimestamp = struct.getTimestamp(column); @@ -384,7 +384,7 @@ private static java.time.Instant fromSpannerTimestamp(Timestamp spannerTimestamp // Default DATETIME behavior: convert to Joda DateTime return Instant.ofEpochSecond(spannerTimestamp.getSeconds()).toDateTime(); - // TODO: implement logical date + // TODO: implement logical date case DATE: return DateTime.parse(struct.getDate(column).toString()); case INT64: @@ -445,7 +445,7 @@ private static java.time.Instant fromSpannerTimestamp(Timestamp spannerTimestamp .map(timestamp -> Instant.ofEpochSecond(timestamp.getSeconds()).toDateTime()) .collect(toList()); - // TODO: implement logical date + // TODO: implement logical date case DATE: return struct.getDateList(column).stream() .map(date -> DateTime.parse(date.toString())) diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/testing/FakeDatasetService.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/testing/FakeDatasetService.java index 67dc802f1181..865b97e2f950 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/testing/FakeDatasetService.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/testing/FakeDatasetService.java @@ -943,7 +943,8 @@ public ApiFuture appendRows(long offset, ProtoRows rows) @Override public com.google.cloud.bigquery.storage.v1.@org.checkerframework.checker.nullness.qual .Nullable - TableSchema getUpdatedSchema() { + TableSchema + getUpdatedSchema() { return this.updatedSchema; } diff --git a/sdks/java/io/iceberg/build.gradle b/sdks/java/io/iceberg/build.gradle index 3f949c5cecb2..8d7c5cf08be0 100644 --- a/sdks/java/io/iceberg/build.gradle +++ b/sdks/java/io/iceberg/build.gradle @@ -221,7 +221,7 @@ task dataflowIntegrationTest(type: Test) { dependsOn ":runners:google-cloud-dataflow-java:buildAndPushDockerJavaContainer" finalizedBy ":runners:google-cloud-dataflow-java:cleanUpDockerJavaImages" def dockerJavaImageName = project.project(':runners:google-cloud-dataflow-java').ext.dockerJavaImageName - def dataflowLegacyWorkerJar = project.project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath + def dataflowLegacyWorkerJar = project.project(":runners:google-cloud-dataflow-java:worker").shadowJar.archiveFile.get().asFile def args = [ "--runner=DataflowRunner", diff --git a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaIO.java b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaIO.java index a41db9fbeaf4..bf4fdeba020a 100644 --- a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaIO.java +++ b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaIO.java @@ -2638,12 +2638,12 @@ public abstract static class ReadSourceDescriptors abstract @Nullable CheckStopReadingFn getCheckStopReadingFn(); @Pure - abstract @Nullable - SerializableFunction, Instant> getExtractOutputTimestampFn(); + abstract @Nullable SerializableFunction, Instant> + getExtractOutputTimestampFn(); @Pure - abstract @Nullable - SerializableFunction> getCreateWatermarkEstimatorFn(); + abstract @Nullable SerializableFunction> + getCreateWatermarkEstimatorFn(); @Pure abstract boolean isCommitOffsetEnabled(); @@ -3321,8 +3321,8 @@ public abstract static class WriteRecords public abstract Map getProducerConfig(); @Pure - public abstract @Nullable - SerializableFunction, Producer> getProducerFactoryFn(); + public abstract @Nullable SerializableFunction, Producer> + getProducerFactoryFn(); @Pure public abstract @Nullable Class> getKeySerializer(); @@ -3331,8 +3331,8 @@ public abstract static class WriteRecords public abstract @Nullable Class> getValueSerializer(); @Pure - public abstract @Nullable - KafkaPublishTimestampFunction> getPublishTimestampFunction(); + public abstract @Nullable KafkaPublishTimestampFunction> + getPublishTimestampFunction(); // Configuration for EOS sink @Pure @@ -3349,8 +3349,8 @@ public abstract static class WriteRecords public abstract int getNumShards(); @Pure - public abstract @Nullable - SerializableFunction, ? extends Consumer> getConsumerFactoryFn(); + public abstract @Nullable SerializableFunction, ? extends Consumer> + getConsumerFactoryFn(); @Pure public abstract BadRecordRouter getBadRecordRouter(); diff --git a/sdks/java/io/messaging-expansion-service/build.gradle b/sdks/java/io/messaging-expansion-service/build.gradle index 5d30a9270c06..212f305122ef 100644 --- a/sdks/java/io/messaging-expansion-service/build.gradle +++ b/sdks/java/io/messaging-expansion-service/build.gradle @@ -18,7 +18,9 @@ apply plugin: 'org.apache.beam.module' apply plugin: 'application' -mainClassName = "org.apache.beam.sdk.expansion.service.ExpansionService" +application { + mainClass = "org.apache.beam.sdk.expansion.service.ExpansionService" +} applyJavaNature( automaticModuleName: 'org.apache.beam.sdk.io.messaging.expansion.service', diff --git a/sdks/java/io/snowflake/expansion-service/build.gradle b/sdks/java/io/snowflake/expansion-service/build.gradle index f23825600693..500062dab31a 100644 --- a/sdks/java/io/snowflake/expansion-service/build.gradle +++ b/sdks/java/io/snowflake/expansion-service/build.gradle @@ -18,7 +18,9 @@ apply plugin: 'org.apache.beam.module' apply plugin: 'application' -mainClassName = "org.apache.beam.sdk.expansion.service.ExpansionService" +application { + mainClass = "org.apache.beam.sdk.expansion.service.ExpansionService" +} applyJavaNature( automaticModuleName: 'org.apache.beam.sdk.io.expansion.service', diff --git a/sdks/java/testing/load-tests/build.gradle b/sdks/java/testing/load-tests/build.gradle index ac3e391866d6..1e661abc2890 100644 --- a/sdks/java/testing/load-tests/build.gradle +++ b/sdks/java/testing/load-tests/build.gradle @@ -136,7 +136,7 @@ def getLoadTestArgs = { def sdkContainerImage = project.findProperty('sdkContainerImage') ?: project(":runners:google-cloud-dataflow-java").dockerJavaImageName loadTestArgsList.add("--sdkContainerImage=${sdkContainerImage}") } else { - def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?: project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath + def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?: project(":runners:google-cloud-dataflow-java:worker").shadowJar.archiveFile.get().asFile // Provide job with a customizable worker jar. // With legacy worker jar, containerImage is set to empty (i.e. to use the internal build). // More context and discussions can be found in PR#6694. diff --git a/sdks/java/testing/nexmark/build.gradle b/sdks/java/testing/nexmark/build.gradle index 9dab21aae884..87d28d26111e 100644 --- a/sdks/java/testing/nexmark/build.gradle +++ b/sdks/java/testing/nexmark/build.gradle @@ -146,7 +146,7 @@ def getNexmarkArgs = { nexmarkArgsList.add("--skipQueries=LOCAL_ITEM_SUGGESTION,AVERAGE_PRICE_FOR_CATEGORY,AVERAGE_SELLING_PRICE_BY_SELLER,HIGHEST_BID,WINNING_BIDS,SESSION_SIDE_INPUT_JOIN,BOUNDED_SIDE_INPUT_JOIN") // 3, 4, 6, 7, 9, 13, 14, 15 } } else { - def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?: project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath + def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?: project(":runners:google-cloud-dataflow-java:worker").shadowJar.archiveFile.get().asFile // Provide job with a customizable worker jar. // With legacy worker jar, containerImage is set to empty (i.e. to use the internal build). // More context and discussions can be found in PR#6694. diff --git a/sdks/java/testing/tpcds/build.gradle b/sdks/java/testing/tpcds/build.gradle index 4fe0f936fb36..60471d0a696e 100644 --- a/sdks/java/testing/tpcds/build.gradle +++ b/sdks/java/testing/tpcds/build.gradle @@ -137,7 +137,7 @@ task run(type: JavaExec) { def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?: project(":runners:google-cloud-dataflow-java:worker") - .shadowJar.archivePath + .shadowJar.archiveFile.get().asFile // Provide job with a customizable worker jar. // With legacy worker jar, containerImage is set to empty (i.e. to use the internal build). // More context and discussions can be found in PR#6694. diff --git a/sdks/java/testing/watermarks/build.gradle b/sdks/java/testing/watermarks/build.gradle index 74955685b7c9..7132379b3448 100644 --- a/sdks/java/testing/watermarks/build.gradle +++ b/sdks/java/testing/watermarks/build.gradle @@ -103,7 +103,7 @@ task run(type: JavaExec) { if (isDataflowRunner) { dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar" - def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?: project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath + def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?: project(":runners:google-cloud-dataflow-java:worker").shadowJar.archiveFile.get().asFile // Provide job with a customizable worker jar. // With legacy worker jar, containerImage is set to empty (i.e. to use the internal build). // More context and discussions can be found in PR#6694. diff --git a/sdks/java/transform-service/app/build.gradle b/sdks/java/transform-service/app/build.gradle index bf7a47a462e2..30c72ad928e3 100644 --- a/sdks/java/transform-service/app/build.gradle +++ b/sdks/java/transform-service/app/build.gradle @@ -25,7 +25,9 @@ applyJavaNature( validateShadowJar: false, shadowClosure: {}, ) -mainClassName = "org.apache.beam.sdk.transformservice.launcher.TransformServiceLauncher" +application { + mainClass = "org.apache.beam.sdk.transformservice.launcher.TransformServiceLauncher" +} description = "Apache Beam :: SDKs :: Java :: Transform Service :: App" ext.summary = """Runs a transform service.""" diff --git a/sdks/python/test-suites/portable/common.gradle b/sdks/python/test-suites/portable/common.gradle index 6cc30fcadab6..f8bdca51a9a5 100644 --- a/sdks/python/test-suites/portable/common.gradle +++ b/sdks/python/test-suites/portable/common.gradle @@ -34,7 +34,7 @@ ext { def createFlinkRunnerTestTask(String workerType, String flinkVersion) { String taskName - // project(":runners:flink:${flinkVersion}:job-server").shadowJar.archivePath is not resolvable until runtime, so hard-code it here. + // project(":runners:flink:${flinkVersion}:job-server").shadowJar.archiveFile.get().asFile is not resolvable until runtime, so hard-code it here. def jobServerJar = "${rootDir}/runners/flink/${flinkVersion}/job-server/build/libs/beam-runners-flink-${flinkVersion}-job-server-${version}.jar" def options = "--flink_job_server_jar=${jobServerJar} --environment_type=${workerType}" if (flinkVersion.startsWith('1')) { @@ -148,7 +148,7 @@ tasks.register("portableLocalRunnerTestWithRequirementsFile") { def createSparkRunnerTestTask(String workerType) { def taskName = "sparkCompatibilityMatrix${workerType}" - // `project(':runners:spark:3:job-server').shadowJar.archivePath` is not resolvable until runtime, so hard-code it here. + // `project(':runners:spark:3:job-server').shadowJar.archiveFile.get().asFile` is not resolvable until runtime, so hard-code it here. def jobServerJar = "${rootDir}/runners/spark/3/job-server/build/libs/beam-runners-spark-3-job-server-${version}.jar" def options = "--spark_job_server_jar=${jobServerJar} --environment_type=${workerType}" if (workerType == 'PROCESS') { @@ -296,7 +296,7 @@ project.tasks.register("flinkExamples") { "--project=apache-beam-testing", "--environment_type=LOOPBACK", "--temp_location=gs://temp-storage-for-end-to-end-tests/temp-it", - "--flink_job_server_jar=${project(":runners:flink:${latestFlink1Version}:job-server").shadowJar.archivePath}", + "--flink_job_server_jar=${project(":runners:flink:${latestFlink1Version}:job-server").shadowJar.archiveFile.get().asFile}", "--flink_conf_dir=${flink_conf_dir}", // Log spam for Flink runner with embedded jar '--job_server_jvm_properties=-Dslf4j.provider=org.slf4j.simple.SimpleServiceProvider', @@ -421,7 +421,7 @@ project.tasks.register("postCommitPy${pythonVersionSuffix}IT") { "--project=apache-beam-testing", "--environment_type=LOOPBACK", "--temp_location=gs://temp-storage-for-end-to-end-tests/temp-it", - "--flink_job_server_jar=${project(":runners:flink:${latestFlink1Version}:job-server").shadowJar.archivePath}", + "--flink_job_server_jar=${project(":runners:flink:${latestFlink1Version}:job-server").shadowJar.archiveFile.get().asFile}", "--flink_conf_dir=${flink_conf_dir}", // Log spam for Flink runner with embedded jar '--job_server_jvm_properties=-Dslf4j.provider=org.slf4j.simple.SimpleServiceProvider', @@ -435,7 +435,7 @@ project.tasks.register("postCommitPy${pythonVersionSuffix}IT") { "suite": "postCommitIT-flink-py${pythonVersionSuffix}", "pipeline_opts": pipelineOpts.join(" "), ]) - def kafkaJar = project(":sdks:java:testing:kafka-service:").buildTestKafkaServiceJar.archivePath + def kafkaJar = project(":sdks:java:testing:kafka-service:").buildTestKafkaServiceJar.archiveFile.get().asFile exec { if (fork_java_home != null) { @@ -471,7 +471,7 @@ project.tasks.register("xlangSpannerIOIT") { "--project=apache-beam-testing", "--environment_type=LOOPBACK", "--temp_location=gs://temp-storage-for-end-to-end-tests/temp-it", - "--flink_job_server_jar=${project(":runners:flink:${latestFlink1Version}:job-server").shadowJar.archivePath}", + "--flink_job_server_jar=${project(":runners:flink:${latestFlink1Version}:job-server").shadowJar.archiveFile.get().asFile}", // Log spam for Flink runner with embedded jar '--job_server_jvm_properties=-Dslf4j.provider=org.slf4j.simple.SimpleServiceProvider', '--job_server_jvm_properties=-Dorg.slf4j.simpleLogger.log.org.apache.flink.streaming=error', @@ -500,7 +500,7 @@ def addTestJavaJarCreator(String runner, Task jobServerJarTask) { executable "sh" def options = [ "--runner ${runner}", - "--job_server_jar ${jobServerJarTask.archivePath}", + "--job_server_jar ${jobServerJarTask.archiveFile.get().asFile}", "--env_dir ${project.rootProject.buildDir}/gradleenv/${project.path.hashCode()}", "--python_root_dir ${project.rootDir}/sdks/python", "--python_version ${project.ext.pythonVersion}", @@ -527,8 +527,8 @@ def addTestFlinkUberJar(boolean saveMainSession) { exec { executable "sh" def options = [ - "--flink_job_server_jar ${tasks.getByPath(":runners:flink:${latestFlinkVersion}:job-server:shadowJar").archivePath}", - "--flink_mini_cluster_jar ${tasks.getByPath(":runners:flink:${latestFlinkVersion}:job-server:miniCluster").archivePath}", + "--flink_job_server_jar ${tasks.getByPath(":runners:flink:${latestFlinkVersion}:job-server:shadowJar").archiveFile.get().asFile}", + "--flink_mini_cluster_jar ${tasks.getByPath(":runners:flink:${latestFlinkVersion}:job-server:miniCluster").archiveFile.get().asFile}", "--env_dir ${project.rootProject.buildDir}/gradleenv/${project.path.hashCode()}", "--python_root_dir ${project.rootDir}/sdks/python", "--python_version ${project.ext.pythonVersion}", diff --git a/settings.gradle.kts b/settings.gradle.kts index b45c3f9ffa4c..5659e004d819 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -215,7 +215,6 @@ include(":sdks:java:extensions:schemaio-expansion-service") include(":sdks:java:extensions:sketching") include(":sdks:java:extensions:sorter") include(":sdks:java:extensions:sql") -include(":sdks:java:extensions:sql:payloads") include(":sdks:java:extensions:sql:perf-tests") include(":sdks:java:extensions:sql:jdbc") include(":sdks:java:extensions:sql:hcatalog") @@ -375,7 +374,6 @@ include("beam-test-gha") project(":beam-test-gha").projectDir = file(".github") include("beam-validate-runner") project(":beam-validate-runner").projectDir = file(".test-infra/validate-runner") -include("com.google.api.gax.batching") include("sdks:java:io:kafka:kafka-392") findProject(":sdks:java:io:kafka:kafka-392")?.name = "kafka-392" include("sdks:java:managed")