diff --git a/.github/trigger_files/beam_PreCommit_Java_Delta_IO_Direct.json b/.github/trigger_files/beam_PreCommit_Java_Delta_IO_Direct.json index e3d6056a5de9..b26833333238 100644 --- a/.github/trigger_files/beam_PreCommit_Java_Delta_IO_Direct.json +++ b/.github/trigger_files/beam_PreCommit_Java_Delta_IO_Direct.json @@ -1,4 +1,4 @@ { "comment": "Modify this file in a trivial way to cause this test suite to run", - "modification": 1 + "modification": 2 } diff --git a/.github/workflows/beam_PreCommit_Java_Delta_IO_Direct.yml b/.github/workflows/beam_PreCommit_Java_Delta_IO_Direct.yml index ee40f3ef6a48..0d316a73c9cb 100644 --- a/.github/workflows/beam_PreCommit_Java_Delta_IO_Direct.yml +++ b/.github/workflows/beam_PreCommit_Java_Delta_IO_Direct.yml @@ -95,6 +95,16 @@ jobs: -PtestJavaVersion=17 \ -Pjava17Home=$JAVA_HOME_17_X64 \ --info + - name: run Delta IO Prism integration tests + uses: $/.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:delta:prismIntegrationTest + arguments: | + -PdisableSpotlessCheck=true \ + -PdisableCheckStyle=true \ + -PtestJavaVersion=17 \ + -Pjava17Home=$JAVA_HOME_17_X64 \ + --info - name: Archive JUnit Test Results uses: actions/upload-artifact@v7 if: ${{ !success() }} diff --git a/sdks/java/io/delta/build.gradle b/sdks/java/io/delta/build.gradle index 2787c4bb606d..dc2e964ea878 100644 --- a/sdks/java/io/delta/build.gradle +++ b/sdks/java/io/delta/build.gradle @@ -78,6 +78,7 @@ dependencies { testImplementation library.java.testcontainers_localstack testImplementation library.java.hadoop_aws testRuntimeOnly project(path: ":runners:google-cloud-dataflow-java") + testRuntimeOnly project(path: ":runners:prism:java") testRuntimeOnly library.java.slf4j_simple } @@ -138,3 +139,32 @@ task integrationTest(type: Test) { systemProperty "org.slf4j.simpleLogger.log.org.apache.hadoop.fs.s3a", "info" systemProperty "org.slf4j.simpleLogger.log.software.amazon.awssdk", "info" } + +project.evaluationDependsOn(":runners:prism") + +task prismIntegrationTest(type: Test) { + group = "Verification" + description = "Runs local Delta Lake Managed IO tests with the Prism runner." + + def prismProject = project(":runners:prism") + dependsOn prismProject.tasks.named("build") + def args = [ + "--runner=TestPrismRunner", + "--defaultEnvironmentType=LOOPBACK", + "--enableWebUI=false", + "--prismLocation=${prismProject.property('buildTarget').toString()}", + "--prismLogLevel=warn", + ] + systemProperty "beamTestPipelineOptions", JsonOutput.toJson(args) + + outputs.upToDateWhen { false } + include '**/DeltaIOTest.class' + filter { + includeTestsMatching 'org.apache.beam.sdk.io.delta.DeltaIOTest.testManagedDeltaRead' + includeTestsMatching 'org.apache.beam.sdk.io.delta.DeltaIOTest.testManagedDeltaReadWithVersion' + includeTestsMatching 'org.apache.beam.sdk.io.delta.DeltaIOTest.testManagedDeltaReadWithTimestamp' + includeTestsMatching 'org.apache.beam.sdk.io.delta.DeltaIOTest.testReadChangesWithManaged' + } + classpath = sourceSets.test.runtimeClasspath + testClassesDirs = sourceSets.test.output.classesDirs +}