W-12334626: run old test suites using language level for regression t… (#70)

* W-12334626: run old test suites using language level for regression testing

Author: andres-rad

.github/workflows/main.yml

@@ -51,6 +51,17 @@ jobs:
         run: |
           ./gradlew --stacktrace build
         shell: bash
+
+      #Run regression tests
+      - name: Run regression test 2.5
+        run: |
+          ./gradlew --stacktrace -PweaveTestSuiteVersion=2.5.0-SNAPSHOT native-cli-integration-tests:test
+        shell: bash
+
+      - name: Run regression test 2.4
+        run: |
+          ./gradlew --stacktrace -PweaveTestSuiteVersion=2.4.0-HF-SNAPSHOT native-cli-integration-tests:test
+        shell: bash
 
       # Generate distro
       - name: Create Distro

gradle.properties

@@ -1,4 +1,5 @@
 weaveVersion=2.6.0-SNAPSHOT
+weaveTestSuiteVersion=2.5.0-SNAPSHOT
 nativeVersion=100.100.100
 scalaVersion=2.12.11
 ioVersion=1.0.0-SNAPSHOT

native-cli-integration-tests/build.gradle

@@ -2,21 +2,45 @@ plugins {
     id "com.github.maiflai.scalatest" version "${scalaTestPluginVersion}"
 }
 
+configurations {
+    weaveSuite
+}
+
+sourceSets {
+    test {
+        resources {
+            srcDirs += ["build/resources"]
+        }
+    }
+}
 
 dependencies {
-    api project(":native-cli")
+    api(project(":native-cli"))
+
     testImplementation group: 'org.scalatest', name: 'scalatest_2.12', version: scalaTestVersion
     testImplementation group: 'org.mule.weave', name: 'parser', version: weaveVersion
     testImplementation group: 'org.mule.weave', name: 'test-helpers', version: weaveVersion
-    testImplementation "org.mule.weave:runtime:${weaveVersion}:test@zip"
-    testImplementation "org.mule.weave:yaml-module:${weaveVersion}:test@zip"
+
+    weaveSuite "org.mule.weave:runtime:${weaveTestSuiteVersion}:test@zip"
+    weaveSuite "org.mule.weave:yaml-module:${weaveTestSuiteVersion}:test@zip"
+
     testImplementation 'commons-io:commons-io:2.11.0'
     testRuntimeOnly 'org.pegdown:pegdown:1.6.0'
-
     testImplementation group: 'com.sun.mail', name: 'jakarta.mail', version: '1.6.4'
     testImplementation group: 'xerces', name: 'xercesImpl', version: '2.12.1'
     testImplementation group: 'xalan', name: 'xalan', version: '2.7.2'
     testImplementation group: 'commons-beanutils', name: 'commons-beanutils', version: '1.9.4'
 }
 
+task cleanTestSuites(type: Delete) {
+    delete "$projectDir/build/resources/weave-suites"
+}
+
+task downloadTestSuites(type: Copy) {
+    from configurations.weaveSuite
+    into "$projectDir/build/resources/weave-suites"
+}
+
+downloadTestSuites.dependsOn(cleanTestSuites)
+test.dependsOn(downloadTestSuites)
 test.dependsOn(":native-cli:nativeCompile")

native-cli-integration-tests/src/test/scala/org/mule/weave/native/NativeCliRuntimeIT.scala

@@ -21,19 +21,20 @@ import org.mule.weave.v2.parser.ast.header.directives.OutputDirective
 import org.mule.weave.v2.parser.ast.structure.StringNode
 import org.mule.weave.v2.sdk.ParsingContextFactory
 import org.mule.weave.v2.sdk.WeaveResourceFactory
+import org.mule.weave.v2.utils.DataWeaveVersion
 import org.mule.weave.v2.utils.StringHelper.toStringTransformer
+import org.mule.weave.v2.version.ComponentVersion
 import org.scalatest.Assertion
 import org.scalatest.FunSpec
 import org.scalatest.Matchers
+import sun.net.www.protocol.file.FileURLConnection
 
 import java.io.ByteArrayInputStream
 import java.io.File
 import java.io.FileFilter
 import java.io.FileInputStream
 import java.io.IOException
 import java.io.InputStream
-import java.net.JarURLConnection
-import java.nio.charset.Charset
 import java.nio.charset.StandardCharsets
 import java.nio.file.Files
 import java.nio.file.Path
@@ -45,8 +46,6 @@ import javax.mail.util.ByteArrayDataSource
 import scala.collection.JavaConverters._
 import scala.io.BufferedSource
 import scala.io.Source
-import scala.io.Source.fromFile
-import scala.util.Try
 
 class NativeCliRuntimeIT extends FunSpec
   with Matchers
@@ -56,20 +55,19 @@ class NativeCliRuntimeIT extends FunSpec
 
   private val TIMEOUT: (Int, TimeUnit) = (30, TimeUnit.SECONDS)
   private val INPUT_FILE_CONFIG_PROPERTY_PATTERN = Pattern.compile("in[0-9]+-config\\.properties")
-  private val OUTPUT_FILE_CONFIG_PROPERTY_PATTERN = Pattern.compile("out[0-9]+-config\\.properties")
+  private val OUTPUT_FILE_CONFIG_PROPERTY_PATTERN = Pattern.compile("out[0-9]*-config\\.properties")
   private val INPUT_FILE_PATTERN = Pattern.compile("in[0-9]+\\.[a-zA-Z]+")
   private val OUTPUT_FILE_PATTERN = Pattern.compile("out\\.[a-zA-Z]+")
 
-  val testSuites =
-    Seq(
-      TestSuite("master", loadTestZipFile("access_raw_value")),
-      TestSuite("yaml", loadTestZipFile("comments"))
+  val testSuites = Seq(
+      TestSuite("master", loadTestZipFile(s"weave-suites/runtime-${ComponentVersion.weaveSuiteVersion}-test.zip")),
+      TestSuite("yaml", loadTestZipFile(s"weave-suites/yaml-module-${ComponentVersion.weaveSuiteVersion}-test.zip"))
     )
 
   private def loadTestZipFile(testSuiteExample: String): File = {
     val url = getResource(testSuiteExample)
-    val connection = url.openConnection.asInstanceOf[JarURLConnection]
-    val zipFile = new File(connection.getJarFileURL.toURI)
+    val connection = url.openConnection.asInstanceOf[FileURLConnection]
+    val zipFile = new File(connection.getURL.toURI)
     zipFile
   }
 
@@ -209,6 +207,9 @@ class NativeCliRuntimeIT extends FunSpec
               throw ioe
           }
 
+          val languageLevel = DataWeaveVersion(ComponentVersion.weaveSuiteVersion).toString()
+          args = args :+ "--language-level" :+ languageLevel
+
           args = args :+ "-f"
           args = args :+ cliTransform.getAbsolutePath
 
@@ -354,7 +355,7 @@ class NativeCliRuntimeIT extends FunSpec
 
   override def ignoreTests(): Array[String] = {
     // Encoding issues
-    Array("csv-invalid-utf8") ++
+    val baseArray = Array("csv-invalid-utf8") ++
       // Fail in java11 because broken backwards
       Array("coerciones_toString", "date-coercion") ++
       // Use resources (dwl files) that is present in the Tests but not in Cli (e.g: org::mule::weave::v2::libs::)
@@ -382,6 +383,7 @@ class NativeCliRuntimeIT extends FunSpec
         "runtime_run_coercionException",
         "runtime_run_fibo",
         "runtime_run_null_java",
+        "sql_date_mapping",
         "write-function-with-null"
       ) ++
       // Multipart Object has empty `parts` and expects at least one part
@@ -395,6 +397,36 @@ class NativeCliRuntimeIT extends FunSpec
       // Take too long time
       Array("array-concat") ++
       Array("runtime_run")
+
+    if (DataWeaveVersion(ComponentVersion.weaveSuiteVersion).toString() == "2.4") {
+      baseArray ++
+        // A change to json streaming in 2.5.0 breaks this test
+        Array("default_with_extended_null_type") ++
+        // Change in validations in 2.5.0 breaks these tests
+        Array("logical-and",
+          "logical-or"
+        ) ++
+        Array("coerciones_toBinary") ++
+        // 2.5.0 dwl now prints metadata breaking these tests
+        Array("dfl-inline-default-namespace",
+          "dfl-inline-namespace",
+          "dfl-maxCollectionSize",
+          "dfl-overwrite-namespace",
+          "multipart-base64-to-multipart",
+          "xml-nill-multiple-attributes-nested",
+          "xml-nill-multiple-attributes",
+          "read_scalar_values"
+        ) ++
+        // A change of positions on dw::Core 2.5.0 breaks this test
+        Array(
+          "runtime_run_unhandled_compilation_exception"
+        ) ++
+        Array("as-operator",
+          "type-equality"
+        )
+    } else {
+      baseArray
+    }
   }
 }
 

native-cli/build.gradle

@@ -62,6 +62,7 @@ task genVersions() {
     outputPrinter.println("object ComponentVersion {")
     outputPrinter.println("\tval weaveVersion = \"" + weaveVersion + "\"")
     outputPrinter.println("\tval nativeVersion = \"" + nativeVersion + "\"")
+    outputPrinter.println("\tval weaveSuiteVersion = \"" + weaveTestSuiteVersion + "\"")
     outputPrinter.println("}")
     outputPrinter.close()
 }