Refine D2D with Grammar (.g/.g4) Class Mapping #1808 (#1942)

Signed-off-by: Chin Yeung Li <tli@nexb.com>

Author: chinyeungli

scanpipe/pipelines/deploy_to_develop.py

@@ -80,6 +80,9 @@ def steps(cls):
             cls.find_kotlin_packages,
             cls.map_kotlin_to_class,
             cls.map_jar_to_kotlin_source,
+            cls.find_grammar_packages,
+            cls.map_grammar_to_class,
+            cls.map_jar_to_grammar_source,
             cls.map_javascript,
             cls.map_javascript_symbols,
             cls.map_javascript_strings,
@@ -235,6 +238,27 @@ def map_jar_to_kotlin_source(self):
             project=self.project, jvm_lang=jvm.KotlinLanguage, logger=self.log
         )
 
+    @optional_step("Grammar")
+    def find_grammar_packages(self):
+        """Find the java package of the .g/.g4 source files."""
+        d2d.find_jvm_packages(
+            project=self.project, jvm_lang=jvm.GrammarLanguage, logger=self.log
+        )
+
+    @optional_step("Grammar")
+    def map_grammar_to_class(self):
+        """Map a .class compiled file to its .g/.g4 source."""
+        d2d.map_jvm_to_class(
+            project=self.project, jvm_lang=jvm.GrammarLanguage, logger=self.log
+        )
+
+    @optional_step("Grammar")
+    def map_jar_to_grammar_source(self):
+        """Map .jar files to their related source directory."""
+        d2d.map_jar_to_jvm_source(
+            project=self.project, jvm_lang=jvm.GrammarLanguage, logger=self.log
+        )
+
     @optional_step("JavaScript")
     def map_javascript(self):
         """

scanpipe/pipes/jvm.py

@@ -208,6 +208,15 @@ def get_normalized_path(cls, path, extension):
         return str(path.parent / f"{class_name}{extension}")
 
 
+class GrammarLanguage(JvmLanguage):
+    name = "grammar"
+    source_extensions = (".g", ".g4")
+    binary_extensions = (".class",)
+    source_package_attribute_name = "grammar_package"
+    package_regex = re.compile(r"^\s*package\s+([\w\.]+)\s*;?")
+    binary_map_type = "grammar_to_class"
+
+
 def get_fully_qualified_path(jvm_package, filename):
     """
     Return a fully qualified path of a ``filename`` in a

scanpipe/tests/pipes/test_d2d.py

@@ -435,6 +435,32 @@ def test_scanpipe_pipes_d2d_map_java_to_class_with_java_in_deploy(self):
         expected = "Mapping 1 .class resources to 1 ('.java',)"
         self.assertIn(expected, buffer.getvalue())
 
+    def test_scanpipe_pipes_d2d_map_grammar_to_class(self):
+        from1 = make_resource_file(
+            self.project1,
+            path="from/antlr4-4.5.1-beta-1/tool/src/org/antlr/v4/parse/BlockSetTransformer.g",
+            extra_data={"grammar_package": "org.antlr.v4.parse"},
+        )
+
+        to1 = make_resource_file(
+            self.project1,
+            path="to/org/antlr/v4/parse/BlockSetTransformer.class",
+        )
+
+        buffer = io.StringIO()
+        d2d.map_jvm_to_class(
+            self.project1, logger=buffer.write, jvm_lang=jvm.GrammarLanguage
+        )
+
+        expected = "Mapping 1 .class resources to 1 ('.g', '.g4')"
+        self.assertIn(expected, buffer.getvalue())
+        self.assertEqual(1, self.project1.codebaserelations.count())
+
+        r1 = self.project1.codebaserelations.get(to_resource=to1, from_resource=from1)
+        self.assertEqual("grammar_to_class", r1.map_type)
+        expected = {"from_source_root": "from/antlr4-4.5.1-beta-1/tool/src/"}
+        self.assertEqual(expected, r1.extra_data)
+
     def test_scanpipe_pipes_d2d_map_java_to_class_no_java(self):
         make_resource_file(self.project1, path="to/Abstract.class")
         buffer = io.StringIO()