Add ecosystem specific inclusions or exclusions (#1550)

* Add ecosystem specific inclusions or exclusions

Also ignore specific files paths containing metadata in ruby
gems.

Reference: #1438
Reference: #1476

Signed-off-by: Ayan Sinha Mahapatra <ayansmahapatra@gmail.com>
Signed-off-by: Philippe Ombredanne <pombredanne@nexb.com>
Co-authored-by: Philippe Ombredanne <pombredanne@nexb.com>

Author: AyanSinhaMahapatra

scanpipe/pipelines/__init__.py

@@ -78,7 +78,10 @@ def flag_ignored_resources(self):
             ignored_patterns = ignored_patterns.splitlines()
         ignored_patterns.extend(flag.DEFAULT_IGNORED_PATTERNS)
 
-        flag.flag_ignored_patterns(self.project, patterns=ignored_patterns)
+        flag.flag_ignored_patterns(
+            codebaseresources=self.project.codebaseresources.no_status(),
+            patterns=ignored_patterns,
+        )
 
     def extract_archive(self, location, target):
         """Extract archive at `location` to `target`. Save errors as messages."""
@@ -179,6 +182,8 @@ def __init__(self, run_instance):
         self.selected_groups = run_instance.selected_groups
         self.selected_steps = run_instance.selected_steps
 
+        self.ecosystem_config = None
+
     @classmethod
     def get_initial_steps(cls):
         """Add the ``download_inputs`` step as an initial step if enabled."""

scanpipe/pipelines/deploy_to_develop.py

@@ -24,6 +24,7 @@
 from scanpipe import pipes
 from scanpipe.pipelines import Pipeline
 from scanpipe.pipes import d2d
+from scanpipe.pipes import d2d_config
 from scanpipe.pipes import flag
 from scanpipe.pipes import input
 from scanpipe.pipes import matchcode
@@ -64,6 +65,8 @@ def steps(cls):
             cls.flag_empty_files,
             cls.flag_whitespace_files,
             cls.flag_ignored_resources,
+            cls.load_ecosystem_config,
+            cls.map_ruby,
             cls.map_about_files,
             cls.map_checksum,
             cls.match_archives_to_purldb,
@@ -94,33 +97,6 @@ def steps(cls):
             cls.create_local_files_packages,
         )
 
-    purldb_package_extensions = [".jar", ".war", ".zip"]
-    purldb_resource_extensions = [
-        ".map",
-        ".js",
-        ".mjs",
-        ".ts",
-        ".d.ts",
-        ".jsx",
-        ".tsx",
-        ".css",
-        ".scss",
-        ".less",
-        ".sass",
-        ".soy",
-        ".class",
-    ]
-    doc_extensions = [
-        ".pdf",
-        ".doc",
-        ".docx",
-        ".ppt",
-        ".pptx",
-        ".tex",
-        ".odt",
-        ".odp",
-    ]
-
     def get_inputs(self):
         """Locate the ``from`` and ``to`` input files."""
         self.from_files, self.to_files = d2d.get_inputs(self.project)
@@ -155,6 +131,15 @@ def flag_whitespace_files(self):
         """Flag whitespace files with size less than or equal to 100 byte as ignored."""
         d2d.flag_whitespace_files(project=self.project)
 
+    def load_ecosystem_config(self):
+        """Load ecosystem specific configurations for d2d steps for selected options."""
+        d2d_config.load_ecosystem_config(pipeline=self, options=self.selected_groups)
+
+    @optional_step("Ruby")
+    def map_ruby(self):
+        """Load Ruby specific configurations for d2d steps."""
+        pass
+
     def map_about_files(self):
         """Map ``from/`` .ABOUT files to their related ``to/`` resources."""
         d2d.map_about_files(project=self.project, logger=self.log)
@@ -171,7 +156,7 @@ def match_archives_to_purldb(self):
 
         d2d.match_purldb_resources(
             project=self.project,
-            extensions=self.purldb_package_extensions,
+            extensions=self.matchable_package_extensions,
             matcher_func=d2d.match_purldb_package,
             logger=self.log,
         )
@@ -249,7 +234,7 @@ def match_resources_to_purldb(self):
 
         d2d.match_purldb_resources(
             project=self.project,
-            extensions=self.purldb_resource_extensions,
+            extensions=self.matchable_resource_extensions,
             matcher_func=d2d.match_purldb_resource,
             logger=self.log,
         )
@@ -287,6 +272,7 @@ def flag_mapped_resources_archives_and_ignored_directories(self):
     def perform_house_keeping_tasks(self):
         """
         On deployed side
+            - Ignore specific files based on ecosystem based configurations.
             - PurlDB match files with ``no-java-source`` and empty status,
                 if no match is found update status to ``requires-review``.
             - Update status for uninteresting files.
@@ -297,9 +283,14 @@ def perform_house_keeping_tasks(self):
         """
         d2d.match_resources_with_no_java_source(project=self.project, logger=self.log)
         d2d.handle_dangling_deployed_legal_files(project=self.project, logger=self.log)
+        d2d.ignore_unmapped_resources_from_config(
+            project=self.project,
+            patterns_to_ignore=self.ecosystem_config.deployed_resource_path_exclusions,
+            logger=self.log,
+        )
         d2d.match_unmapped_resources(
             project=self.project,
-            matched_extensions=self.purldb_resource_extensions,
+            matched_extensions=self.ecosystem_config.matchable_resource_extensions,
             logger=self.log,
         )
         d2d.flag_undeployed_resources(project=self.project)
@@ -335,5 +326,5 @@ def flag_deployed_from_resources_with_missing_license(self):
         """Update the status for deployed from files with missing license."""
         d2d.flag_deployed_from_resources_with_missing_license(
             self.project,
-            doc_extensions=self.doc_extensions,
+            doc_extensions=self.ecosystem_config.doc_extensions,
         )

scanpipe/pipes/d2d.py

@@ -54,6 +54,7 @@
 from scanpipe.models import CodebaseRelation
 from scanpipe.models import CodebaseResource
 from scanpipe.models import convert_glob_to_django_regex
+from scanpipe.pipes import d2d_config
 from scanpipe.pipes import flag
 from scanpipe.pipes import get_resource_diff_ratio
 from scanpipe.pipes import js
@@ -1463,6 +1464,20 @@ def match_resources_with_no_java_source(project, logger=None):
         )
 
 
+def ignore_unmapped_resources_from_config(project, patterns_to_ignore, logger=None):
+    """Ignore unmapped resources for a project using `patterns_to_ignore`."""
+    ignored_resources_count = flag.flag_ignored_patterns(
+        codebaseresources=project.codebaseresources.to_codebase().no_status(),
+        patterns=patterns_to_ignore,
+        status=flag.IGNORED_FROM_CONFIG,
+    )
+    if logger:
+        logger(
+            f"Ignoring {ignored_resources_count:,d} to/ resources with "
+            "ecosystem specific configurations."
+        )
+
+
 def match_unmapped_resources(project, matched_extensions=None, logger=None):
     """
     Match resources with empty status to PurlDB, if unmatched
@@ -1925,7 +1940,10 @@ def map_rust_binaries_with_symbols(project, logger=None):
     )
 
     # Collect source symbols from rust source files
-    rust_from_resources = from_resources.filter(extension=".rs")
+    rust_config = d2d_config.get_ecosystem_config(ecosystem="Rust")
+    rust_from_resources = from_resources.filter(
+        extension__in=rust_config.source_symbol_extensions
+    )
 
     map_binaries_with_symbols(
         project=project,
@@ -1945,7 +1963,10 @@ def map_elfs_binaries_with_symbols(project, logger=None):
     )
 
     # Collect source symbols from elf related source files
-    elf_from_resources = from_resources.filter(extension__in=[".c", ".cpp", ".h"])
+    elf_config = d2d_config.get_ecosystem_config(ecosystem="Elf")
+    elf_from_resources = from_resources.filter(
+        extension__in=elf_config.source_symbol_extensions
+    )
 
     map_binaries_with_symbols(
         project=project,
@@ -1968,8 +1989,9 @@ def map_macho_binaries_with_symbols(project, logger=None):
     )
 
     # Collect source symbols from macos related source files
+    macos_config = d2d_config.get_ecosystem_config(ecosystem="MacOS")
     mac_from_resources = from_resources.filter(
-        extension__in=[".c", ".cpp", ".h", ".m", ".swift"]
+        extension__in=macos_config.source_symbol_extensions,
     )
 
     map_binaries_with_symbols(
@@ -1990,8 +2012,9 @@ def map_winpe_binaries_with_symbols(project, logger=None):
     )
 
     # Collect source symbols from windows related source files
+    windows_config = d2d_config.get_ecosystem_config(ecosystem="Windows")
     windows_from_resources = from_resources.filter(
-        extension__in=[".c", ".cpp", ".h", ".cs"]
+        extension__in=windows_config.source_symbol_extensions,
     )
 
     map_binaries_with_symbols(
@@ -2057,16 +2080,17 @@ def map_javascript_symbols(project, logger=None):
     """Map deployed JavaScript, TypeScript to its sources using symbols."""
     project_files = project.codebaseresources.files()
 
+    js_config = d2d_config.get_ecosystem_config(ecosystem="JavaScript")
     javascript_to_resources = (
         project_files.to_codebase()
         .has_no_relation()
-        .filter(extension__in=[".ts", ".js"])
+        .filter(extension__in=js_config.source_symbol_extensions)
     )
 
     javascript_from_resources = (
         project_files.from_codebase()
         .exclude(path__contains="/test/")
-        .filter(extension__in=[".ts", ".js"])
+        .filter(extension__in=js_config.source_symbol_extensions)
     )
 
     if not (javascript_from_resources.exists() and javascript_to_resources.exists()):