Merge pull request #16 from nexB/v21.4.28-prep

Version v21.4.28

Author: pombredanne

.gitattributes

@@ -1,2 +1,2 @@
 # Ignore all Git auto CR/LF line endings conversions
-* binary
+* -text

CHANGELOG.rst

@@ -4,6 +4,14 @@ Release notes
 vNext
 -----
 
+
+Version 21.4.28
+---------------
+
+*2020-04-28*
+- Add new function to get a Resource path stripped from its root path segment
+
+
 Version 21.1.21
 ---------------
 

src/commoncode/resource.py

@@ -60,6 +60,7 @@
 from commoncode.fileutils import splitext_name
 
 from commoncode import ignore
+from commoncode import paths
 
 """
 This module provides Codebase and Resource objects as an abstraction for files
@@ -1054,6 +1055,12 @@ def type(self, value):
         else:
             self.is_file = False
 
+    def get_path(self, strip_root=False):
+        if strip_root:
+            return strip_first_path_segment(self.path)
+        else:
+            return self.path
+
     @property
     def is_dir(self):
         # note: we only store is_file
@@ -1354,6 +1361,29 @@ def get_path(root_location, location, full_root=False, strip_root=False):
     return posix_loc.replace(posix_root_loc, '', 1)
 
 
+def strip_first_path_segment(path):
+    """
+    Return a POSIX path stripped from its first path segment.
+
+    For example::
+        >>> strip_first_path_segment('')
+        ''
+        >>> strip_first_path_segment('foo')
+        'foo'
+        >>> strip_first_path_segment('foo/bar/baz')
+        'bar/baz'
+        >>> strip_first_path_segment('/foo/bar/baz/')
+        'bar/baz'
+        >>> strip_first_path_segment('foo/')
+        'foo/'
+    """
+    segments = paths.split(path)
+    if not segments or len(segments) == 1:
+        return path
+    stripped = segments[1:]
+    return '/'.join(stripped)
+
+
 def get_codebase_cache_dir(temp_dir):
     """
     Return a new, created and unique per-run cache storage directory path rooted
@@ -1506,20 +1536,27 @@ def _get_or_create_parent(self, path, parent_by_path):
         """
         Return a parent resource for a given `path` from `parent_by_path`.
 
-        If a parent resource for a `path` does not exist in `parent_by_path`, it is created recursively.
+        If a parent resource for a `path` does not exist in `parent_by_path`, it
+        is created recursively.
 
-        Note: the root path and root Resource must already be in `parent_by_path` or else this
-        function does not work.
+        Note: the root path and root Resource must already be in
+        `parent_by_path` or else this function does not work.
         """
-        parent_path = parent_directory(path).rstrip('/').rstrip('\\')
+        parent_path = parent_directory(path).rstrip('/').rstrip('\\').lstrip("/")
         existing_parent = parent_by_path.get(parent_path)
         if existing_parent:
             return existing_parent
         parent_parent = self._get_or_create_parent(parent_path, parent_by_path)
         parent_name = file_base_name(parent_path)
         parent_is_file = False
         parent_resource_data = self._create_empty_resource_data()
-        parent_resource = self._create_resource(parent_name, parent_parent, parent_is_file, parent_path, parent_resource_data)
+        parent_resource = self._create_resource(
+            parent_name,
+            parent_parent,
+            parent_is_file,
+            parent_path,
+            parent_resource_data,
+        )
         parent_by_path[parent_path] = parent_resource
         return parent_resource
 
@@ -1620,7 +1657,12 @@ def _populate(self, scan_data):
         root_name = root_path
         root_is_file = False
         root_data = self._create_empty_resource_data()
-        root_resource = self._create_root_resource(root_name, root_path, root_is_file, root_data)
+        root_resource = self._create_root_resource(
+            name=root_name,
+            path=root_path,
+            is_file=root_is_file,
+            root_data=root_data,
+        )
 
         # To help recreate the resource tree we keep a mapping by path of any
         # parent resource
@@ -1630,24 +1672,32 @@ def _populate(self, scan_data):
             path = resource_data.get('path')
             # Append virtual_root path to imported Resource path if we are merging multiple scans
             if multiple_input:
-                path = os.path.join(root_path, path)
+                path = posixpath.join(root_path, path)
+
             name = resource_data.get('name', None)
             if not name:
                 name = file_name(path)
+
             is_file = resource_data.get('type', 'file') == 'file'
 
             existing_parent = parent_by_path.get(path)
             if existing_parent:
-                # We update the empty parent Resouorce we in _get_or_create_parent() with the data
-                # from the scan
+                # We update the empty parent Resouorce we in
+                # _get_or_create_parent() with the data from the scan
                 for k, v in resource_data.items():
                     setattr(existing_parent, k, v)
                 self.save_resource(existing_parent)
             else:
-                # Note: `root_path`: `root_resource` must be in `parent_by_path` in order for
-                # `_get_or_create_parent` to work
+                # Note: `root_path`: `root_resource` must be in `parent_by_path`
+                # in order for `_get_or_create_parent` to work
                 parent = self._get_or_create_parent(path, parent_by_path)
-                resource = self._create_resource(name, parent, is_file, path, resource_data)
+                resource = self._create_resource(
+                    name=name,
+                    parent=parent,
+                    is_file=is_file,
+                    path=path,
+                    resource_data=resource_data,
+                )
 
                 # Files are not parents (for now), so we do not need to add this
                 # to the parent_by_path mapping
@@ -1665,7 +1715,14 @@ def _create_root_resource(self, name, path, is_file, root_data):
         if root_data:
             root_data = remove_properties_and_basics(root_data)
         root = self.resource_class(
-            name=name, location=None, path=path, rid=0, pid=None, is_file=is_file, **root_data)
+            name=name,
+            location=None,
+            path=path,
+            rid=0,
+            pid=None,
+            is_file=is_file,
+            **root_data,
+        )
 
         self.resource_ids.add(0)
         self.resources[0] = root

src/commoncode/system.py

@@ -20,7 +20,6 @@
 
 import getpass
 import os
-import subprocess
 import sys
 
 

tests/data/resource/virtual_codebase/path_full_root.json

@@ -0,0 +1,36 @@
+{
+    "scancode_notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.",
+    "scancode_version": "2.2.1.post12.6d07756e",
+    "scancode_options": {
+      "--info": true,
+      "--license-score": 0,
+      "--full-root": true,
+      "--format": "json-pp"
+    },
+    "files_count": 1,
+    "files": [
+      {
+        "path": "/Users/sesser/code/nexb/scancode-toolkit/samples/README",
+        "type": "file",
+        "name": "README",
+        "base_name": "README",
+        "extension": "",
+        "date": "2017-09-22",
+        "size": 236,
+        "sha1": "2e07e32c52d607204fad196052d70e3d18fb8636",
+        "md5": "effc6856ef85a9250fb1a470792b3f38",
+        "files_count": null,
+        "mime_type": "text/plain",
+        "file_type": "ASCII text",
+        "programming_language": null,
+        "is_binary": false,
+        "is_text": true,
+        "is_archive": false,
+        "is_media": false,
+        "is_source": false,
+        "is_script": false,
+        "scan_errors": []
+      }
+    ]
+  }
+  

tests/data/resource/virtual_codebase/stripped-and-skipped-root.json

@@ -0,0 +1,19 @@
+{
+  "files": [
+    {
+      "path": "samples",
+      "type": "directory",
+      "scan_errors": []
+    },
+    {
+      "path": "samples/README",
+      "type": "file",
+      "scan_errors": []
+    },
+    {
+      "path": "samples/screenshot.png",
+      "type": "file",
+      "scan_errors": []
+    }
+  ]
+}

tests/test_resource.py

@@ -802,6 +802,13 @@ def test_virtual_codebase_walk_skip_root_basic(self):
         ]
         assert expected == [(r.name, r.is_file) for r in results]
 
+    def test_virtual_codebase_get_path_with_strip_root_and_walk_with_skip_root(self):
+        scan_data = self.get_test_loc('resource/virtual_codebase/stripped-and-skipped-root.json')
+        virtual_codebase = VirtualCodebase(location=scan_data)
+        results = [r.get_path(strip_root=True) for r in virtual_codebase.walk(skip_root=True)]
+        expected = ['README', 'screenshot.png']
+        assert results == expected
+
     def test_virtual_codebase_walk_filtered_with_filtered_root(self):
         scan_data = self.get_test_loc('resource/virtual_codebase/virtual_codebase.json')
         virtual_codebase = VirtualCodebase(location=scan_data)
@@ -1323,6 +1330,13 @@ def test_VirtualCodebase_create_from_multiple_scans(self):
         ]
         assert expected == results
 
+    def test_VirtualCodebase_scanning_full_root(self):
+        test_file = self.get_test_loc("resource/virtual_codebase/path_full_root.json")
+        codebase = VirtualCodebase(test_file)
+        resource = sorted(r for r in codebase.walk())[0]
+        assert resource.path == "/Users/sesser/code/nexb/scancode-toolkit/samples/README"
+        assert codebase.compute_counts()[0] == 1
+
 
 class TestResource(FileBasedTesting):
     test_data_dir = join(dirname(__file__), 'data')