Merge pull request #22 from nexB/resource-bug

Fix VirtualCodebase bug

Author: pombredanne

CHANGELOG.rst

@@ -5,10 +5,17 @@ vNext
 -----
 
 
+Version 21.6.10
+---------------
+
+- Do not fail if a Codebase file.size is None and not zero
+- Bump pinned dependencies including pkg:pypi/urllib3 for CVE-2021-33503
+
+
 Version 21.5.25
 ---------------
 
-- Fix click-realted bug https://github.com/nexB/scancode-toolkit/issues/2529
+- Fix click-related bug https://github.com/nexB/scancode-toolkit/issues/2529
 - Add tests to run on the latest of every dependency
 
 

requirements.txt

@@ -1,15 +1,15 @@
-attrs==19.3.0
-beautifulsoup4==4.7.1
-certifi==2020.12.5
-chardet==3.0.4
-click==6.7
+attrs==21.2.0
+beautifulsoup4==4.9.3
+certifi==2021.5.30
+chardet==4.0.0
+click==8.0.1
 idna==2.10
 intbitset==2.4.1
 PyYAML==5.4.1
 requests==2.25.1
 saneyaml==0.5.2
-six==1.12.0
-soupsieve==1.9.2
-text-unidecode==1.2
-typing==3.6.4
-urllib3==1.26.4
+six==1.16.0
+soupsieve==2.2.1
+text-unidecode==1.3
+typing==3.6.6
+urllib3==1.26.5

setup.cfg

@@ -27,19 +27,21 @@ keywords =
     scancode-toolkit
 
 [options]
-package_dir=
-    =src
-packages=find:
+package_dir = =src
+packages = find:
 include_package_data = true
 zip_safe = false
+
+python_requires = >=3.6.*, <4
+
 install_requires =
     attrs >= 18.1, !=20.1.0
-    click >= 6.0.0
-    text_unidecode >= 1.0
     Beautifulsoup4 >= 4.0.0, < 5.0.0
-    requests >= 2.7.0, < 3.0.0
+    click >= 6.7, !=7.0
     intbitset >= 2.3.0, < 3.0
-    saneyaml
+    requests >= 2.7.0, < 3.0.0
+    saneyaml >= 0.5.2
+    text_unidecode >= 1.0
     typing >=3.6, < 3.7; python_version < "3.7"
 
 setup_requires = setuptools_scm[toml] >= 4

src/commoncode/resource.py

@@ -884,7 +884,7 @@ def compute_counts(self, skip_root=False, skip_filtered=False):
             files_count += 1
         else:
             dirs_count += 1
-        size_count += root.size
+        size_count += root.size or 0
 
         return files_count, dirs_count, size_count
 
@@ -1133,9 +1133,9 @@ def _compute_children_counts(self, codebase, skip_filtered=False):
         """
         files_count = dirs_count = size_count = 0
         for child in self.children(codebase):
-            files_count += child.files_count
-            dirs_count += child.dirs_count
-            size_count += child.size_count
+            files_count += child.files_count or 0
+            dirs_count += child.dirs_count or 0
+            size_count += child.size_count or 0
 
             if skip_filtered and child.is_filtered:
                 continue
@@ -1144,7 +1144,7 @@ def _compute_children_counts(self, codebase, skip_filtered=False):
                 files_count += 1
             else:
                 dirs_count += 1
-            size_count += child.size
+            size_count += child.size or 0
 
         self.files_count = files_count
         self.dirs_count = dirs_count

tests/data/resource/virtual_codebase/node-16-slim.json

@@ -0,0 +1,26 @@
+{
+"headers": [],
+"files": [
+{
+  "for_packages": [
+    "pkg:deb/debian/bash@5.0-4?arch=amd64"
+  ],
+  "compliance_alert": "",
+  "path": "/node_16_slim.tar-extract/02c055ef67f5904019f43a41ea5f099996d8e7633749b6e606c400526b2c4b33/bin/bash",
+  "size": 1168776,
+  "type": "file",
+  "name": "bash",
+  "extension": ""
+},
+{
+  "for_packages": [
+    "pkg:deb/debian/gzip@1.9-3?arch=amd64"
+  ],
+  "compliance_alert": "",
+  "path": "/node_16_slim.tar-extract/02c055ef67f5904019f43a41ea5f099996d8e7633749b6e606c400526b2c4b33/bin/uncompress",
+  "size": null,
+  "type": "file",
+  "name": "uncompress",
+  "extension": ""
+}]
+}

tests/test_resource.py

@@ -1325,6 +1325,13 @@ def test_VirtualCodebase_scanning_full_root(self):
         assert "/Users/sesser/code/nexb/scancode-toolkit/samples/README" == resource.path
         assert 1 == codebase.compute_counts()[0]
 
+    def test_VirtualCodebase_can_compute_counts_witrh_null(self):
+        # was failing with 
+        # size_count += child.size
+        # TypeError: unsupported operand type(s) for +=: 'int' and 'NoneType'
+        test_file = self.get_test_loc("resource/virtual_codebase/node-16-slim.json")
+        codebase = VirtualCodebase(test_file)
+        codebase.compute_counts()
 
 class TestResource(FileBasedTesting):
     test_data_dir = join(dirname(__file__), 'data')