Skip to content

Commit 452c2b5

Browse files
pombredannepombredanne
authored
Merge pull request #24 from nexB/23-recursion-error
Do not trigger a RecursionError #23
2 parents 4b627bd + 02ec56d commit 452c2b5

6 files changed

Lines changed: 371 additions & 10 deletions

File tree

‎CHANGELOG.rst‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,14 @@ vNext
55
-----
66

77

8+
Version 21.6.11
9+
---------------
10+
11+
- Do not fail if a Codebase does not have a common shared root #23
12+
- Consider all Resource attributes when building a VirtualCodebase #23
13+
- Do not ignore by default sccs and rcs dirs https://github.com/nexB/scancode-toolkit/issues/1422
14+
15+
816
Version 21.6.10
917
---------------
1018

‎src/commoncode/ignore.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -184,7 +184,6 @@ def get_ignores(location, include_defaults=True):
184184

185185
'CVS': 'Default ignore: CVS artifact',
186186
'.cvsignore': 'Default ignore: CVS config artifact',
187-
'*/SCCS': 'Default ignore: CVS artifact',
188187

189188
'*/_MTN': 'Default ignore: Monotone artifact',
190189
'*/_darcs': 'Default ignore: Darcs artifact',

‎src/commoncode/resource.py‎

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
import posixpath
1313
import traceback
1414
import sys
15+
from pathlib import Path
1516

1617
from collections import deque
1718
from functools import partial
@@ -1530,7 +1531,7 @@ def _get_or_create_parent(self, path, parent_by_path):
15301531
Note: the root path and root Resource must already be in
15311532
`parent_by_path` or else this function does not work.
15321533
"""
1533-
parent_path = parent_directory(path).rstrip('/').rstrip('\\').lstrip("/")
1534+
parent_path = parent_directory(path).rstrip('/').rstrip('\\').lstrip('/')
15341535
existing_parent = parent_by_path.get(parent_path)
15351536
if existing_parent:
15361537
return existing_parent
@@ -1548,6 +1549,13 @@ def _get_or_create_parent(self, path, parent_by_path):
15481549
parent_by_path[parent_path] = parent_resource
15491550
return parent_resource
15501551

1552+
def _set_new_root_directory(self, resources_data, new_root_directory_path):
1553+
for resource_data in resources_data:
1554+
resource_path = Path(resource_data['path'])
1555+
new_resource_path = Path(new_root_directory_path)
1556+
new_resource_path = new_resource_path.joinpath(resource_path)
1557+
resource_data['path'] = str(new_resource_path)
1558+
15511559
def _populate(self, scan_data):
15521560
"""
15531561
Populate this codebase with Resource objects.
@@ -1645,6 +1653,21 @@ def _populate(self, scan_data):
16451653
sample_resource_path = sample_resource_data['path']
16461654
sample_resource_path = sample_resource_path.strip('/')
16471655
root_path = sample_resource_path.split('/')[0]
1656+
1657+
# Check to see if the Resources from the scan we received has a common root directory.
1658+
for resource_data in resources_data:
1659+
resource_path = resource_data.get('path')
1660+
resource_path = resource_path.strip('/')
1661+
resource_root_path = resource_path.split('/')[0]
1662+
# If not, set a common root directory for all Resources.
1663+
if resource_root_path != root_path:
1664+
self._set_new_root_directory(
1665+
resources_data=resources_data,
1666+
new_root_directory_path='virtual_root'
1667+
)
1668+
root_path = 'virtual_root'
1669+
break
1670+
16481671
root_name = root_path
16491672
root_is_file = False
16501673
root_data = self._create_empty_resource_data()

‎tests/data/resource/virtual_codebase/docker-hello-world.json‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
{
2+
"headers": [
3+
{
4+
"tool_name": "scanpipe",
5+
"tool_version": "21.5.12",
6+
"notice": "Generated with ScanCode.io and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied.\nNo content created from ScanCode.io should be considered or used as legal advice.\nConsult an Attorney for any legal advice.\nScanCode.io is a free software code scanning tool from nexB Inc. and others\nlicensed under the Apache License version 2.0.\nScanCode is a trademark of nexB Inc.\nVisit https://github.com/nexB/scancode.io for support and download.\n",
7+
"uuid": "2f8e9a35-048e-4d9f-ae56-feea8863d6eb",
8+
"created_date": "2021-06-10T13:56:44.418Z",
9+
"input_files": [
10+
"hello_world.tar"
11+
],
12+
"runs": [
13+
{
14+
"pipeline_name": "docker",
15+
"description": "A pipeline to analyze a Docker image.",
16+
"uuid": "106296b1-6334-4393-9f1a-209459954812",
17+
"created_date": "2021-06-10T13:56:44.421302Z",
18+
"task_id": "c851f2e7-7cc1-4952-8d93-1cdee4a147bd",
19+
"task_start_date": "2021-06-10T13:56:44.434313Z",
20+
"task_end_date": "2021-06-10T13:56:44.819773Z",
21+
"task_exitcode": 0,
22+
"task_output": "",
23+
"log": "2021-06-10 13:56:44.43 Pipeline [docker] starting\n2021-06-10 13:56:44.43 Step [extract_images] starting\n2021-06-10 13:56:44.44 Step [extract_images] completed in 0.00 seconds\n2021-06-10 13:56:44.44 Step [extract_layers] starting\n2021-06-10 13:56:44.44 Step [extract_layers] completed in 0.00 seconds\n2021-06-10 13:56:44.45 Step [find_images_linux_distro] starting\n2021-06-10 13:56:44.45 Step [find_images_linux_distro] completed in 0.00 seconds\n2021-06-10 13:56:44.45 Step [collect_images_information] starting\n2021-06-10 13:56:44.45 Step [collect_images_information] completed in 0.00 seconds\n2021-06-10 13:56:44.46 Step [collect_and_create_codebase_resources] starting\n2021-06-10 13:56:44.46 Step [collect_and_create_codebase_resources] completed in 0.01 seconds\n2021-06-10 13:56:44.47 Step [collect_and_create_system_packages] starting\n2021-06-10 13:56:44.47 Step [collect_and_create_system_packages] completed in 0.00 seconds\n2021-06-10 13:56:44.47 Step [tag_uninteresting_codebase_resources] starting\n2021-06-10 13:56:44.48 Step [tag_uninteresting_codebase_resources] completed in 0.00 seconds\n2021-06-10 13:56:44.48 Step [tag_empty_files] starting\n2021-06-10 13:56:44.48 Step [tag_empty_files] completed in 0.00 seconds\n2021-06-10 13:56:44.48 Step [scan_for_application_packages] starting\n2021-06-10 13:56:44.53 Step [scan_for_application_packages] completed in 0.05 seconds\n2021-06-10 13:56:44.54 Step [scan_for_files] starting\n2021-06-10 13:56:44.80 Step [scan_for_files] completed in 0.26 seconds\n2021-06-10 13:56:44.80 Step [analyze_scanned_files] starting\n2021-06-10 13:56:44.81 Step [analyze_scanned_files] completed in 0.00 seconds\n2021-06-10 13:56:44.81 Step [tag_not_analyzed_codebase_resources] starting\n2021-06-10 13:56:44.81 Step [tag_not_analyzed_codebase_resources] completed in 0.00 seconds\n2021-06-10 13:56:44.81 Pipeline completed\n",
24+
"execution_time": 0
25+
}
26+
],
27+
"extra_data": {
28+
"images": [
29+
{
30+
"os": "linux",
31+
"tags": [],
32+
"author": null,
33+
"distro": null,
34+
"labels": {},
35+
"sha256": null,
36+
"comment": null,
37+
"created": "2021-03-05T23:25:25.230064203Z",
38+
"history": [
39+
{
40+
"created": "2021-03-05T23:25:25.052495292Z",
41+
"created_by": "/bin/sh -c #(nop) COPY file:7bf12aab75c3867a023fe3b8bd6d113d43a4fcc415f3cc27cbcf0fff37b65a02 in / "
42+
},
43+
{
44+
"created": "2021-03-05T23:25:25.230064203Z",
45+
"created_by": "/bin/sh -c #(nop) CMD [\"/hello\"]",
46+
"empty_layer": true
47+
}
48+
],
49+
"variant": null,
50+
"image_id": "d1165f2212346b2bab48cb01c1e39ee8ad1be46b87873d9ca7a4e434980a7726",
51+
"os_version": null,
52+
"architecture": "amd64",
53+
"image_format": "docker",
54+
"config_digest": "sha256:d1165f2212346b2bab48cb01c1e39ee8ad1be46b87873d9ca7a4e434980a7726",
55+
"docker_version": "19.03.12"
56+
}
57+
]
58+
}
59+
}],
60+
"packages": [
61+
],
62+
"files": [
63+
{
64+
"for_packages": [],
65+
"compliance_alert": "",
66+
"path": "/hello_world.tar-extract/f22b99068db93900abe17f7f5e09ec775c2826ecfe9db961fea68293744144bd/hello",
67+
"size": 13336,
68+
"sha1": "ab2ceef6fe98ef7acd088f5d79034dc859ff0d27",
69+
"md5": "773e022bbbd0b4cc88edb6c7d12daf5d",
70+
"copyrights": [],
71+
"holders": [],
72+
"authors": [],
73+
"licenses": [],
74+
"license_expressions": [],
75+
"emails": [],
76+
"urls": [
77+
{
78+
"url": "https://hub.docker.com/",
79+
"end_line": 19,
80+
"start_line": 19
81+
},
82+
{
83+
"url": "https://docs.docker.com/get-started/",
84+
"end_line": 21,
85+
"start_line": 21
86+
}
87+
],
88+
"status": "no-licenses",
89+
"type": "file",
90+
"extra_data": {},
91+
"name": "hello",
92+
"extension": "",
93+
"programming_language": "",
94+
"mime_type": "application/x-executable",
95+
"file_type": "ELF 64-bit LSB executable, x86-64, version 1 (SYSV), statically linked, stripped",
96+
"is_binary": true,
97+
"is_text": false,
98+
"is_archive": false
99+
}]
100+
}

‎tests/data/resource/virtual_codebase/zephyr-binary.json‎

Lines changed: 220 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,220 @@
1+
{
2+
"headers": [
3+
{
4+
"tool_name": "scanpipe",
5+
"tool_version": "21.5.12",
6+
"notice": "Generated with ScanCode.io and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied.\nNo content created from ScanCode.io should be considered or used as legal advice.\nConsult an Attorney for any legal advice.\nScanCode.io is a free software code scanning tool from nexB Inc. and others\nlicensed under the Apache License version 2.0.\nScanCode is a trademark of nexB Inc.\nVisit https://github.com/nexB/scancode.io for support and download.\n",
7+
"uuid": "d0d9bf3d-980a-4822-ac5b-e4949470499d",
8+
"created_date": "2021-06-10T23:17:30.522Z",
9+
"input_files": [
10+
"SBOM-Plugfest.zip"
11+
],
12+
"runs": [
13+
{
14+
"pipeline_name": "scan_codebase",
15+
"description": "A pipeline to scan a codebase with ScanCode-toolkit.\n\nThe input files are copied to the project codebase/ directory and extracted\nin place before running the scan.\nAlternatively, the code can be manually copied to the project codebase/\ndirectory.",
16+
"uuid": "13efbd78-3921-4895-b4bd-e0b393b92ec3",
17+
"created_date": "2021-06-10T23:17:30.526728Z",
18+
"task_id": "f68d383a-4910-4e05-ad64-82ef1f0b5888",
19+
"task_start_date": "2021-06-10T23:17:30.546024Z",
20+
"task_end_date": "2021-06-10T23:17:36.396574Z",
21+
"task_exitcode": 0,
22+
"task_output": "",
23+
"log": "2021-06-10 23:17:30.54 Pipeline [scan_codebase] starting\n2021-06-10 23:17:30.54 Step [copy_inputs_to_codebase_directory] starting\n2021-06-10 23:17:30.55 Step [copy_inputs_to_codebase_directory] completed in 0.00 seconds\n2021-06-10 23:17:30.55 Step [run_extractcode] starting\n2021-06-10 23:17:30.83 Step [run_extractcode] completed in 0.28 seconds\n2021-06-10 23:17:30.84 Step [run_scancode] starting\n2021-06-10 23:17:36.33 Step [run_scancode] completed in 5.49 seconds\n2021-06-10 23:17:36.34 Step [build_inventory_from_scan] starting\n2021-06-10 23:17:36.37 Step [build_inventory_from_scan] completed in 0.03 seconds\n2021-06-10 23:17:36.37 Step [csv_output] starting\n2021-06-10 23:17:36.39 Step [csv_output] completed in 0.02 seconds\n2021-06-10 23:17:36.39 Pipeline completed\n",
24+
"execution_time": 5
25+
}
26+
],
27+
"extra_data": {}
28+
}],
29+
"packages": [
30+
],
31+
"files": [
32+
{
33+
"for_packages": [],
34+
"compliance_alert": "",
35+
"path": "SBOM-Plugfest.zip",
36+
"size": 1762970,
37+
"sha1": "db3fa3422912685484860aec79960f3ef3de95a8",
38+
"md5": "8a9423ff930d0afd9399600f351b0507",
39+
"copyrights": [],
40+
"holders": [],
41+
"authors": [],
42+
"licenses": [],
43+
"license_expressions": [],
44+
"emails": [],
45+
"urls": [],
46+
"status": "",
47+
"type": "file",
48+
"extra_data": {},
49+
"name": "SBOM-Plugfest.zip",
50+
"extension": ".zip",
51+
"programming_language": "",
52+
"mime_type": "application/zip",
53+
"file_type": "Zip archive data, at least v2.0 to extract",
54+
"is_binary": true,
55+
"is_text": false,
56+
"is_archive": true
57+
},
58+
{
59+
"for_packages": [],
60+
"compliance_alert": "",
61+
"path": "SBOM-Plugfest.zip-extract",
62+
"size": 0,
63+
"sha1": "",
64+
"md5": "",
65+
"copyrights": [],
66+
"holders": [],
67+
"authors": [],
68+
"licenses": [],
69+
"license_expressions": [],
70+
"emails": [],
71+
"urls": [],
72+
"status": "",
73+
"type": "directory",
74+
"extra_data": {},
75+
"name": "SBOM-Plugfest.zip-extract",
76+
"extension": "",
77+
"programming_language": "",
78+
"mime_type": "",
79+
"file_type": "",
80+
"is_binary": false,
81+
"is_text": false,
82+
"is_archive": false
83+
},
84+
{
85+
"for_packages": [],
86+
"compliance_alert": "",
87+
"path": "SBOM-Plugfest.zip-extract/SBOM-Plugfest",
88+
"size": 0,
89+
"sha1": "",
90+
"md5": "",
91+
"copyrights": [],
92+
"holders": [],
93+
"authors": [],
94+
"licenses": [],
95+
"license_expressions": [],
96+
"emails": [],
97+
"urls": [],
98+
"status": "",
99+
"type": "directory",
100+
"extra_data": {},
101+
"name": "SBOM-Plugfest",
102+
"extension": "",
103+
"programming_language": "",
104+
"mime_type": "",
105+
"file_type": "",
106+
"is_binary": false,
107+
"is_text": false,
108+
"is_archive": false
109+
},
110+
{
111+
"for_packages": [],
112+
"compliance_alert": "",
113+
"path": "SBOM-Plugfest.zip-extract/SBOM-Plugfest/Readme.md",
114+
"size": 238,
115+
"sha1": "b3edec38166cb4747ce67e7e229cbe9279e011c3",
116+
"md5": "2c2fdcabb9e3a46bbc60f9e235ec0389",
117+
"copyrights": [],
118+
"holders": [],
119+
"authors": [],
120+
"licenses": [],
121+
"license_expressions": [],
122+
"emails": [],
123+
"urls": [
124+
{
125+
"url": "https://github.com/LibreSolar/charge-controller-firmware/tree/sbom-plugfest",
126+
"end_line": 7,
127+
"start_line": 7
128+
}
129+
],
130+
"status": "",
131+
"type": "file",
132+
"extra_data": {},
133+
"name": "Readme.md",
134+
"extension": ".md",
135+
"programming_language": "",
136+
"mime_type": "text/plain",
137+
"file_type": "ASCII text",
138+
"is_binary": false,
139+
"is_text": true,
140+
"is_archive": false
141+
},
142+
{
143+
"for_packages": [],
144+
"compliance_alert": "",
145+
"path": "SBOM-Plugfest.zip-extract/SBOM-Plugfest/zephyr.bin",
146+
"size": 95600,
147+
"sha1": "3e5510346a723c4ecdd698330436d1d89f2e6428",
148+
"md5": "fdc8b20a54f7960b204abce20f8dae98",
149+
"copyrights": [],
150+
"holders": [],
151+
"authors": [],
152+
"licenses": [],
153+
"license_expressions": [],
154+
"emails": [],
155+
"urls": [],
156+
"status": "",
157+
"type": "file",
158+
"extra_data": {},
159+
"name": "zephyr.bin",
160+
"extension": ".bin",
161+
"programming_language": "",
162+
"mime_type": "application/octet-stream",
163+
"file_type": "data",
164+
"is_binary": true,
165+
"is_text": false,
166+
"is_archive": false
167+
},
168+
{
169+
"for_packages": [],
170+
"compliance_alert": "",
171+
"path": "SBOM-Plugfest.zip-extract/SBOM-Plugfest/zephyr.elf",
172+
"size": 1397388,
173+
"sha1": "a0c9fca9a551a935026c86fb56c4af563f0a926f",
174+
"md5": "b7f937d8a1fbd71a0a48f16c845ed36e",
175+
"copyrights": [],
176+
"holders": [],
177+
"authors": [],
178+
"licenses": [],
179+
"license_expressions": [],
180+
"emails": [],
181+
"urls": [],
182+
"status": "",
183+
"type": "file",
184+
"extra_data": {},
185+
"name": "zephyr.elf",
186+
"extension": ".elf",
187+
"programming_language": "",
188+
"mime_type": "application/x-executable",
189+
"file_type": "ELF 32-bit LSB executable, ARM, EABI5 version 1 (SYSV), statically linked, with debug_info, not stripped",
190+
"is_binary": true,
191+
"is_text": false,
192+
"is_archive": false
193+
},
194+
{
195+
"for_packages": [],
196+
"compliance_alert": "",
197+
"path": "SBOM-Plugfest.zip-extract/SBOM-Plugfest/zephyr.hex",
198+
"size": 269060,
199+
"sha1": "c70fc409d4003e437ea2e84d9546de8083333ab2",
200+
"md5": "2505d99bb160d8b05e8892026209923d",
201+
"copyrights": [],
202+
"holders": [],
203+
"authors": [],
204+
"licenses": [],
205+
"license_expressions": [],
206+
"emails": [],
207+
"urls": [],
208+
"status": "",
209+
"type": "file",
210+
"extra_data": {},
211+
"name": "zephyr.hex",
212+
"extension": ".hex",
213+
"programming_language": "",
214+
"mime_type": "text/plain",
215+
"file_type": "ASCII text, with CRLF line terminators",
216+
"is_binary": false,
217+
"is_text": true,
218+
"is_archive": false
219+
}]
220+
}

0 commit comments

Comments
 (0)