From ae01634b07ac584b73239783aa249e05472e27f2 Mon Sep 17 00:00:00 2001 From: Adrien CABARBAYE Date: Fri, 2 Oct 2026 22:05:05 +0100 Subject: [PATCH 1/8] :bug: Auto-download Go module dependencies for licensing --- continuous_delivery_scripts/plugins/golang.py | 13 ++++++++++++ news/20261002191514321000000.bugfix | 1 + tests/plugin/test_go_licensing.py | 21 +++++++++++++++++-- 3 files changed, 33 insertions(+), 2 deletions(-) create mode 100644 news/20261002191514321000000.bugfix diff --git a/continuous_delivery_scripts/plugins/golang.py b/continuous_delivery_scripts/plugins/golang.py index 918ce2a..77c7fed 100644 --- a/continuous_delivery_scripts/plugins/golang.py +++ b/continuous_delivery_scripts/plugins/golang.py @@ -135,6 +135,18 @@ def _go_module_name(module_dir: Path, env: MutableMapping[str, str]) -> str: return str(module["Path"]) +def _download_go_module_dependencies(module_directories: List[Path], env: MutableMapping[str, str]) -> None: + """Download Go dependencies for each detected module. + + `go-licenses` expects the module graph to be available locally. Fetching the + dependencies here means callers do not need a separate CI step before + running `cd-check-licence-compliance` or `cd-generate-spdx`. + """ + for module_directory in module_directories: + logger.info("Downloading Go module dependencies in [%s].", module_directory) + check_call(["go", "mod", "download", "all"], cwd=module_directory, env=env) + + def _parse_go_licences(output: str) -> List[PackageMetadata]: """Translate go-licenses template output into shared package metadata.""" packages = [] @@ -235,6 +247,7 @@ def fetch_project_metadata(self) -> ProjectMetadata: module_directories = _go_licence_module_directories() env = os.environ.copy() env[ENVVAR_GO_MOD] = GO_MOD_ON_VALUE + _download_go_module_dependencies(module_directories, env) executable = _ensure_go_tool_installed( tool_name="go-licenses", version_command=["go-licenses", "report", "--help"], diff --git a/news/20261002191514321000000.bugfix b/news/20261002191514321000000.bugfix new file mode 100644 index 0000000..99cfcb3 --- /dev/null +++ b/news/20261002191514321000000.bugfix @@ -0,0 +1 @@ +:bug: Automatically run `go mod download all` for detected Go modules before licence and SPDX dependency analysis so CI does not need a separate pre-download step. diff --git a/tests/plugin/test_go_licensing.py b/tests/plugin/test_go_licensing.py index 20dda28..faf30ae 100644 --- a/tests/plugin/test_go_licensing.py +++ b/tests/plugin/test_go_licensing.py @@ -169,11 +169,25 @@ def test_stages_root_licence_for_modules_without_one_and_cleans_up(self): self.assertFalse((module_without_licence / "LICENSE").exists()) self.assertEqual(existing_licence.read_text(encoding="utf8"), "module licence") + @mock.patch.object(golang, "check_call") + def test_downloads_go_module_dependencies_for_each_module(self, check_call): + modules = [Path("module-a"), Path("module-b")] + env = {"GO111MODULE": "on"} + + golang._download_go_module_dependencies(modules, env) + + check_call.assert_any_call(["go", "mod", "download", "all"], cwd=Path("module-a"), env=env) + check_call.assert_any_call(["go", "mod", "download", "all"], cwd=Path("module-b"), env=env) + self.assertEqual(check_call.call_count, 2) + @mock.patch.object(golang, "_go_licence_module_directories") + @mock.patch.object(golang, "_download_go_module_dependencies") @mock.patch.object(golang, "_ensure_go_tool_installed", return_value="go-licenses") @mock.patch.object(golang, "check_output") @mock.patch.object(golang, "run") - def test_go_project_uses_shared_metadata(self, run_mock, check_output, ensure_installed, module_directories): + def test_go_project_uses_shared_metadata( + self, run_mock, check_output, ensure_installed, download_dependencies, module_directories + ): with TemporaryDirectory() as temp_dir: module_directories.return_value = [Path(temp_dir)] @@ -223,14 +237,16 @@ def run_side_effect(command, **kwargs): self.assertEqual(project.dependencies_metadata[0].licence, "MIT") self.assertEqual(project.dependencies_metadata[1].licence_source, "unknown") self.assertEqual(project.dependencies_metadata[1].licence_candidates, ["Apache-2.0", "BSD-3-Clause", "MIT"]) + download_dependencies.assert_called_once() ensure_installed.assert_called_once() @mock.patch.object(golang, "_go_licence_module_directories") + @mock.patch.object(golang, "_download_go_module_dependencies") @mock.patch.object(golang, "_ensure_go_tool_installed", return_value="go-licenses") @mock.patch.object(golang, "check_output") @mock.patch.object(golang, "run") def test_go_project_temporarily_stages_root_licence_for_submodules( - self, run_mock, check_output, _ensure_installed, module_directories + self, run_mock, check_output, _ensure_installed, download_dependencies, module_directories ): with TemporaryDirectory() as temp_dir: root = Path(temp_dir) @@ -272,6 +288,7 @@ def run_side_effect(command, **kwargs): self.assertEqual([package.name for package in project.dependencies_metadata], ["github.com/acme/dep"]) self.assertFalse((module / "LICENSE").exists()) + download_dependencies.assert_called_once() def test_go_import_paths_have_safe_spdx_filenames_and_ids(self): package = SpdxPackage( From 004d916a4c92e34f1e853b72a04893af44c5d330 Mon Sep 17 00:00:00 2001 From: Adrien CABARBAYE Date: Fri, 2 Oct 2026 22:07:57 +0100 Subject: [PATCH 2/8] :sparkles: Add opt-out for Go dependency prefetch --- continuous_delivery_scripts/plugins/golang.py | 13 ++- .../utils/configuration.py | 3 + news/20261002191514321000000.bugfix | 2 +- tests/plugin/test_go_licensing.py | 80 ++++++++++++++++++- 4 files changed, 94 insertions(+), 4 deletions(-) diff --git a/continuous_delivery_scripts/plugins/golang.py b/continuous_delivery_scripts/plugins/golang.py index 77c7fed..7af2293 100644 --- a/continuous_delivery_scripts/plugins/golang.py +++ b/continuous_delivery_scripts/plugins/golang.py @@ -147,6 +147,14 @@ def _download_go_module_dependencies(module_directories: List[Path], env: Mutabl check_call(["go", "mod", "download", "all"], cwd=module_directory, env=env) +def _should_download_go_module_dependencies() -> bool: + """Return whether Go module dependencies should be prefetched automatically.""" + configured = configuration.get_value_or_default(ConfigurationVariable.SKIP_GO_MODULE_DOWNLOAD_FOR_LICENSING, False) + if isinstance(configured, str): + configured = configured.strip().lower() in ("true", "1", "yes", "on") + return not bool(configured) + + def _parse_go_licences(output: str) -> List[PackageMetadata]: """Translate go-licenses template output into shared package metadata.""" packages = [] @@ -247,7 +255,10 @@ def fetch_project_metadata(self) -> ProjectMetadata: module_directories = _go_licence_module_directories() env = os.environ.copy() env[ENVVAR_GO_MOD] = GO_MOD_ON_VALUE - _download_go_module_dependencies(module_directories, env) + if _should_download_go_module_dependencies(): + _download_go_module_dependencies(module_directories, env) + else: + logger.info("Skipping Go module dependency downloads before licence analysis by configuration.") executable = _ensure_go_tool_installed( tool_name="go-licenses", version_command=["go-licenses", "report", "--help"], diff --git a/continuous_delivery_scripts/utils/configuration.py b/continuous_delivery_scripts/utils/configuration.py index 1f43f8d..e563cff 100644 --- a/continuous_delivery_scripts/utils/configuration.py +++ b/continuous_delivery_scripts/utils/configuration.py @@ -109,6 +109,8 @@ class ConfigurationVariable(enum.Enum): """Project-specific records explaining manual reviews of REVIEW assessments.""" LICENCE_ASSESSMENT_FAIL_ON = 45 """Optional top-level list of assessment statuses that fail compliance checks.""" + SKIP_GO_MODULE_DOWNLOAD_FOR_LICENSING = 46 + """Skip automatic `go mod download all` before Go licence and SPDX dependency analysis.""" @staticmethod def choices() -> List[str]: @@ -219,6 +221,7 @@ class StaticConfig(GenericConfig): LICENCE_ASSESSMENT_FAIL_ON = None """When unset, use fail_on from the embedded or project licence assessment rules.""" GENERATE_LICENSING_SUMMARY_ON_RELEASE = False + SKIP_GO_MODULE_DOWNLOAD_FOR_LICENSING = False DEPENDENCY_UPDATE_NEWS_MESSAGE = "Dependency upgrade: {message}" DEPENDENCY_UPDATE_NEWS_TYPE = NewsType.bugfix DEPENDENCY_UPDATE_BRANCH_PATTERN = r"^\s*[Dd]ependabot\/.+\/(?P.+)" diff --git a/news/20261002191514321000000.bugfix b/news/20261002191514321000000.bugfix index 99cfcb3..be5055e 100644 --- a/news/20261002191514321000000.bugfix +++ b/news/20261002191514321000000.bugfix @@ -1 +1 @@ -:bug: Automatically run `go mod download all` for detected Go modules before licence and SPDX dependency analysis so CI does not need a separate pre-download step. +:bug: Automatically run `go mod download all` for detected Go modules before licence and SPDX dependency analysis, with an opt-out flag for CI setups that already prefetch dependencies. diff --git a/tests/plugin/test_go_licensing.py b/tests/plugin/test_go_licensing.py index faf30ae..753ee9b 100644 --- a/tests/plugin/test_go_licensing.py +++ b/tests/plugin/test_go_licensing.py @@ -180,13 +180,42 @@ def test_downloads_go_module_dependencies_for_each_module(self, check_call): check_call.assert_any_call(["go", "mod", "download", "all"], cwd=Path("module-b"), env=env) self.assertEqual(check_call.call_count, 2) + def test_download_go_module_dependencies_enabled_by_default(self): + get_value = configuration.get_value + + with mock.patch.object( + configuration, + "get_value", + side_effect=lambda key: get_value(key), + ): + self.assertTrue(golang._should_download_go_module_dependencies()) + + def test_download_go_module_dependencies_can_be_skipped_from_configuration(self): + get_value = configuration.get_value + + with mock.patch.object( + configuration, + "get_value", + side_effect=lambda key: ( + "true" if key == ConfigurationVariable.SKIP_GO_MODULE_DOWNLOAD_FOR_LICENSING else get_value(key) + ), + ): + self.assertFalse(golang._should_download_go_module_dependencies()) + @mock.patch.object(golang, "_go_licence_module_directories") + @mock.patch.object(golang, "_should_download_go_module_dependencies", return_value=True) @mock.patch.object(golang, "_download_go_module_dependencies") @mock.patch.object(golang, "_ensure_go_tool_installed", return_value="go-licenses") @mock.patch.object(golang, "check_output") @mock.patch.object(golang, "run") def test_go_project_uses_shared_metadata( - self, run_mock, check_output, ensure_installed, download_dependencies, module_directories + self, + run_mock, + check_output, + ensure_installed, + download_dependencies, + _should_download, + module_directories, ): with TemporaryDirectory() as temp_dir: module_directories.return_value = [Path(temp_dir)] @@ -241,12 +270,19 @@ def run_side_effect(command, **kwargs): ensure_installed.assert_called_once() @mock.patch.object(golang, "_go_licence_module_directories") + @mock.patch.object(golang, "_should_download_go_module_dependencies", return_value=True) @mock.patch.object(golang, "_download_go_module_dependencies") @mock.patch.object(golang, "_ensure_go_tool_installed", return_value="go-licenses") @mock.patch.object(golang, "check_output") @mock.patch.object(golang, "run") def test_go_project_temporarily_stages_root_licence_for_submodules( - self, run_mock, check_output, _ensure_installed, download_dependencies, module_directories + self, + run_mock, + check_output, + _ensure_installed, + download_dependencies, + _should_download, + module_directories, ): with TemporaryDirectory() as temp_dir: root = Path(temp_dir) @@ -290,6 +326,46 @@ def run_side_effect(command, **kwargs): self.assertFalse((module / "LICENSE").exists()) download_dependencies.assert_called_once() + @mock.patch.object(golang, "_go_licence_module_directories") + @mock.patch.object(golang, "_should_download_go_module_dependencies", return_value=False) + @mock.patch.object(golang, "_download_go_module_dependencies") + @mock.patch.object(golang, "_ensure_go_tool_installed", return_value="go-licenses") + @mock.patch.object(golang, "check_output", return_value='{"Path": "example.com/acme"}') + @mock.patch.object( + golang, + "run", + return_value=subprocess.CompletedProcess( + args=["go-licenses"], + returncode=0, + stdout="github.com/acme/dep\tv1.2.3\thttps://example.com/LICENSE\tMIT\n", + stderr="", + ), + ) + def test_go_project_can_skip_dependency_prefetch_when_configured( + self, + _run, + _check_output, + _ensure_installed, + download_dependencies, + _should_download, + module_directories, + ): + with TemporaryDirectory() as temp_dir: + module_directories.return_value = [Path(temp_dir)] + get_value = configuration.get_value + with mock.patch.object( + configuration, + "get_value", + side_effect=lambda key: ( + "Acme" + if key == ConfigurationVariable.PROJECT_NAME + else "Apache-2.0" if key == ConfigurationVariable.FILE_LICENCE_IDENTIFIER else get_value(key) + ), + ): + golang.GoProjectMetadataFetcher().project_metadata + + download_dependencies.assert_not_called() + def test_go_import_paths_have_safe_spdx_filenames_and_ids(self): package = SpdxPackage( PackageInfo( From 2cb721eb8a366825d0eed7076655aae982128dbd Mon Sep 17 00:00:00 2001 From: Adrien CABARBAYE Date: Fri, 2 Oct 2026 22:10:20 +0100 Subject: [PATCH 3/8] :sparkles: Add CLI opt-out for Go dependency prefetch --- .../check_licence_compliance.py | 12 ++++++++++++ .../report_third_party_ip.py | 13 +++++++++++++ news/20261002191514321000000.bugfix | 2 +- tests/spdx/test_check_licence_compliance.py | 11 +++++++++++ tests/spdx/test_generate_spdx_cli.py | 18 ++++++++++++++++++ 5 files changed, 55 insertions(+), 1 deletion(-) diff --git a/continuous_delivery_scripts/check_licence_compliance.py b/continuous_delivery_scripts/check_licence_compliance.py index 7fb48fe..f504341 100644 --- a/continuous_delivery_scripts/check_licence_compliance.py +++ b/continuous_delivery_scripts/check_licence_compliance.py @@ -6,6 +6,7 @@ import argparse import logging +import os import sys from pathlib import Path from typing import TYPE_CHECKING @@ -22,6 +23,11 @@ logger = logging.getLogger(__name__) +def _apply_go_module_download_options(skip_go_module_download: bool) -> None: + if skip_go_module_download: + os.environ[ConfigurationVariable.SKIP_GO_MODULE_DOWNLOAD_FOR_LICENSING.name] = "true" + + def _log_assessment_details(project: "SpdxProject") -> None: """Log per-dependency assessment details for statuses that need attention.""" if not logger.isEnabledFor(logging.WARNING): @@ -90,9 +96,15 @@ def main() -> int: action="store_true", help="Consult ScanCode LicenseDB for unclassified licences or missing assessment rules.", ) + parser.add_argument( + "--skip-go-module-download", + action="store_true", + help="Skip automatic `go mod download all` before Go licence checks and SPDX dependency analysis.", + ) parser.add_argument("-v", "--verbose", action="count", default=0, help="Increase logging verbosity.") args = parser.parse_args() set_log_level(args.verbose) + _apply_go_module_download_options(args.skip_go_module_download) try: plugin = get_language_specifics() diff --git a/continuous_delivery_scripts/report_third_party_ip.py b/continuous_delivery_scripts/report_third_party_ip.py index 056a884..e2cf5ed 100644 --- a/continuous_delivery_scripts/report_third_party_ip.py +++ b/continuous_delivery_scripts/report_third_party_ip.py @@ -14,11 +14,13 @@ import argparse import logging +import os import sys from pathlib import Path from typing import TYPE_CHECKING, Any, Optional from continuous_delivery_scripts.language_specifics import get_language_specifics +from continuous_delivery_scripts.utils.configuration import ConfigurationVariable from continuous_delivery_scripts.utils.logging import set_log_level, log_exception if TYPE_CHECKING: @@ -27,6 +29,11 @@ logger = logging.getLogger(__name__) +def _apply_go_module_download_options(skip_go_module_download: bool) -> None: + if skip_go_module_download: + os.environ[ConfigurationVariable.SKIP_GO_MODULE_DOWNLOAD_FOR_LICENSING.name] = "true" + + def generate_spdx_project_reports(project: "SpdxProject", output_directory: Path) -> "SpdxProject": """Generates all the SPDX reports for a given project.""" logger.info("Generating SPDX report.") @@ -66,6 +73,11 @@ def convert_to_path(arg: Any) -> Path: action="store_true", help="Consult ScanCode LicenseDB for unclassified licences or missing assessment rules.", ) + parser.add_argument( + "--skip-go-module-download", + action="store_true", + help="Skip automatic `go mod download all` before Go licence checks and SPDX dependency analysis.", + ) parser.add_argument( "-v", @@ -76,6 +88,7 @@ def convert_to_path(arg: Any) -> Path: ) args = parser.parse_args() set_log_level(args.verbose) + _apply_go_module_download_options(args.skip_go_module_download) try: if get_language_specifics().can_get_project_metadata(): diff --git a/news/20261002191514321000000.bugfix b/news/20261002191514321000000.bugfix index be5055e..1ed195c 100644 --- a/news/20261002191514321000000.bugfix +++ b/news/20261002191514321000000.bugfix @@ -1 +1 @@ -:bug: Automatically run `go mod download all` for detected Go modules before licence and SPDX dependency analysis, with an opt-out flag for CI setups that already prefetch dependencies. +:bug: Automatically run `go mod download all` for detected Go modules before licence and SPDX dependency analysis, with config and command-line opt-outs for CI setups that already prefetch dependencies. diff --git a/tests/spdx/test_check_licence_compliance.py b/tests/spdx/test_check_licence_compliance.py index dcf06a4..9cee548 100644 --- a/tests/spdx/test_check_licence_compliance.py +++ b/tests/spdx/test_check_licence_compliance.py @@ -8,6 +8,7 @@ import csv from io import BytesIO, StringIO import json +import os from pathlib import Path from tempfile import TemporaryDirectory from unittest import TestCase @@ -174,3 +175,13 @@ def test_unsupported_plugin_does_not_silently_pass(self): self.assertEqual(self._run_command(), 1) self.plugin.get_current_spdx_project.assert_not_called() + + def test_skip_go_module_download_flag_sets_configuration_override(self): + with patch.dict(os.environ, {}, clear=False): + self.assertNotIn(ConfigurationVariable.SKIP_GO_MODULE_DOWNLOAD_FOR_LICENSING.name, os.environ) + self.assertEqual(self._run_command("--skip-go-module-download"), 0) + + self.assertEqual( + os.environ[ConfigurationVariable.SKIP_GO_MODULE_DOWNLOAD_FOR_LICENSING.name], + "true", + ) diff --git a/tests/spdx/test_generate_spdx_cli.py b/tests/spdx/test_generate_spdx_cli.py index 8e69f31..dc93bda 100644 --- a/tests/spdx/test_generate_spdx_cli.py +++ b/tests/spdx/test_generate_spdx_cli.py @@ -6,11 +6,13 @@ from contextlib import redirect_stderr from io import StringIO +import os from pathlib import Path from unittest import TestCase from unittest.mock import Mock, patch from continuous_delivery_scripts.report_third_party_ip import main +from continuous_delivery_scripts.utils.configuration import ConfigurationVariable class TestGenerateSpdxCli(TestCase): @@ -53,3 +55,19 @@ def test_failed_check_does_not_present_lookup_as_a_successful_remediation(self): project.scancode_follow_up_warnings.assert_not_called() self.assertNotIn("WARNING: dependency", output.getvalue()) + + def test_skip_go_module_download_flag_sets_configuration_override(self): + project = Mock() + plugin = Mock() + plugin.can_get_project_metadata.return_value = True + + with patch.dict(os.environ, {}, clear=False), patch( + "sys.argv", ["cd-generate-spdx", "-o", "reports", "--skip-go-module-download"] + ), patch( + "continuous_delivery_scripts.report_third_party_ip.get_language_specifics", return_value=plugin + ), patch( + "continuous_delivery_scripts.report_third_party_ip.generate_spdx_reports", return_value=project + ): + self.assertNotIn(ConfigurationVariable.SKIP_GO_MODULE_DOWNLOAD_FOR_LICENSING.name, os.environ) + self.assertEqual(main(), 0) + self.assertEqual(os.environ[ConfigurationVariable.SKIP_GO_MODULE_DOWNLOAD_FOR_LICENSING.name], "true") From c405b1c0663fa3e8d0ad587149ad549f11c8d4a8 Mon Sep 17 00:00:00 2001 From: Adrien CABARBAYE Date: Fri, 2 Oct 2026 22:14:09 +0100 Subject: [PATCH 4/8] :recycle: Pass Go dependency prefetch flag explicitly --- .../check_licence_compliance.py | 9 +-------- continuous_delivery_scripts/plugins/basic.py | 2 +- continuous_delivery_scripts/plugins/ci.py | 2 +- continuous_delivery_scripts/plugins/docker.py | 2 +- continuous_delivery_scripts/plugins/golang.py | 9 +++++---- continuous_delivery_scripts/plugins/noop.py | 2 +- continuous_delivery_scripts/plugins/python.py | 2 +- .../report_third_party_ip.py | 20 +++++++++---------- .../utils/language_specifics_base.py | 2 +- tests/spdx/test_check_licence_compliance.py | 11 ++-------- tests/spdx/test_generate_spdx_cli.py | 16 ++++++--------- 11 files changed, 29 insertions(+), 48 deletions(-) diff --git a/continuous_delivery_scripts/check_licence_compliance.py b/continuous_delivery_scripts/check_licence_compliance.py index f504341..5e7dac8 100644 --- a/continuous_delivery_scripts/check_licence_compliance.py +++ b/continuous_delivery_scripts/check_licence_compliance.py @@ -6,7 +6,6 @@ import argparse import logging -import os import sys from pathlib import Path from typing import TYPE_CHECKING @@ -23,11 +22,6 @@ logger = logging.getLogger(__name__) -def _apply_go_module_download_options(skip_go_module_download: bool) -> None: - if skip_go_module_download: - os.environ[ConfigurationVariable.SKIP_GO_MODULE_DOWNLOAD_FOR_LICENSING.name] = "true" - - def _log_assessment_details(project: "SpdxProject") -> None: """Log per-dependency assessment details for statuses that need attention.""" if not logger.isEnabledFor(logging.WARNING): @@ -104,13 +98,12 @@ def main() -> int: parser.add_argument("-v", "--verbose", action="count", default=0, help="Increase logging verbosity.") args = parser.parse_args() set_log_level(args.verbose) - _apply_go_module_download_options(args.skip_go_module_download) try: plugin = get_language_specifics() if not plugin.can_get_project_metadata(): raise ValueError("The selected language plugin cannot provide metadata for licence compliance checks.") - project = plugin.get_current_spdx_project() + project = plugin.get_current_spdx_project(skip_go_module_download=args.skip_go_module_download) if project is None: raise ValueError("The selected language plugin did not return a project for licence compliance checks.") if args.lookup_scancode: diff --git a/continuous_delivery_scripts/plugins/basic.py b/continuous_delivery_scripts/plugins/basic.py index 8610e9e..c9a8d48 100644 --- a/continuous_delivery_scripts/plugins/basic.py +++ b/continuous_delivery_scripts/plugins/basic.py @@ -51,6 +51,6 @@ def can_get_project_metadata(self) -> bool: """States whether project metadata can be retrieved.""" return False - def get_current_spdx_project(self) -> Optional["SpdxProject"]: + def get_current_spdx_project(self, skip_go_module_download: bool = False) -> Optional["SpdxProject"]: """Basic.""" return None diff --git a/continuous_delivery_scripts/plugins/ci.py b/continuous_delivery_scripts/plugins/ci.py index ed3bc37..94a5a70 100644 --- a/continuous_delivery_scripts/plugins/ci.py +++ b/continuous_delivery_scripts/plugins/ci.py @@ -61,7 +61,7 @@ def can_get_project_metadata(self) -> bool: """States whether project metadata can be retrieved.""" return False - def get_current_spdx_project(self) -> Optional["SpdxProject"]: + def get_current_spdx_project(self, skip_go_module_download: bool = False) -> Optional["SpdxProject"]: """Gets current SPDX description.""" # TODO return None diff --git a/continuous_delivery_scripts/plugins/docker.py b/continuous_delivery_scripts/plugins/docker.py index f45d4b0..1b3d7fe 100644 --- a/continuous_delivery_scripts/plugins/docker.py +++ b/continuous_delivery_scripts/plugins/docker.py @@ -53,7 +53,7 @@ def can_get_project_metadata(self) -> bool: """States whether project metadata can be retrieved.""" return False - def get_current_spdx_project(self) -> Optional["SpdxProject"]: + def get_current_spdx_project(self, skip_go_module_download: bool = False) -> Optional["SpdxProject"]: """Gets current SPDX description.""" # TODO return None diff --git a/continuous_delivery_scripts/plugins/golang.py b/continuous_delivery_scripts/plugins/golang.py index 7af2293..ea89178 100644 --- a/continuous_delivery_scripts/plugins/golang.py +++ b/continuous_delivery_scripts/plugins/golang.py @@ -246,16 +246,17 @@ def _stage_root_licence_files_for_modules(module_directories: List[Path]) -> Ite class GoProjectMetadataFetcher(ProjectMetadataFetcher): """Retrieve Go dependency licences for the shared SPDX and compliance report.""" - def __init__(self) -> None: + def __init__(self, skip_go_module_download: bool = False) -> None: """Initialise with the configured project name.""" super().__init__(str(configuration.get_value(ConfigurationVariable.PROJECT_NAME))) + self._skip_go_module_download = skip_go_module_download def fetch_project_metadata(self) -> ProjectMetadata: """Collect module dependencies using go-licenses without replacing shared policy checks.""" module_directories = _go_licence_module_directories() env = os.environ.copy() env[ENVVAR_GO_MOD] = GO_MOD_ON_VALUE - if _should_download_go_module_dependencies(): + if not self._skip_go_module_download and _should_download_go_module_dependencies(): _download_go_module_dependencies(module_directories, env) else: logger.info("Skipping Go module dependency downloads before licence analysis by configuration.") @@ -486,9 +487,9 @@ def get_secret_registry_exclude_files(self) -> List[str]: (r"^\.github[\\/]workflows[\\/].*"), ] - def get_current_spdx_project(self) -> Optional[SpdxProject]: + def get_current_spdx_project(self, skip_go_module_download: bool = False) -> Optional[SpdxProject]: """Gets current SPDX description.""" - return SpdxProject(GoProjectMetadataFetcher()) + return SpdxProject(GoProjectMetadataFetcher(skip_go_module_download=skip_go_module_download)) def should_clean_before_packaging(self) -> bool: """States whether the repository must be cleaned before packaging happens.""" diff --git a/continuous_delivery_scripts/plugins/noop.py b/continuous_delivery_scripts/plugins/noop.py index 28e74b4..b91dfcd 100644 --- a/continuous_delivery_scripts/plugins/noop.py +++ b/continuous_delivery_scripts/plugins/noop.py @@ -53,7 +53,7 @@ def can_get_project_metadata(self) -> bool: """States whether project metadata can be retrieved.""" return False - def get_current_spdx_project(self) -> Optional["SpdxProject"]: + def get_current_spdx_project(self, skip_go_module_download: bool = False) -> Optional["SpdxProject"]: """No Op.""" return None diff --git a/continuous_delivery_scripts/plugins/python.py b/continuous_delivery_scripts/plugins/python.py index 40dbfaa..8d0b7a9 100644 --- a/continuous_delivery_scripts/plugins/python.py +++ b/continuous_delivery_scripts/plugins/python.py @@ -198,6 +198,6 @@ def should_include_spdx_in_package(self) -> bool: # FIXME Comment out SPDX package as no longer working return False - def get_current_spdx_project(self) -> Optional["SpdxProject"]: + def get_current_spdx_project(self, skip_go_module_download: bool = False) -> Optional["SpdxProject"]: """Gets the current SPDX description.""" return _get_current_spdx_project() diff --git a/continuous_delivery_scripts/report_third_party_ip.py b/continuous_delivery_scripts/report_third_party_ip.py index e2cf5ed..0931520 100644 --- a/continuous_delivery_scripts/report_third_party_ip.py +++ b/continuous_delivery_scripts/report_third_party_ip.py @@ -14,13 +14,11 @@ import argparse import logging -import os import sys from pathlib import Path from typing import TYPE_CHECKING, Any, Optional from continuous_delivery_scripts.language_specifics import get_language_specifics -from continuous_delivery_scripts.utils.configuration import ConfigurationVariable from continuous_delivery_scripts.utils.logging import set_log_level, log_exception if TYPE_CHECKING: @@ -29,11 +27,6 @@ logger = logging.getLogger(__name__) -def _apply_go_module_download_options(skip_go_module_download: bool) -> None: - if skip_go_module_download: - os.environ[ConfigurationVariable.SKIP_GO_MODULE_DOWNLOAD_FOR_LICENSING.name] = "true" - - def generate_spdx_project_reports(project: "SpdxProject", output_directory: Path) -> "SpdxProject": """Generates all the SPDX reports for a given project.""" logger.info("Generating SPDX report.") @@ -43,9 +36,11 @@ def generate_spdx_project_reports(project: "SpdxProject", output_directory: Path return project -def generate_spdx_reports(output_directory: Path, lookup_scancode: bool = False) -> Optional["SpdxProject"]: +def generate_spdx_reports( + output_directory: Path, lookup_scancode: bool = False, skip_go_module_download: bool = False +) -> Optional["SpdxProject"]: """Generates all the SPDX reports for the current project.""" - project = get_language_specifics().get_current_spdx_project() + project = get_language_specifics().get_current_spdx_project(skip_go_module_download=skip_go_module_download) if not project: return None if lookup_scancode: @@ -88,11 +83,14 @@ def convert_to_path(arg: Any) -> Path: ) args = parser.parse_args() set_log_level(args.verbose) - _apply_go_module_download_options(args.skip_go_module_download) try: if get_language_specifics().can_get_project_metadata(): - project = generate_spdx_reports(args.output_dir, lookup_scancode=args.lookup_scancode) + project = generate_spdx_reports( + args.output_dir, + lookup_scancode=args.lookup_scancode, + skip_go_module_download=args.skip_go_module_download, + ) if project: project.check_licence_compliance() if args.lookup_scancode: diff --git a/continuous_delivery_scripts/utils/language_specifics_base.py b/continuous_delivery_scripts/utils/language_specifics_base.py index dc2e0d7..a7afd55 100644 --- a/continuous_delivery_scripts/utils/language_specifics_base.py +++ b/continuous_delivery_scripts/utils/language_specifics_base.py @@ -121,7 +121,7 @@ def release_package_to_repository(self, mode: CommitType, version: str) -> None: pass @abstractmethod - def get_current_spdx_project(self) -> Optional["SpdxProject"]: + def get_current_spdx_project(self, skip_go_module_download: bool = False) -> Optional["SpdxProject"]: """Gets current project SPDX.""" pass diff --git a/tests/spdx/test_check_licence_compliance.py b/tests/spdx/test_check_licence_compliance.py index 9cee548..7f94c86 100644 --- a/tests/spdx/test_check_licence_compliance.py +++ b/tests/spdx/test_check_licence_compliance.py @@ -8,7 +8,6 @@ import csv from io import BytesIO, StringIO import json -import os from pathlib import Path from tempfile import TemporaryDirectory from unittest import TestCase @@ -177,11 +176,5 @@ def test_unsupported_plugin_does_not_silently_pass(self): self.plugin.get_current_spdx_project.assert_not_called() def test_skip_go_module_download_flag_sets_configuration_override(self): - with patch.dict(os.environ, {}, clear=False): - self.assertNotIn(ConfigurationVariable.SKIP_GO_MODULE_DOWNLOAD_FOR_LICENSING.name, os.environ) - self.assertEqual(self._run_command("--skip-go-module-download"), 0) - - self.assertEqual( - os.environ[ConfigurationVariable.SKIP_GO_MODULE_DOWNLOAD_FOR_LICENSING.name], - "true", - ) + self.assertEqual(self._run_command("--skip-go-module-download"), 0) + self.plugin.get_current_spdx_project.assert_called_once_with(skip_go_module_download=True) diff --git a/tests/spdx/test_generate_spdx_cli.py b/tests/spdx/test_generate_spdx_cli.py index dc93bda..2b869e8 100644 --- a/tests/spdx/test_generate_spdx_cli.py +++ b/tests/spdx/test_generate_spdx_cli.py @@ -6,13 +6,11 @@ from contextlib import redirect_stderr from io import StringIO -import os from pathlib import Path from unittest import TestCase from unittest.mock import Mock, patch from continuous_delivery_scripts.report_third_party_ip import main -from continuous_delivery_scripts.utils.configuration import ConfigurationVariable class TestGenerateSpdxCli(TestCase): @@ -33,7 +31,7 @@ def test_successful_opt_in_lookup_prints_follow_up(self): ): self.assertEqual(main(), 0) - generate.assert_called_once_with(Path("reports"), lookup_scancode=True) + generate.assert_called_once_with(Path("reports"), lookup_scancode=True, skip_go_module_download=False) project.check_licence_compliance.assert_called_once_with() self.assertIn("WARNING: dependency: add", output.getvalue()) self.assertIn("[ProjectConfig.LICENCE_ASSESSMENT_RULES.classifications]", output.getvalue()) @@ -56,18 +54,16 @@ def test_failed_check_does_not_present_lookup_as_a_successful_remediation(self): project.scancode_follow_up_warnings.assert_not_called() self.assertNotIn("WARNING: dependency", output.getvalue()) - def test_skip_go_module_download_flag_sets_configuration_override(self): + def test_skip_go_module_download_flag_is_forwarded_to_spdx_generation(self): project = Mock() plugin = Mock() plugin.can_get_project_metadata.return_value = True - with patch.dict(os.environ, {}, clear=False), patch( - "sys.argv", ["cd-generate-spdx", "-o", "reports", "--skip-go-module-download"] - ), patch( + with patch("sys.argv", ["cd-generate-spdx", "-o", "reports", "--skip-go-module-download"]), patch( "continuous_delivery_scripts.report_third_party_ip.get_language_specifics", return_value=plugin ), patch( "continuous_delivery_scripts.report_third_party_ip.generate_spdx_reports", return_value=project - ): - self.assertNotIn(ConfigurationVariable.SKIP_GO_MODULE_DOWNLOAD_FOR_LICENSING.name, os.environ) + ) as generate: self.assertEqual(main(), 0) - self.assertEqual(os.environ[ConfigurationVariable.SKIP_GO_MODULE_DOWNLOAD_FOR_LICENSING.name], "true") + + generate.assert_called_once_with(Path("reports"), lookup_scancode=False, skip_go_module_download=True) From 8e3951858a4bbe3acb48c8be63c4d9e8af0f7d8c Mon Sep 17 00:00:00 2001 From: Adrien CABARBAYE Date: Fri, 2 Oct 2026 22:15:03 +0100 Subject: [PATCH 5/8] :recycle: Rename dependency download skip options --- .../check_licence_compliance.py | 6 +++--- continuous_delivery_scripts/plugins/basic.py | 2 +- continuous_delivery_scripts/plugins/ci.py | 2 +- continuous_delivery_scripts/plugins/docker.py | 2 +- continuous_delivery_scripts/plugins/golang.py | 12 ++++++------ continuous_delivery_scripts/plugins/noop.py | 2 +- continuous_delivery_scripts/plugins/python.py | 2 +- continuous_delivery_scripts/report_third_party_ip.py | 10 +++++----- continuous_delivery_scripts/utils/configuration.py | 6 +++--- .../utils/language_specifics_base.py | 2 +- tests/plugin/test_go_licensing.py | 2 +- tests/spdx/test_check_licence_compliance.py | 6 +++--- tests/spdx/test_generate_spdx_cli.py | 8 ++++---- 13 files changed, 31 insertions(+), 31 deletions(-) diff --git a/continuous_delivery_scripts/check_licence_compliance.py b/continuous_delivery_scripts/check_licence_compliance.py index 5e7dac8..41ee6c9 100644 --- a/continuous_delivery_scripts/check_licence_compliance.py +++ b/continuous_delivery_scripts/check_licence_compliance.py @@ -91,9 +91,9 @@ def main() -> int: help="Consult ScanCode LicenseDB for unclassified licences or missing assessment rules.", ) parser.add_argument( - "--skip-go-module-download", + "--skip-dependency-download", action="store_true", - help="Skip automatic `go mod download all` before Go licence checks and SPDX dependency analysis.", + help="Skip automatic dependency downloads before licence checks and SPDX dependency analysis.", ) parser.add_argument("-v", "--verbose", action="count", default=0, help="Increase logging verbosity.") args = parser.parse_args() @@ -103,7 +103,7 @@ def main() -> int: plugin = get_language_specifics() if not plugin.can_get_project_metadata(): raise ValueError("The selected language plugin cannot provide metadata for licence compliance checks.") - project = plugin.get_current_spdx_project(skip_go_module_download=args.skip_go_module_download) + project = plugin.get_current_spdx_project(skip_dependency_download=args.skip_dependency_download) if project is None: raise ValueError("The selected language plugin did not return a project for licence compliance checks.") if args.lookup_scancode: diff --git a/continuous_delivery_scripts/plugins/basic.py b/continuous_delivery_scripts/plugins/basic.py index c9a8d48..16dc0ff 100644 --- a/continuous_delivery_scripts/plugins/basic.py +++ b/continuous_delivery_scripts/plugins/basic.py @@ -51,6 +51,6 @@ def can_get_project_metadata(self) -> bool: """States whether project metadata can be retrieved.""" return False - def get_current_spdx_project(self, skip_go_module_download: bool = False) -> Optional["SpdxProject"]: + def get_current_spdx_project(self, skip_dependency_download: bool = False) -> Optional["SpdxProject"]: """Basic.""" return None diff --git a/continuous_delivery_scripts/plugins/ci.py b/continuous_delivery_scripts/plugins/ci.py index 94a5a70..d220f4c 100644 --- a/continuous_delivery_scripts/plugins/ci.py +++ b/continuous_delivery_scripts/plugins/ci.py @@ -61,7 +61,7 @@ def can_get_project_metadata(self) -> bool: """States whether project metadata can be retrieved.""" return False - def get_current_spdx_project(self, skip_go_module_download: bool = False) -> Optional["SpdxProject"]: + def get_current_spdx_project(self, skip_dependency_download: bool = False) -> Optional["SpdxProject"]: """Gets current SPDX description.""" # TODO return None diff --git a/continuous_delivery_scripts/plugins/docker.py b/continuous_delivery_scripts/plugins/docker.py index 1b3d7fe..2b27ecf 100644 --- a/continuous_delivery_scripts/plugins/docker.py +++ b/continuous_delivery_scripts/plugins/docker.py @@ -53,7 +53,7 @@ def can_get_project_metadata(self) -> bool: """States whether project metadata can be retrieved.""" return False - def get_current_spdx_project(self, skip_go_module_download: bool = False) -> Optional["SpdxProject"]: + def get_current_spdx_project(self, skip_dependency_download: bool = False) -> Optional["SpdxProject"]: """Gets current SPDX description.""" # TODO return None diff --git a/continuous_delivery_scripts/plugins/golang.py b/continuous_delivery_scripts/plugins/golang.py index ea89178..9f4b372 100644 --- a/continuous_delivery_scripts/plugins/golang.py +++ b/continuous_delivery_scripts/plugins/golang.py @@ -149,7 +149,7 @@ def _download_go_module_dependencies(module_directories: List[Path], env: Mutabl def _should_download_go_module_dependencies() -> bool: """Return whether Go module dependencies should be prefetched automatically.""" - configured = configuration.get_value_or_default(ConfigurationVariable.SKIP_GO_MODULE_DOWNLOAD_FOR_LICENSING, False) + configured = configuration.get_value_or_default(ConfigurationVariable.SKIP_DEPENDENCY_DOWNLOAD_FOR_LICENSING, False) if isinstance(configured, str): configured = configured.strip().lower() in ("true", "1", "yes", "on") return not bool(configured) @@ -246,17 +246,17 @@ def _stage_root_licence_files_for_modules(module_directories: List[Path]) -> Ite class GoProjectMetadataFetcher(ProjectMetadataFetcher): """Retrieve Go dependency licences for the shared SPDX and compliance report.""" - def __init__(self, skip_go_module_download: bool = False) -> None: + def __init__(self, skip_dependency_download: bool = False) -> None: """Initialise with the configured project name.""" super().__init__(str(configuration.get_value(ConfigurationVariable.PROJECT_NAME))) - self._skip_go_module_download = skip_go_module_download + self._skip_dependency_download = skip_dependency_download def fetch_project_metadata(self) -> ProjectMetadata: """Collect module dependencies using go-licenses without replacing shared policy checks.""" module_directories = _go_licence_module_directories() env = os.environ.copy() env[ENVVAR_GO_MOD] = GO_MOD_ON_VALUE - if not self._skip_go_module_download and _should_download_go_module_dependencies(): + if not self._skip_dependency_download and _should_download_go_module_dependencies(): _download_go_module_dependencies(module_directories, env) else: logger.info("Skipping Go module dependency downloads before licence analysis by configuration.") @@ -487,9 +487,9 @@ def get_secret_registry_exclude_files(self) -> List[str]: (r"^\.github[\\/]workflows[\\/].*"), ] - def get_current_spdx_project(self, skip_go_module_download: bool = False) -> Optional[SpdxProject]: + def get_current_spdx_project(self, skip_dependency_download: bool = False) -> Optional[SpdxProject]: """Gets current SPDX description.""" - return SpdxProject(GoProjectMetadataFetcher(skip_go_module_download=skip_go_module_download)) + return SpdxProject(GoProjectMetadataFetcher(skip_dependency_download=skip_dependency_download)) def should_clean_before_packaging(self) -> bool: """States whether the repository must be cleaned before packaging happens.""" diff --git a/continuous_delivery_scripts/plugins/noop.py b/continuous_delivery_scripts/plugins/noop.py index b91dfcd..35aca57 100644 --- a/continuous_delivery_scripts/plugins/noop.py +++ b/continuous_delivery_scripts/plugins/noop.py @@ -53,7 +53,7 @@ def can_get_project_metadata(self) -> bool: """States whether project metadata can be retrieved.""" return False - def get_current_spdx_project(self, skip_go_module_download: bool = False) -> Optional["SpdxProject"]: + def get_current_spdx_project(self, skip_dependency_download: bool = False) -> Optional["SpdxProject"]: """No Op.""" return None diff --git a/continuous_delivery_scripts/plugins/python.py b/continuous_delivery_scripts/plugins/python.py index 8d0b7a9..e9e0fb2 100644 --- a/continuous_delivery_scripts/plugins/python.py +++ b/continuous_delivery_scripts/plugins/python.py @@ -198,6 +198,6 @@ def should_include_spdx_in_package(self) -> bool: # FIXME Comment out SPDX package as no longer working return False - def get_current_spdx_project(self, skip_go_module_download: bool = False) -> Optional["SpdxProject"]: + def get_current_spdx_project(self, skip_dependency_download: bool = False) -> Optional["SpdxProject"]: """Gets the current SPDX description.""" return _get_current_spdx_project() diff --git a/continuous_delivery_scripts/report_third_party_ip.py b/continuous_delivery_scripts/report_third_party_ip.py index 0931520..ba873fb 100644 --- a/continuous_delivery_scripts/report_third_party_ip.py +++ b/continuous_delivery_scripts/report_third_party_ip.py @@ -37,10 +37,10 @@ def generate_spdx_project_reports(project: "SpdxProject", output_directory: Path def generate_spdx_reports( - output_directory: Path, lookup_scancode: bool = False, skip_go_module_download: bool = False + output_directory: Path, lookup_scancode: bool = False, skip_dependency_download: bool = False ) -> Optional["SpdxProject"]: """Generates all the SPDX reports for the current project.""" - project = get_language_specifics().get_current_spdx_project(skip_go_module_download=skip_go_module_download) + project = get_language_specifics().get_current_spdx_project(skip_dependency_download=skip_dependency_download) if not project: return None if lookup_scancode: @@ -69,9 +69,9 @@ def convert_to_path(arg: Any) -> Path: help="Consult ScanCode LicenseDB for unclassified licences or missing assessment rules.", ) parser.add_argument( - "--skip-go-module-download", + "--skip-dependency-download", action="store_true", - help="Skip automatic `go mod download all` before Go licence checks and SPDX dependency analysis.", + help="Skip automatic dependency downloads before licence checks and SPDX dependency analysis.", ) parser.add_argument( @@ -89,7 +89,7 @@ def convert_to_path(arg: Any) -> Path: project = generate_spdx_reports( args.output_dir, lookup_scancode=args.lookup_scancode, - skip_go_module_download=args.skip_go_module_download, + skip_dependency_download=args.skip_dependency_download, ) if project: project.check_licence_compliance() diff --git a/continuous_delivery_scripts/utils/configuration.py b/continuous_delivery_scripts/utils/configuration.py index e563cff..4e61b6b 100644 --- a/continuous_delivery_scripts/utils/configuration.py +++ b/continuous_delivery_scripts/utils/configuration.py @@ -109,8 +109,8 @@ class ConfigurationVariable(enum.Enum): """Project-specific records explaining manual reviews of REVIEW assessments.""" LICENCE_ASSESSMENT_FAIL_ON = 45 """Optional top-level list of assessment statuses that fail compliance checks.""" - SKIP_GO_MODULE_DOWNLOAD_FOR_LICENSING = 46 - """Skip automatic `go mod download all` before Go licence and SPDX dependency analysis.""" + SKIP_DEPENDENCY_DOWNLOAD_FOR_LICENSING = 46 + """Skip automatic dependency downloads before licence and SPDX dependency analysis.""" @staticmethod def choices() -> List[str]: @@ -221,7 +221,7 @@ class StaticConfig(GenericConfig): LICENCE_ASSESSMENT_FAIL_ON = None """When unset, use fail_on from the embedded or project licence assessment rules.""" GENERATE_LICENSING_SUMMARY_ON_RELEASE = False - SKIP_GO_MODULE_DOWNLOAD_FOR_LICENSING = False + SKIP_DEPENDENCY_DOWNLOAD_FOR_LICENSING = False DEPENDENCY_UPDATE_NEWS_MESSAGE = "Dependency upgrade: {message}" DEPENDENCY_UPDATE_NEWS_TYPE = NewsType.bugfix DEPENDENCY_UPDATE_BRANCH_PATTERN = r"^\s*[Dd]ependabot\/.+\/(?P.+)" diff --git a/continuous_delivery_scripts/utils/language_specifics_base.py b/continuous_delivery_scripts/utils/language_specifics_base.py index a7afd55..4a79b37 100644 --- a/continuous_delivery_scripts/utils/language_specifics_base.py +++ b/continuous_delivery_scripts/utils/language_specifics_base.py @@ -121,7 +121,7 @@ def release_package_to_repository(self, mode: CommitType, version: str) -> None: pass @abstractmethod - def get_current_spdx_project(self, skip_go_module_download: bool = False) -> Optional["SpdxProject"]: + def get_current_spdx_project(self, skip_dependency_download: bool = False) -> Optional["SpdxProject"]: """Gets current project SPDX.""" pass diff --git a/tests/plugin/test_go_licensing.py b/tests/plugin/test_go_licensing.py index 753ee9b..78e2341 100644 --- a/tests/plugin/test_go_licensing.py +++ b/tests/plugin/test_go_licensing.py @@ -197,7 +197,7 @@ def test_download_go_module_dependencies_can_be_skipped_from_configuration(self) configuration, "get_value", side_effect=lambda key: ( - "true" if key == ConfigurationVariable.SKIP_GO_MODULE_DOWNLOAD_FOR_LICENSING else get_value(key) + "true" if key == ConfigurationVariable.SKIP_DEPENDENCY_DOWNLOAD_FOR_LICENSING else get_value(key) ), ): self.assertFalse(golang._should_download_go_module_dependencies()) diff --git a/tests/spdx/test_check_licence_compliance.py b/tests/spdx/test_check_licence_compliance.py index 7f94c86..7b38a8d 100644 --- a/tests/spdx/test_check_licence_compliance.py +++ b/tests/spdx/test_check_licence_compliance.py @@ -175,6 +175,6 @@ def test_unsupported_plugin_does_not_silently_pass(self): self.assertEqual(self._run_command(), 1) self.plugin.get_current_spdx_project.assert_not_called() - def test_skip_go_module_download_flag_sets_configuration_override(self): - self.assertEqual(self._run_command("--skip-go-module-download"), 0) - self.plugin.get_current_spdx_project.assert_called_once_with(skip_go_module_download=True) + def test_skip_dependency_download_flag_is_forwarded_to_plugin(self): + self.assertEqual(self._run_command("--skip-dependency-download"), 0) + self.plugin.get_current_spdx_project.assert_called_once_with(skip_dependency_download=True) diff --git a/tests/spdx/test_generate_spdx_cli.py b/tests/spdx/test_generate_spdx_cli.py index 2b869e8..9264657 100644 --- a/tests/spdx/test_generate_spdx_cli.py +++ b/tests/spdx/test_generate_spdx_cli.py @@ -31,7 +31,7 @@ def test_successful_opt_in_lookup_prints_follow_up(self): ): self.assertEqual(main(), 0) - generate.assert_called_once_with(Path("reports"), lookup_scancode=True, skip_go_module_download=False) + generate.assert_called_once_with(Path("reports"), lookup_scancode=True, skip_dependency_download=False) project.check_licence_compliance.assert_called_once_with() self.assertIn("WARNING: dependency: add", output.getvalue()) self.assertIn("[ProjectConfig.LICENCE_ASSESSMENT_RULES.classifications]", output.getvalue()) @@ -54,16 +54,16 @@ def test_failed_check_does_not_present_lookup_as_a_successful_remediation(self): project.scancode_follow_up_warnings.assert_not_called() self.assertNotIn("WARNING: dependency", output.getvalue()) - def test_skip_go_module_download_flag_is_forwarded_to_spdx_generation(self): + def test_skip_dependency_download_flag_is_forwarded_to_spdx_generation(self): project = Mock() plugin = Mock() plugin.can_get_project_metadata.return_value = True - with patch("sys.argv", ["cd-generate-spdx", "-o", "reports", "--skip-go-module-download"]), patch( + with patch("sys.argv", ["cd-generate-spdx", "-o", "reports", "--skip-dependency-download"]), patch( "continuous_delivery_scripts.report_third_party_ip.get_language_specifics", return_value=plugin ), patch( "continuous_delivery_scripts.report_third_party_ip.generate_spdx_reports", return_value=project ) as generate: self.assertEqual(main(), 0) - generate.assert_called_once_with(Path("reports"), lookup_scancode=False, skip_go_module_download=True) + generate.assert_called_once_with(Path("reports"), lookup_scancode=False, skip_dependency_download=True) From cc687b6bf58d1076468c82c36edeb7713feddfe4 Mon Sep 17 00:00:00 2001 From: Adrien CABARBAYE Date: Fri, 2 Oct 2026 22:18:45 +0100 Subject: [PATCH 6/8] :book: Document dependency download behaviour --- README.md | 4 ++-- continuous_delivery_scripts/plugins/GOLAND.MD | 9 ++++++++- guides/checking-licence-compliance.md | 11 ++++++++--- guides/generating-an-spdx-sbom.md | 5 ++++- guides/third-party-ip-reporting.md | 9 ++++++--- 5 files changed, 28 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 348347a..d7c59b0 100644 --- a/README.md +++ b/README.md @@ -185,8 +185,8 @@ same commands whichever CI system the team chooses. | [`cd-get-config`](./guides/reading-project-configuration.md) | Prints a project configuration value. | Either `--key` (`-k`) or `--config-variable` (`-c`) | Developer or CI | | [`cd-tag-and-release`](./guides/automating-releases.md) | Runs the release flow, including documentation, licensing summaries, Git tagging, packaging and publication through the selected [language plugin](./continuous_delivery_scripts/plugins). | Required `--release-type` (`-t`); optional `--current-branch` (`-b`) | CI release flow | | [`cd-generate-docs`](./guides/generating-code-documentation.md) | Generates code documentation in the configured output directory (or one specified on the command line). | `--output_directory` | Developer preview or CI | -| [`cd-generate-spdx`](./guides/generating-an-spdx-sbom.md) | Generates SPDX documents and third-party licence summaries, then checks licence compliance when project metadata is available. | Required `--output-dir` (`-o`); optional `--lookup-scancode` | CI audit; developer review also possible | -| [`cd-check-licence-compliance`](./guides/checking-licence-compliance.md) | Checks project and dependency licences without creating SPDX documents; optionally writes third-party IP summaries. | Optional `--output-dir` (`-o`) for reports, `--lookup-scancode` | CI gate or developer review | +| [`cd-generate-spdx`](./guides/generating-an-spdx-sbom.md) | Generates SPDX documents and third-party licence summaries, then checks licence compliance when project metadata is available. | Required `--output-dir` (`-o`); optional `--lookup-scancode`, `--skip-dependency-download` | CI audit; developer review also possible | +| [`cd-check-licence-compliance`](./guides/checking-licence-compliance.md) | Checks project and dependency licences without creating SPDX documents; optionally writes third-party IP summaries. | Optional `--output-dir` (`-o`) for reports, `--lookup-scancode`, `--skip-dependency-download` | CI gate or developer review | | [`cd-license-files`](./guides/licence-header-management.md) | Adds or updates source-file licence and copyright headers when the language plugin supports them. | `--verbose` (`-v`) | Developer or CI | | [`cd-record-secrets`](./guides/recording-secrets.md) | Records accepted findings in the project's [detect-secrets](https://github.com/Yelp/detect-secrets) registry. | `--registry-file` (`-r`); defaults to the configured registry | Developer (registry maintenance) | | [`cd-detect-secrets`](./guides/checking-for-secrets.md) | Checks Git-tracked files against that registry and fails if new secrets are found. | `--registry-file` (`-r`); defaults to the configured registry | CI; developer checks also possible | diff --git a/continuous_delivery_scripts/plugins/GOLAND.MD b/continuous_delivery_scripts/plugins/GOLAND.MD index e8c829a..6cf8a2a 100644 --- a/continuous_delivery_scripts/plugins/GOLAND.MD +++ b/continuous_delivery_scripts/plugins/GOLAND.MD @@ -20,6 +20,13 @@ usable `go-licenses` binary on `PATH` before installing must be available; an installed binary can also be located under `GOBIN` or the first `GOPATH` entry's `bin` directory. +For licence and SPDX analysis, the plugin also runs `go mod download all` for +each detected Go module by default so callers do not need a separate dependency +prefetch step. If your environment already downloads dependencies and you want +to skip that work, use `--skip-dependency-download` on +`cd-check-licence-compliance` or `cd-generate-spdx`, or set +`SKIP_DEPENDENCY_DOWNLOAD_FOR_LICENSING = true` in configuration. + Set `PROGRAMMING_LANGUAGE = "golang"`, `PROJECT_NAME`, `PROJECT_ROOT` and `SOURCE_DIR` in `[ProjectConfig]`. Put a `go.mod` under the source or project root; `go.work` modules and nested Go modules are also considered. Supply @@ -27,7 +34,7 @@ root; `go.work` modules and nested Go modules are also considered. Supply and `[spdx]` namespace fields as explained in the [SPDX configuration guide](../../DEVELOPMENT.md#spdx-document-identity). -With the project and its Go dependencies available, run: +With the project available, run: ```bash mkdir -p licensing diff --git a/guides/checking-licence-compliance.md b/guides/checking-licence-compliance.md index d5480dd..bec3428 100644 --- a/guides/checking-licence-compliance.md +++ b/guides/checking-licence-compliance.md @@ -8,9 +8,9 @@ without generating SPDX documents. The command writes no files by default. ## Inputs and example -Install the dependencies you intend to audit and select a language plugin that -provides project metadata. Configure `ACCEPTED_THIRD_PARTY_LICENCES` and any -documented manual reviews in `[ProjectConfig]` in `pyproject.toml`. Then run: +Select a language plugin that provides project metadata. Configure +`ACCEPTED_THIRD_PARTY_LICENCES` and any documented manual reviews in +`[ProjectConfig]` in `pyproject.toml`. Then run: ```bash cd-check-licence-compliance @@ -28,6 +28,11 @@ The optional `--output-dir` (`-o`) writes `third_party_IP_report.html`, `.csv`, `.txt` and `.json`. It does not create `.spdx` files. The reports are written before the compliance check, so they remain available if the check fails. +For Go projects, CDS downloads module dependencies automatically before running +licence analysis so CI does not need a separate `go mod download all` step. +Pass `--skip-dependency-download` when your environment already prefetches the +required dependencies and you want to skip that extra work. + ## Result The command succeeds when the discovered project and dependency licences meet diff --git a/guides/generating-an-spdx-sbom.md b/guides/generating-an-spdx-sbom.md index 651515c..839dc71 100644 --- a/guides/generating-an-spdx-sbom.md +++ b/guides/generating-an-spdx-sbom.md @@ -9,7 +9,6 @@ see [TPIP reporting](third-party-ip-reporting.md). ## Inputs and example -Install the project and its dependencies in the environment being audited. Configure `pyproject.toml` with `PROJECT_ROOT`, `SOURCE_DIR`, `PROGRAMMING_LANGUAGE`, your licence policy and any values required by the selected language plugin. Provide @@ -27,6 +26,10 @@ the command returns without a report. The selected plugin uses language-specific tools to obtain dependency and licence information; consult the [plugin documentation](https://github.com/ARMmbed/continuous-delivery-scripts/tree/main/continuous_delivery_scripts/plugins) for its prerequisites and metadata support. +For Go projects, CDS downloads module dependencies automatically before +running the dependency scan. Pass `--skip-dependency-download` when the +environment already prefetches those dependencies and you want to skip that +extra step. SPDX tag-value generation also requires an SDK compatible with this project's writer. Check the output files before treating an audit as complete. Installed packages and platform-specific dependency markers determine what is covered; diff --git a/guides/third-party-ip-reporting.md b/guides/third-party-ip-reporting.md index 05ee709..4c65cae 100644 --- a/guides/third-party-ip-reporting.md +++ b/guides/third-party-ip-reporting.md @@ -10,9 +10,9 @@ see [SPDX generation](generating-an-spdx-sbom.md). ## Inputs and example -Install the dependencies you intend to audit, configure your language plugin -and set `ACCEPTED_THIRD_PARTY_LICENCES` and any documented manual checks in -`pyproject.toml`. Choose an existing output directory: +Configure your language plugin and set `ACCEPTED_THIRD_PARTY_LICENCES` and any +documented manual checks in `pyproject.toml`. Choose an existing output +directory: ```bash mkdir -p licensing @@ -27,6 +27,9 @@ for language-specific prerequisites and support. An audit covers the installed environment, so run it for each supported platform if dependencies differ by operating system. Review unknown licences and packaged notices rather than assuming a missing value means permission. +For Go projects, CDS downloads module dependencies automatically before the +scan. Pass `--skip-dependency-download` when those dependencies have already +been prefetched and you want to skip that extra work. ## Output From 9fc8d14615988238694adedc789f858d28b193ed Mon Sep 17 00:00:00 2001 From: Adrien CABARBAYE Date: Fri, 2 Oct 2026 22:21:30 +0100 Subject: [PATCH 7/8] :book: Clarify Python audit dependency setup --- continuous_delivery_scripts/plugins/PYTHON.MD | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/continuous_delivery_scripts/plugins/PYTHON.MD b/continuous_delivery_scripts/plugins/PYTHON.MD index 44690bf..83031c6 100644 --- a/continuous_delivery_scripts/plugins/PYTHON.MD +++ b/continuous_delivery_scripts/plugins/PYTHON.MD @@ -17,8 +17,8 @@ The plugin uses [pdoc](https://pdoc3.github.io/pdoc/) to generate the code refer The Python plugin reads metadata from the **installed** project distribution and walks its `Requires-Dist` entries to find installed, transitive dependencies. -Run the audit in the environment containing the dependencies to be released; -it does not install packages. +Install the dependencies you intend to audit, then run the audit in that +environment; the plugin does not install packages itself. > **Warning — platform limitation:** The audit only sees dependencies installed > for the current platform and Python version. It does **not** inspect From c917abf293e088b3f5b4d0c1099a8248b3e818b3 Mon Sep 17 00:00:00 2001 From: Adrien CABARBAYE Date: Fri, 2 Oct 2026 22:23:00 +0100 Subject: [PATCH 8/8] :book: Explain Python dependency audit setup --- continuous_delivery_scripts/plugins/PYTHON.MD | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/continuous_delivery_scripts/plugins/PYTHON.MD b/continuous_delivery_scripts/plugins/PYTHON.MD index 83031c6..0fb5fad 100644 --- a/continuous_delivery_scripts/plugins/PYTHON.MD +++ b/continuous_delivery_scripts/plugins/PYTHON.MD @@ -18,7 +18,9 @@ The plugin uses [pdoc](https://pdoc3.github.io/pdoc/) to generate the code refer The Python plugin reads metadata from the **installed** project distribution and walks its `Requires-Dist` entries to find installed, transitive dependencies. Install the dependencies you intend to audit, then run the audit in that -environment; the plugin does not install packages itself. +environment; the plugin does not install packages itself. This is deliberate: +Python projects can define dependencies in many different ways, so CDS cannot +reliably infer and reproduce the intended audit environment automatically. > **Warning — platform limitation:** The audit only sees dependencies installed > for the current platform and Python version. It does **not** inspect