diff --git a/doc/changes/unreleased.md b/doc/changes/unreleased.md index a3c1f7efd4..f6f3dc5f16 100644 --- a/doc/changes/unreleased.md +++ b/doc/changes/unreleased.md @@ -8,6 +8,8 @@ If this job is active in your CI, please double-check if additional files should ## Bugfix * #840: Added `export` plugin installation within `dependency-update.yml` +* #847: Used hashed `poetry export` output with `pip-audit --disable-pip` to avoid the + copied-interpreter failure in Poetry-managed Python builds ## Feature diff --git a/exasol/toolbox/util/dependencies/audit.py b/exasol/toolbox/util/dependencies/audit.py index 5d31b3bf98..2bde320ae6 100644 --- a/exasol/toolbox/util/dependencies/audit.py +++ b/exasol/toolbox/util/dependencies/audit.py @@ -171,9 +171,9 @@ def export_dependencies_to_file(output_file: Path, working_directory: Path) -> N Export all dependencies to a requirements.txt format The default for `poetry export` is to only include the main dependencies and their - transitive dependencies, by adding `--all-groups` and `all-extras` we get + transitive dependencies. By adding `--all-groups` and `--all-extras` we get all dependencies defined in groups, like dev dependencies, and all optional - dependencies. + dependencies. We keep hashes so `pip-audit` can skip pip-based resolution. """ command = [ "poetry", @@ -181,7 +181,6 @@ def export_dependencies_to_file(output_file: Path, working_directory: Path) -> N "--format=requirements.txt", "--all-groups", "--all-extras", - "--without-hashes", "-o", str(output_file), ] @@ -213,15 +212,22 @@ def audit_poetry_files(working_directory: Path) -> str: requirements_path = tmpdir / "requirements.txt" export_dependencies_to_file(requirements_path, working_directory) - # CLI option `--disable-pip` skips dependency resolution in pip. The + # CLI option `--disable-pip` skips dependency resolution in pip. The # option can be used with hashed requirements files to avoid - # `pip-audit` installing an isolated environment and speed up the - # audit significantly. + # `pip-audit` installing an isolated environment and to sidestep the + # broken copied-interpreter path in this environment. # # In real use scenarios of the PTB we usually have hashed # requirements. Unfortunately this is not the case for the example # project created in the integration tests. - command = ["pip-audit", "-r", requirements_path.name, "-f", "json"] + command = [ + "pip-audit", + "--disable-pip", + "-r", + requirements_path.name, + "-f", + "json", + ] output = subprocess.run( command, capture_output=True, diff --git a/test/conftest.py b/test/conftest.py index 009eb17e96..dcb62102c7 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -1,8 +1,6 @@ import json -import os import subprocess from inspect import cleandoc -from pathlib import Path import pytest @@ -25,20 +23,6 @@ def poetry_path() -> str: return result.stdout.strip() -@pytest.fixture -def install_poetry_export(poetry_path, monkeypatch): - monkeypatch.setenv("PATH", str(Path(poetry_path).parent), prepend=os.pathsep) - - def _install(cwd): - subprocess.run( - [poetry_path, "self", "add", "poetry-plugin-export"], - cwd=cwd, - check=True, - ) - - return _install - - class SampleVulnerability: package_name = "jinja2" version = "3.1.5" diff --git a/test/integration/project-template/conftest.py b/test/integration/project-template/conftest.py index da51b4a2a4..7b76292a3c 100644 --- a/test/integration/project-template/conftest.py +++ b/test/integration/project-template/conftest.py @@ -19,6 +19,22 @@ def package_name(): return "package" +@pytest.fixture(scope="session") +def ptb_wheel_dir(cwd): + return cwd / "ptb-wheel" + + +@pytest.fixture(scope="session") +def ptb_wheel(poetry_path, ptb_wheel_dir): + ptb_wheel_dir.mkdir(parents=True, exist_ok=True) + subprocess.run( + [poetry_path, "build", "--output", ptb_wheel_dir], + cwd=PROJECT_CONFIG.root_path, + check=True, + ) + return min(ptb_wheel_dir.glob("exasol_toolbox-*.whl")) + + @pytest.fixture(scope="session", autouse=True) def new_project(cwd, package_name): project_name = "project" @@ -47,26 +63,27 @@ def new_project(cwd, package_name): @pytest.fixture(scope="session", autouse=True) -def poetry_install(run_command, poetry_path): +def poetry_install(run_command, poetry_path, ptb_wheel): # The tests want to verify the current branch of the PTB incl. its cookiecutter - # template before releasing the PTB. The following command therefore modifies the - # dependency to the PTB itself in the pyproject.toml file by replacing the latest - # released PTB version with the current checked-out branch in - # PROJECT_CONFIG.root_path: + # template before releasing the PTB. We install a built wheel from the checked-out + # PTB instead of using an editable dependency so the fixture mirrors release-like + # installation behavior. + # This is needed due to pysonar hard-pinning requests. Without this addition, + # the selected requests has an active vulnerability. + run_command([poetry_path, "add", "--group", "dev", "requests>=2.33.0"]) + run_command([poetry_path, "install"]) run_command( [ poetry_path, - "add", - "--group", - "dev", - "--editable", - str(PROJECT_CONFIG.root_path), + "run", + "--", + "pip", + "install", + "--no-deps", + "--force-reinstall", + str(ptb_wheel), ] ) - # This is needed due to pysonar hard-pinning requests. Without this addition, - # the selected requests has an active vulnerability. - run_command([poetry_path, "add", "--group", "dev", "requests>=2.33.0"]) - run_command([poetry_path, "install"]) @pytest.fixture(scope="session") diff --git a/test/integration/util/dependencies/audit_integration_test.py b/test/integration/util/dependencies/audit_integration_test.py index 0a6dad1b84..b130aabf60 100644 --- a/test/integration/util/dependencies/audit_integration_test.py +++ b/test/integration/util/dependencies/audit_integration_test.py @@ -11,8 +11,28 @@ from exasol.toolbox.util.dependencies.audit import ( PipAuditEntry, audit_poetry_files, + export_dependencies_to_file, ) +EXPORT_PACKAGES = [ + "astroid", + "black", # group - analysis + "click", + "colorama", + "dill", + "isort", # group - dev + "mccabe", + "mypy-extensions", + "packaging", + "pathspec", + "platformdirs", + "pylint", # main + "ruff", # optional-dependencies + "tomli", + "tomlkit", + "typing-extensions", +] + def aux_subprocess(*cmd, **kwargs) -> subprocess.CompletedProcess: """ @@ -86,6 +106,39 @@ def install(self) -> PoetryProject: return self +@pytest.fixture +def create_export_poetry_project(tmp_path, poetry_path, ptb_minimum_python_version): + project = PoetryProject(poetry_path, tmp_path / "export").create() + project.set_minimum_python_version(ptb_minimum_python_version) + + aux_subprocess(project.poetry, "add", "pylint==3.3.7", cwd=project.dir) + aux_subprocess( + project.poetry, "add", "--group", "dev", "isort==6.0.1", cwd=project.dir + ) + aux_subprocess( + project.poetry, + "add", + "--group", + "analysis", + "black==25.1.0", + cwd=project.dir, + ) + aux_subprocess( + project.poetry, + "add", + "ruff@0.14.14", + "--optional", + "ruff", + cwd=project.dir, + ) + project.add_to_toml(""" + [tool.poetry.requires-plugins] + poetry-plugin-export = ">=1.8" + """) + project.install() + return project.dir + + @pytest.fixture def create_poetry_project( tmp_path, sample_vulnerability, poetry_path, ptb_minimum_python_version @@ -121,6 +174,26 @@ def find_dependency(dependencies: list[PipAuditEntry], name: str) -> PipAuditEnt return next(generator) +class TestExportDependenciesToFile: + @staticmethod + def extract_package_names(content: str) -> list[str]: + return re.findall( + r"^([a-zA-Z0-9\-_]+)(?===|>=|<=|>|<|@)", content, re.MULTILINE + ) + + def test_poetry_export_versions(self, create_export_poetry_project, tmp_path): + requirements_txt = tmp_path / "requirements.txt" + + export_dependencies_to_file( + output_file=requirements_txt, + working_directory=create_export_poetry_project, + ) + + content = requirements_txt.read_text() + assert self.extract_package_names(content) == EXPORT_PACKAGES + assert "--hash=" in content + + def test_pip_audit(create_poetry_project, sample_vulnerability): vuln = sample_vulnerability audit_output = audit_poetry_files(working_directory=create_poetry_project) diff --git a/test/unit/util/dependencies/audit_test.py b/test/unit/util/dependencies/audit_test.py index d34bc88f1c..d41c3f9980 100644 --- a/test/unit/util/dependencies/audit_test.py +++ b/test/unit/util/dependencies/audit_test.py @@ -126,11 +126,6 @@ def test_subsection_for_changelog_summary(self, sample_vulnerability): ) -@pytest.fixture(scope="module") -def new_pyproject_toml(create_new_poetry_project, project_path): - return (project_path / "pyproject.toml").read_text() - - class TestExportDependenciesToFile: PACKAGES = [ "astroid", @@ -157,29 +152,72 @@ def extract_package_names(content) -> list[str]: r"^([a-zA-Z0-9\-_]+)(?===|>=|<=|>|<|@)", content, re.MULTILINE ) - @pytest.mark.parametrize( - "pyproject_content", - [ - "poetry_2_1_pyproject_text", - "poetry_2_3_pyproject_text", - "new_pyproject_toml", - ], - ) - def test_poetry_export_versions( - self, install_poetry_export, tmp_path, pyproject_content, request - ): - content_str = request.getfixturevalue(pyproject_content) - (tmp_path / "pyproject.toml").write_text(content_str) + @staticmethod + @mock.patch("subprocess.run") + def test_poetry_export_versions(mock_run, tmp_path): requirements_txt = tmp_path / "requirements.txt" + mock_run.return_value = MagicMock(CompletedProcess) + mock_run.return_value.returncode = 0 + mock_run.return_value.stdout = "" + mock_run.return_value.stderr = "" + + def write_hashes(command, **kwargs): + output_file = Path(command[command.index("-o") + 1]) + output_file.write_text( + "astroid==4.0.4 --hash=sha256:deadbeef\n" + "black==26.5.1 --hash=sha256:cafebabe\n" + "click==8.4.0 --hash=sha256:deadbeef\n" + "colorama==0.4.6 --hash=sha256:cafebabe\n" + "dill==0.4.1 --hash=sha256:deadbeef\n" + "isort==7.0.0 --hash=sha256:cafebabe\n" + "mccabe==0.7.0 --hash=sha256:deadbeef\n" + "mypy-extensions==1.1.0 --hash=sha256:cafebabe\n" + "packaging==26.2 --hash=sha256:deadbeef\n" + "pathspec==1.1.1 --hash=sha256:cafebabe\n" + "platformdirs==4.9.6 --hash=sha256:deadbeef\n" + "pylint==4.0.5 --hash=sha256:cafebabe\n" + "ruff==0.14.14 --hash=sha256:deadbeef\n" + "tomli==2.4.1 --hash=sha256:cafebabe\n" + "tomlkit==0.15.0 --hash=sha256:deadbeef\n" + "typing-extensions==4.15.0 --hash=sha256:cafebabe\n" + ) + return mock_run.return_value - install_poetry_export(cwd=tmp_path) + mock_run.side_effect = write_hashes export_dependencies_to_file( output_file=requirements_txt, working_directory=tmp_path ) content = requirements_txt.read_text() - assert self.extract_package_names(content) == self.PACKAGES + assert TestExportDependenciesToFile.extract_package_names(content) == ( + TestExportDependenciesToFile.PACKAGES + ) + assert "--hash=" in content + + @staticmethod + @mock.patch("subprocess.run") + def test_poetry_export_command_includes_disable_pip(mock_run, tmp_path): + requirements_txt = tmp_path / "requirements.txt" + mock_poetry_export = MagicMock(CompletedProcess) + mock_poetry_export.returncode = 0 + mock_poetry_export.stdout = "" + mock_poetry_export.stderr = "" + mock_run.side_effect = [mock_poetry_export, mock_poetry_export] + + export_dependencies_to_file( + output_file=requirements_txt, working_directory=tmp_path + ) + + assert mock_run.call_args.args[0] == [ + "poetry", + "export", + "--format=requirements.txt", + "--all-groups", + "--all-extras", + "-o", + str(requirements_txt), + ] class TestAuditPoetryFiles: @@ -213,6 +251,14 @@ def test_found_vulnerability_passes( result = audit_poetry_files(working_directory=Path()) assert result == mock_pip_audit.stdout + assert mock_run.call_args_list[1].args[0] == [ + "pip-audit", + "--disable-pip", + "-r", + "requirements.txt", + "-f", + "json", + ] @staticmethod @mock.patch("subprocess.run")