diff --git a/exasol/toolbox/util/dependencies/audit.py b/exasol/toolbox/util/dependencies/audit.py index 553c835f73..b3ffc111db 100644 --- a/exasol/toolbox/util/dependencies/audit.py +++ b/exasol/toolbox/util/dependencies/audit.py @@ -26,7 +26,6 @@ r"^Found \d+ known vulnerabilit\w{1,3} in \d+ package\w?$" ) - PipAuditEntry = dict[str, str | list[str] | tuple[str, ...]] @@ -159,6 +158,34 @@ def subsection_for_changelog_summary(self) -> str: """) +def export_dependencies_to_file(output_file: Path, working_directory: Path) -> None: + """ + Export all dependencies to a requirements.txt format + + The default for `poetry export` is to only include the main dependencies and their + transitive dependencies, by adding `--all-groups` and `all-extras` we get + all dependencies defined in groups, like dev dependencies, and all optional + dependencies. + """ + command = [ + "poetry", + "export", + "--format=requirements.txt", + "--all-groups", + "--all-extras", + ] + output = subprocess.run( + command, + capture_output=True, + text=True, + cwd=working_directory, + ) # nosec: B603 - allow fixed poetry usage + if output.returncode != 0: + raise PipAuditException.from_subprocess(output, command, cwd=working_directory) + + output_file.write_text(output.stdout) + + def audit_poetry_files(working_directory: Path) -> str: """ Audit the `pyproject.toml` and `poetry.lock` files @@ -172,20 +199,10 @@ def audit_poetry_files(working_directory: Path) -> str: and then inspecting the dependencies. """ - requirements_txt = "requirements.txt" - command = ["poetry", "export", "--format=requirements.txt"] - output = subprocess.run( - command, - capture_output=True, - text=True, - cwd=working_directory, - ) # nosec - if output.returncode != 0: - raise PipAuditException.from_subprocess(output, command, cwd=working_directory) - with tempfile.TemporaryDirectory() as path: tmpdir = Path(path) - (tmpdir / requirements_txt).write_text(output.stdout) + requirements_path = tmpdir / "requirements.txt" + export_dependencies_to_file(requirements_path, working_directory) # CLI option `--disable-pip` skips dependency resolution in pip. The # option can be used with hashed requirements files to avoid @@ -195,13 +212,13 @@ def audit_poetry_files(working_directory: Path) -> str: # In real use scenarios of the PTB we usually have hashed # requirements. Unfortunately this is not the case for the example # project created in the integration tests. - command = ["pip-audit", "-r", requirements_txt, "-f", "json"] + command = ["pip-audit", "-r", requirements_path.name, "-f", "json"] output = subprocess.run( command, capture_output=True, text=True, cwd=tmpdir, - ) # nosec + ) # nosec: B603 - allow fixed pip-audit usage if output.returncode != 0: # pip-audit does not distinguish between 1) finding vulnerabilities diff --git a/exasol/toolbox/util/dependencies/poetry_dependencies.py b/exasol/toolbox/util/dependencies/poetry_dependencies.py index 26a0265565..e08b74a853 100644 --- a/exasol/toolbox/util/dependencies/poetry_dependencies.py +++ b/exasol/toolbox/util/dependencies/poetry_dependencies.py @@ -59,14 +59,17 @@ def get_section_dict(self, section: str) -> dict | None: def groups(self) -> tuple[PoetryGroup, ...]: groups = [] + # Main Dependencies main_key = "project.dependencies" if self.get_section_dict(main_key): groups.append(PoetryGroup(name="main", toml_section=main_key)) + # Legacy Poetry Main Dependencies main_dynamic_key = "tool.poetry.dependencies" if self.get_section_dict(main_dynamic_key): groups.append(PoetryGroup(name="main", toml_section=main_dynamic_key)) + # Legacy Poetry Group Dependencies group_key = "tool.poetry.group" if group_dict := self.get_section_dict(group_key): for group, content in group_dict.items(): @@ -78,6 +81,7 @@ def groups(self) -> tuple[PoetryGroup, ...]: ) ) + # Poetry Group Dependencies new_group_key = "dependency-groups" if group_dict := self.get_section_dict(new_group_key): for group, content in group_dict.items(): diff --git a/test/conftest.py b/test/conftest.py index 8c7b692217..8ef0af2784 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -1,4 +1,6 @@ import json +import os +import subprocess from inspect import cleandoc import pytest @@ -14,6 +16,27 @@ ) +@pytest.fixture(scope="session") +def poetry_path() -> str: + result = subprocess.run(["which", "poetry"], capture_output=True, text=True) + poetry_path = result.stdout.strip() + return poetry_path + + +@pytest.fixture +def install_poetry_export(poetry_path, monkeypatch): + monkeypatch.setenv("PATH", poetry_path, prepend=os.pathsep) + + def _install(cwd): + subprocess.run( + ["poetry", "self", "add", "poetry-plugin-export"], + cwd=cwd, + check=True, + ) + + return _install + + class SampleVulnerability: package_name = "jinja2" version = "3.1.5" diff --git a/test/integration/conftest.py b/test/integration/conftest.py index 4fb7039e14..c19423388e 100644 --- a/test/integration/conftest.py +++ b/test/integration/conftest.py @@ -1,4 +1,3 @@ -import subprocess from pathlib import Path import pytest @@ -6,13 +5,6 @@ from exasol.toolbox.config import BaseConfig -@pytest.fixture(scope="session") -def poetry_path() -> str: - result = subprocess.run(["which", "poetry"], capture_output=True, text=True) - poetry_path = result.stdout.strip() - return poetry_path - - @pytest.fixture(scope="session") def ptb_minimum_python_version() -> str: """ diff --git a/test/unit/util/dependencies/audit_test.py b/test/unit/util/dependencies/audit_test.py index ed9ce84031..9dccc73694 100644 --- a/test/unit/util/dependencies/audit_test.py +++ b/test/unit/util/dependencies/audit_test.py @@ -1,4 +1,5 @@ import json +import re from inspect import cleandoc from pathlib import Path from subprocess import CompletedProcess @@ -13,6 +14,7 @@ Vulnerability, VulnerabilitySource, audit_poetry_files, + export_dependencies_to_file, get_vulnerabilities, get_vulnerabilities_from_latest_tag, ) @@ -94,7 +96,6 @@ def test_reference_links(sample_vulnerability, reference: str, expected: list[st ), ) def test_vulnerability_id(self, sample_vulnerability, aliases: list[str], expected): - result = Vulnerability( package=sample_vulnerability.vulnerability.package, id="DUMMY_IDENTIFIER", @@ -124,6 +125,62 @@ def test_subsection_for_changelog_summary(self, sample_vulnerability): ) +@pytest.fixture(scope="module") +def new_pyproject_toml(create_new_poetry_project, project_path): + return (project_path / "pyproject.toml").read_text() + + +class TestExportDependenciesToFile: + PACKAGES = [ + "astroid", + "black", # group - analysis + "click", + "colorama", + "dill", + "isort", # group - dev + "mccabe", + "mypy-extensions", + "packaging", + "pathspec", + "platformdirs", + "pylint", # main + "ruff", # optional-dependencies + "tomli", + "tomlkit", + "typing-extensions", + ] + + @staticmethod + def extract_package_names(content) -> list[str]: + return re.findall( + r"^([a-zA-Z0-9\-_]+)(?===|>=|<=|>|<|@)", content, re.MULTILINE + ) + + @pytest.mark.parametrize( + "pyproject_content", + [ + "poetry_2_1_pyproject_text", + "poetry_2_3_pyproject_text", + "new_pyproject_toml", + ], + ) + def test_poetry_export_versions( + self, install_poetry_export, tmp_path, pyproject_content, request + ): + content_str = request.getfixturevalue(pyproject_content) + (tmp_path / "pyproject.toml").write_text(content_str) + requirements_txt = tmp_path / "requirements.txt" + + install_poetry_export(cwd=tmp_path) + + export_dependencies_to_file( + output_file=requirements_txt, working_directory=tmp_path + ) + + content = requirements_txt.read_text() + assert self.extract_package_names(content) == self.PACKAGES + + class TestAuditPoetryFiles: @staticmethod @mock.patch("subprocess.run") diff --git a/test/unit/util/dependencies/conftest.py b/test/unit/util/dependencies/conftest.py new file mode 100644 index 0000000000..055b793809 --- /dev/null +++ b/test/unit/util/dependencies/conftest.py @@ -0,0 +1,114 @@ +import subprocess +from enum import Enum + +import pytest + + +class SampleVersions(str, Enum): + black = "25.1.0" + isort = "6.0.1" + pylint = "3.3.7" + ruff = "0.14.14" + + +@pytest.fixture(scope="module") +def sample_versions(): + return SampleVersions + + +@pytest.fixture(scope="module") +def poetry_2_1_pyproject_text(sample_versions) -> str: + return f""" + [project] + name = "project" + version = "0.1.0" + description = "" + authors = [] + readme = "README.md" + requires-python = ">=3.10" + dependencies = [ + "pylint (=={sample_versions.pylint})" + ] + + [tool.poetry] + packages = [{{include = "project", from = "src"}}] + + [tool.poetry.group.dev.dependencies] + isort = "{sample_versions.isort}" + + [tool.poetry.group.analysis.dependencies] + black = "{sample_versions.black}" + + [project.optional-dependencies] + ruff = [ "ruff (=={sample_versions.ruff})" ] + + [build-system] + requires = ["poetry-core>=2.0.0,<3.0.0"] + build-backend = "poetry.core.masonry.api" + """ + + +@pytest.fixture(scope="module") +def poetry_2_3_pyproject_text(sample_versions) -> str: + return f""" + [project] + name = "project" + version = "0.1.0" + description = "" + authors = [] + readme = "README.md" + requires-python = ">=3.10" + dependencies = [ + "pylint (=={sample_versions.pylint})" + ] + + [tool.poetry] + packages = [{{include = "project", from = "src"}}] + + [dependency-groups] + dev = [ + "isort=={sample_versions.isort}", + ] + analysis = [ + "black=={sample_versions.black}" + ] + + [project.optional-dependencies] + ruff = [ "ruff (=={sample_versions.ruff})" ] + + [build-system] + requires = ["poetry-core>=2.0.0,<3.0.0"] + build-backend = "poetry.core.masonry.api" + """ + + +@pytest.fixture(scope="module") +def cwd(tmp_path_factory): + return tmp_path_factory.mktemp("test") + + +@pytest.fixture(scope="module") +def project_name(): + return "project" + + +@pytest.fixture(scope="module") +def project_path(cwd, project_name): + return cwd / project_name + + +@pytest.fixture(scope="module") +def create_new_poetry_project( + poetry_path, cwd, project_name, project_path, sample_versions +): + subprocess.run([poetry_path, "new", project_name], cwd=cwd, check=True) + + commands = [ + [poetry_path, "self", "add", "poetry-plugin-export"], + [poetry_path, "add", f"pylint=={sample_versions.pylint}"], + [poetry_path, "add", "--group", "dev", f"isort=={sample_versions.isort}"], + [poetry_path, "add", "--group", "analysis", f"black=={sample_versions.black}"], + [poetry_path, "add", f"ruff@{sample_versions.ruff}", "--optional", "ruff"], + ] + for cmd in commands: + subprocess.run(cmd, cwd=project_path, env={}, check=True) diff --git a/test/unit/util/dependencies/poetry_dependencies_test.py b/test/unit/util/dependencies/poetry_dependencies_test.py index e1983bbadc..ecc7144f51 100644 --- a/test/unit/util/dependencies/poetry_dependencies_test.py +++ b/test/unit/util/dependencies/poetry_dependencies_test.py @@ -1,4 +1,3 @@ -import subprocess from inspect import cleandoc import pytest @@ -14,107 +13,42 @@ from noxconfig import PROJECT_CONFIG MAIN_GROUP = PoetryGroup(name="main", toml_section="project.dependencies") -DEV_GROUP = PoetryGroup(name="dev", toml_section="dependency-groups.dev") -ANALYSIS_GROUP = PoetryGroup(name="analysis", toml_section="dependency-groups.analysis") - -PYLINT = Package(name="pylint", version="3.3.7") -ISORT = Package(name="isort", version="6.0.1") -BLACK = Package(name="black", version="25.1.0") - -DIRECT_DEPENDENCIES = { - MAIN_GROUP.name: {PYLINT.name: PYLINT}, - DEV_GROUP.name: {ISORT.name: ISORT}, - ANALYSIS_GROUP.name: {BLACK.name: BLACK}, -} - - -@pytest.fixture(scope="module") -def cwd(tmp_path_factory): - return tmp_path_factory.mktemp("test") - - -@pytest.fixture(scope="module") -def project_name(): - return "project" - - -@pytest.fixture(scope="module") -def project_path(cwd, project_name): - return cwd / project_name - - -@pytest.fixture(scope="module") -def create_poetry_project(cwd, project_name, project_path): - subprocess.run(["poetry", "new", project_name], cwd=cwd) - subprocess.run( - ["poetry", "add", f"{PYLINT.name}=={PYLINT.version}"], cwd=project_path - ) - subprocess.run( - ["poetry", "add", "--group", "dev", f"{ISORT.name}=={ISORT.version}"], - cwd=project_path, - ) - subprocess.run( - ["poetry", "add", "--group", "analysis", f"{BLACK.name}=={BLACK.version}"], - cwd=project_path, - ) +GROUPS = ( + MAIN_GROUP, + PoetryGroup(name="dev", toml_section="dependency-groups.dev"), + PoetryGroup(name="analysis", toml_section="dependency-groups.analysis"), +) @pytest.fixture(scope="module") -def created_pyproject_toml(project_path, create_poetry_project): +def new_pyproject_toml(project_path, create_new_poetry_project): return PoetryToml.load_from_toml(working_directory=project_path) @pytest.fixture(scope="module") -def poetry_2_1_pyproject_toml(cwd, create_poetry_project): +def poetry_2_1_pyproject_toml(cwd, poetry_2_1_pyproject_text): older_project_path = cwd / "older_project" - pyproject_toml_text = """ - [project] - name = "project" - version = "0.1.0" - description = "" - authors = [] - readme = "README.md" - requires-python = ">=3.10" - dependencies = [ - "pylint (==3.3.7)" - ] - - [tool.poetry] - packages = [{include = "project", from = "src"}] - - - [tool.poetry.group.dev.dependencies] - isort = "6.0.1" - - - [tool.poetry.group.analysis.dependencies] - black = "25.1.0" - - [build-system] - requires = ["poetry-core>=2.0.0,<3.0.0"] - build-backend = "poetry.core.masonry.api" - """ older_project_path.mkdir(parents=True, exist_ok=True) pyproject_toml_path = older_project_path / "pyproject.toml" - pyproject_toml_path.write_text(cleandoc(pyproject_toml_text)) + pyproject_toml_path.write_text(cleandoc(poetry_2_1_pyproject_text)) return PoetryToml.load_from_toml(working_directory=older_project_path) @pytest.mark.slow class TestPoetryToml: @staticmethod - def test_get_section_dict_exists(created_pyproject_toml): - result = created_pyproject_toml.get_section_dict("project") + def test_get_section_dict_exists(new_pyproject_toml): + result = new_pyproject_toml.get_section_dict("project") assert result is not None @staticmethod - def test_get_section_dict_does_not_exist(created_pyproject_toml): - result = created_pyproject_toml.get_section_dict("test") + def test_get_section_dict_does_not_exist(new_pyproject_toml): + result = new_pyproject_toml.get_section_dict("test") assert result is None @staticmethod - def test_groups(created_pyproject_toml): - assert created_pyproject_toml.groups == (MAIN_GROUP, DEV_GROUP, ANALYSIS_GROUP) + def test_groups(new_pyproject_toml): + assert new_pyproject_toml.groups == GROUPS @staticmethod def test_groups_with_poetry_2_1_0(poetry_2_1_pyproject_toml): @@ -160,25 +94,41 @@ def test_extract_from_line(line, expected): @pytest.mark.slow @staticmethod - def test_direct_dependencies(create_poetry_project, project_path): + def test_direct_dependencies( + create_new_poetry_project, project_path, sample_versions + ): poetry_dep = PoetryDependencies( - groups=(MAIN_GROUP, DEV_GROUP, ANALYSIS_GROUP), + groups=GROUPS, working_directory=project_path, ) - assert poetry_dep.direct_dependencies == DIRECT_DEPENDENCIES + assert poetry_dep.direct_dependencies == { + "main": { + "pylint": Package(name="pylint", version=sample_versions.pylint), + "ruff": Package(name="ruff", version=sample_versions.ruff), + }, + "dev": {"isort": Package(name="isort", version=sample_versions.isort)}, + "analysis": {"black": Package(name="black", version=sample_versions.black)}, + } @pytest.mark.slow @staticmethod - def test_all_dependencies(create_poetry_project, project_path): + def test_all_dependencies(create_new_poetry_project, project_path, sample_versions): poetry_dep = PoetryDependencies( - groups=(MAIN_GROUP, DEV_GROUP, ANALYSIS_GROUP), + groups=GROUPS, working_directory=project_path, ) result = poetry_dep.all_dependencies transitive = result.pop("transitive") assert len(transitive) > 0 - assert result == DIRECT_DEPENDENCIES + assert result == { + "main": { + "pylint": Package(name="pylint", version=sample_versions.pylint), + "ruff": Package(name="ruff", version=sample_versions.ruff), + }, + "dev": {"isort": Package(name="isort", version=sample_versions.isort)}, + "analysis": {"black": Package(name="black", version=sample_versions.black)}, + } @pytest.mark.slow