diff --git a/Dockerfile b/Dockerfile index c6216aa69a..cbdb747933 100644 --- a/Dockerfile +++ b/Dockerfile @@ -104,6 +104,7 @@ RUN apt-get update \ libgpgme11 \ libdevmapper1.02.1 \ libmagic1 \ + docker.io \ git \ wait-for-it \ universal-ctags \ diff --git a/docker-compose.yml b/docker-compose.yml index f612b4523c..7a4fec3ba7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -31,6 +31,16 @@ services: timeout: 5s retries: 5 + # Add the Docker-in-Docker daemon + dind: + image: docker.io/library/docker:dind + privileged: true + environment: + - DOCKER_TLS_CERTDIR= + volumes: + - dind_data:/var/lib/docker + - workspace:/var/scancodeio/workspace/ + web: build: . command: sh -c " @@ -38,6 +48,9 @@ services: ./manage.py collectstatic --no-input --verbosity 0 --clear && gunicorn scancodeio.wsgi:application --bind :8000 --timeout 600 \ --workers ${GUNICORN_WORKERS:-8} --worker-tmp-dir /dev/shm" + environment: + - DOCKER_HOST=tcp://dind:2375 # Point to the DinD container + - SCANCODE_NIXPKGS_CACHE_DIR=/var/scancodeio/workspace/.scancode-nix env_file: - docker.env expose: @@ -62,6 +75,9 @@ services: ./manage.py rqworker --worker-class scancodeio.worker.ScanCodeIOWorker --queue-class scancodeio.worker.ScanCodeIOQueue --verbosity 1" + environment: + - DOCKER_HOST=tcp://dind:2375 # Point to the DinD container + - SCANCODE_NIXPKGS_CACHE_DIR=/var/scancodeio/workspace/.scancode-nix env_file: - docker.env volumes: @@ -77,6 +93,8 @@ services: condition: service_healthy web: condition: service_started + dind: + condition: service_started nginx: image: docker.io/library/nginx:1.31.4-alpine @@ -114,4 +132,5 @@ volumes: static: workspace: webroot: - healthycode_venv: \ No newline at end of file + dind_data: + healthycode_venv: diff --git a/docs/built-in-pipelines.rst b/docs/built-in-pipelines.rst index 644f81ee03..4368ab4eb3 100644 --- a/docs/built-in-pipelines.rst +++ b/docs/built-in-pipelines.rst @@ -281,6 +281,12 @@ Scan Maven Package :members: :member-order: bysource +Scan Nix Package +------------------- +.. autoclass:: scanpipe.pipelines.scan_nix_package.ScanNixPackage() + :members: + :member-order: bysource + Fetch Scores (addon) -------------------- .. warning:: diff --git a/pyproject.toml b/pyproject.toml index 341394736c..eed9555e33 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -175,6 +175,7 @@ resolve_dependencies = "scanpipe.pipelines.resolve_dependencies:ResolveDependenc scan_codebase = "scanpipe.pipelines.scan_codebase:ScanCodebase" scan_for_virus = "scanpipe.pipelines.scan_for_virus:ScanForVirus" scan_maven_package = "scanpipe.pipelines.scan_maven_package:ScanMavenPackage" +scan_nix_package = "scanpipe.pipelines.scan_nix_package:ScanNixPackage" scan_single_package = "scanpipe.pipelines.scan_single_package:ScanSinglePackage" scan_repo_health = "scanpipe.pipelines.scan_repo_health:ScanRepoHealth" diff --git a/scanpipe/pipelines/__init__.py b/scanpipe/pipelines/__init__.py index 53f145d50c..267433ec72 100644 --- a/scanpipe/pipelines/__init__.py +++ b/scanpipe/pipelines/__init__.py @@ -107,14 +107,14 @@ def extract_archive(self, location, target): details=details, ) - def extract_archives(self, location=None): + def extract_archives(self, location=None, recurse=True): """Extract archives located in the codebase/ directory with extractcode.""" from scanpipe.pipes import scancode if not location: location = self.project.codebase_path - extract_errors = scancode.extract_archives(location=location, recurse=True) + extract_errors = scancode.extract_archives(location=location, recurse=recurse) for resource_path, errors in extract_errors.items(): self.project.add_error( diff --git a/scanpipe/pipelines/scan_nix_package.py b/scanpipe/pipelines/scan_nix_package.py new file mode 100644 index 0000000000..7984434091 --- /dev/null +++ b/scanpipe/pipelines/scan_nix_package.py @@ -0,0 +1,257 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# http://nexb.com and https://github.com/aboutcode-org/scancode.io +# The ScanCode.io software is licensed under the Apache License version 2.0. +# Data generated with ScanCode.io is provided as-is without warranties. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode.io should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# +# ScanCode.io is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/aboutcode-org/scancode.io for support and download. + +import shutil +from pathlib import Path + +from scanpipe.pipelines.deploy_to_develop import DeployToDevelop +from scanpipe.pipelines.scan_codebase import ScanCodebase +from scanpipe.pipelines.scan_single_package import ScanSinglePackage +from scanpipe.pipes import d2d +from scanpipe.pipes import flag +from scanpipe.pipes import nix +from scanpipe.pipes import utils +from scanpipe.pipes.nix import check_input_and_return_purl +from scanpipe.pipes.nix import fetch_inputs + + +class ScanNixPackage(ScanSinglePackage, DeployToDevelop, ScanCodebase): + """ + Fetch and build a Nix package to detect discrepancies and validate licenses. + + Download the nix source and binary, and run a deployment to development + scan between the binary and the source to detect any discrepancies. + + Scan the sources and confirm that the detected license aligns with + the declared license that is detected from the nix package. + """ + + download_inputs = False + + @classmethod + def steps(cls): + return ( + cls.check_input_and_return_purl, + cls.check_docker_command, + cls.fetch_inputs, + cls.collect_input_info, + cls.extract_input_to_codebase_directory, + cls.extract_codebase_archives, + cls.run_scan, + cls.load_inventory_from_toolkit_scan, + cls.add_from_to_tag, + cls.d2d_steps, + cls.validate_package_license_integrity, + cls.flag_mapped_status, + cls.make_summary_from_scan_results, + ) + + def check_input_and_return_purl(self): + """Validate the input is a PURL string and return the PURL object.""" + self.purl = check_input_and_return_purl(self.project) + + def check_docker_command(self): + """Check if the Docker command is available and multiarch is ready.""" + if not utils.check_docker_command(): + raise RuntimeError("Docker is required and its daemon must be running.") + if not nix.ensure_multiarch_emulation(): + raise RuntimeError( + "Could not install binfmt multi-arch emulators. " + "Cross-architecture Nix builds will not work." + ) + nix.prepare_nix_environment() + + def fetch_inputs(self): + """Fetch the binary and source of the given PURL.""" + from_file = "" + to_file = "" + output_format = "" + try: + from_file, to_file, output_format, error_messages, warning_messages = ( + fetch_inputs(self.purl, self.project.codebase_path) + ) + finally: + nix.cleanup_nixpkgs_worktrees() + self.from_file = from_file + self.to_file = to_file + self.output_format = output_format + + self.d2d_enable = bool(self.from_file and self.to_file) + + if error_messages: + for error_message in error_messages: + self.project.add_error(error_message) + if warning_messages: + for warning_message in warning_messages: + self.project.add_warning(warning_message) + + def collect_input_info(self): + """Collect information about the input.""" + self.input_path = "" + if self.to_file: + self.input_path = Path(self.to_file) + self.collect_input_information() + + def extract_input_to_codebase_directory(self): + """Extract input to project codebase/ directory.""" + if self.input_path: + extracted_path = nix.extract_nar_archive( + self.input_path, self.project.codebase_path, self.output_format + ) + + to_dir = Path(self.project.codebase_path) / "to" + # If the extraction failed (returned "") or we found it was empty + if not extracted_path or (to_dir.exists() and not list(to_dir.rglob("*"))): + if to_dir.exists(): + shutil.rmtree(to_dir) + self.d2d_enable = False + self.project.add_error( + "Failed to extract NAR archive, D2D scan disabled." + ) + + self.env = self.project.get_env() + + def extract_codebase_archives(self): + """Perform extraction of the codebase resources.""" + self.extract_archives(recurse=True) + + def clear_to_codebase_status(self): + """ + Clear the status of the to codebase resources in the project as + having status will prevent D2D from running. + """ + flag.clear_status(self.project.codebaseresources.to_codebase()) + + def add_from_to_tag(self): + """Update 'from' and 'to' tag to resources based on their path.""" + if self.d2d_enable: + d2d.update_from_to_tag(self.project) + + def d2d_steps(self): + """ + Run the deployment to development scan if both the source and + binary are available. + """ + if self.d2d_enable: + self.flag_empty_files() + self.flag_whitespace_files() + self.flag_ignored_resources() + self.map_about_files() + self.map_checksum() + self.match_archives_to_purldb() + self.load_ecosystem_config() + self.d2d_java() + self.d2d_scala() + self.d2d_kotlin() + self.d2d_grammar() + self.d2d_groovy() + self.d2d_aspectj() + self.d2d_clojure() + self.d2d_xtend() + self.d2d_javascript() + self.d2d_haskell() + self.d2d_process() + + def d2d_java(self): + self.find_java_packages() + self.map_java_to_class() + self.map_jar_to_java_source() + + def d2d_scala(self): + self.find_scala_packages() + self.map_scala_to_class() + self.map_jar_to_scala_source() + + def d2d_kotlin(self): + self.find_kotlin_packages() + self.map_kotlin_to_class() + self.map_jar_to_kotlin_source() + + def d2d_grammar(self): + self.find_grammar_packages() + self.map_grammar_to_class() + self.map_jar_to_grammar_source() + + def d2d_groovy(self): + self.find_groovy_packages() + self.map_groovy_to_class() + self.map_jar_to_groovy_source() + + def d2d_aspectj(self): + self.find_aspectj_packages() + self.map_aspectj_to_class() + self.map_jar_to_aspectj_source() + + def d2d_clojure(self): + self.find_clojure_packages() + self.map_clojure_to_class() + self.map_jar_to_clojure_source() + + def d2d_xtend(self): + self.find_xtend_packages() + self.map_xtend_to_class() + + def d2d_javascript(self): + self.map_javascript() + self.map_javascript_symbols() + self.map_javascript_strings() + + def d2d_haskell(self): + self.map_haskell() + + def d2d_process(self): + self.get_symbols_from_binaries() + self.map_elf() + self.map_macho() + self.map_winpe() + self.map_go() + self.map_rust() + self.map_python() + self.match_directories_to_purldb() + self.match_resources_to_purldb() + self.map_javascript_post_purldb_match() + self.map_javascript_path() + self.map_javascript_colocation() + self.map_thirdparty_npm_packages() + self.map_path() + self.flag_mapped_resources_archives_and_ignored_directories() + self.perform_house_keeping_tasks() + self.match_purldb_resources_post_process() + self.remove_packages_without_resources() + self.scan_ignored_to_files() + self.scan_unmapped_to_files() + self.scan_mapped_from_for_files() + self.collect_and_create_license_detections() + self.flag_deployed_from_resources_with_missing_license() + self.create_local_files_packages() + + def validate_package_license_integrity(self): + """ + Validate the correctness of the package license compared with the + detected license from the codebase. + """ + utils.validate_package_license_integrity(self.project) + + def flag_mapped_status(self): + """Flag the from codebase resources that were mapped.""" + if self.d2d_enable: + flag.flag_mapped_resources(self.project) diff --git a/scanpipe/pipes/d2d.py b/scanpipe/pipes/d2d.py index dea2f8e69f..77604401d1 100644 --- a/scanpipe/pipes/d2d.py +++ b/scanpipe/pipes/d2d.py @@ -142,10 +142,14 @@ def _map_checksum_resource(to_resource, from_resources, checksum_field): def map_checksum(project, checksum_field, logger=None): """Map using checksum.""" - project_files = project.codebaseresources.files().no_status() - from_resources = project_files.from_codebase().has_value(checksum_field) + from_resources = ( + project.codebaseresources.files().from_codebase().has_value(checksum_field) + ) to_resources = ( - project_files.to_codebase().has_value(checksum_field).has_no_relation() + project.codebaseresources.files() + .to_codebase() + .has_value(checksum_field) + .has_no_relation() ) resource_count = to_resources.count() @@ -272,7 +276,7 @@ def find_jvm_packages(project, jvm_lang: jvm.JvmLanguage, logger=None): Note: we use the same API as the ScanCode scans by design """ - resources = project.codebaseresources.files().no_status().from_codebase() + resources = project.codebaseresources.files().from_codebase() from_jvm_resources = resources.filter(extension__in=jvm_lang.source_extensions) @@ -524,9 +528,8 @@ def _map_path_resource( def map_path(project, logger=None): """Map using path suffix similarities.""" - project_files = project.codebaseresources.files().no_status() - from_resources = project_files.from_codebase() - to_resources = project_files.to_codebase().has_no_relation() + from_resources = project.codebaseresources.files().from_codebase() + to_resources = project.codebaseresources.files().to_codebase().has_no_relation() resource_count = to_resources.count() if logger: @@ -1896,20 +1899,25 @@ def map_paths_resource( relations_to_create[rel_key] = relation if paths_not_mapped: to_resource.status = flag.REQUIRES_REVIEW - logger( - f"WARNING: #{len(paths_not_mapped)} {map_type} paths NOT mapped for: " - f"{to_resource.path!r}" - ) + if logger: + logger( + f"WARNING: #{len(paths_not_mapped)} {map_type} paths NOT " + f" mapped for: {to_resource.path!r}" + ) to_resource.save() if relations_to_create: rels = CodebaseRelation.objects.bulk_create(relations_to_create.values()) - logger( - f"Created {len(rels)} mappings using " - f"{', '.join(map_types)} for: {to_resource.path!r}" - ) + if logger: + logger( + f"Created {len(rels)} mappings using " + f"{', '.join(map_types)} for: {to_resource.path!r}" + ) else: - logger(f"No mappings using {', '.join(map_types)} for: {to_resource.path!r}") + if logger: + logger( + f"No mappings using {', '.join(map_types)} for: {to_resource.path!r}" + ) def process_paths_in_binary( @@ -2073,9 +2081,17 @@ def map_elfs_with_dwarf_paths(project, logger=None): f"with {from_resources.count():,d} from/ resources." ) - from_resources_index = pathmap.build_index( - from_resources.values_list("id", "path"), with_subpaths=True - ) + # Build the path index, adding virtual aliases for .in template files + from_paths = [] + for res_id, path in from_resources.values_list("id", "path"): + from_paths.append((res_id, path)) + # If the source file is a template ending in '.in', also index its + # target name + if path.endswith(".in"): + target_path = path[:-3] # Removes the trailing '.in' + from_paths.append((res_id, target_path)) + + from_resources_index = pathmap.build_index(from_paths, with_subpaths=True) if logger: logger("Done building from/ resources index.") @@ -2164,6 +2180,15 @@ def map_go_paths(project, logger=None): ) +def update_from_to_tag(project): + """Update 'from' or 'to' tag to resources based on their path.""" + for resource in project.codebaseresources.files(): + if resource.path.startswith("from/"): + resource.update(tag="from") + elif resource.path.startswith("to/"): + resource.update(tag="to") + + RUST_BINARY_OPTIONS = ["Rust"] ELF_BINARY_OPTIONS = ["Python", "Go", "Elf"] MACHO_BINARY_OPTIONS = ["Rust", "Go", "MacOS"] diff --git a/scanpipe/pipes/fetch.py b/scanpipe/pipes/fetch.py index 3cbbb13200..401824f76f 100644 --- a/scanpipe/pipes/fetch.py +++ b/scanpipe/pipes/fetch.py @@ -82,6 +82,13 @@ def get_request_session(uri): """Return a Requests session setup with authentication and headers.""" session = requests.Session() + + # Set a default User-Agent to avoid 403 Forbidden errors on strict + # registries that block default python-requests headers. + session.headers.update( + {"User-Agent": "ScanCode.io (https://github.com/aboutcode-org/scancode.io)"} + ) + netloc = urlparse(uri).netloc if credentials := scanpipe_settings.FETCH_BASIC_AUTH.get(netloc): diff --git a/scanpipe/pipes/flag.py b/scanpipe/pipes/flag.py index ad366045a0..087ff81935 100644 --- a/scanpipe/pipes/flag.py +++ b/scanpipe/pipes/flag.py @@ -66,6 +66,7 @@ REQUIRES_REVIEW = "requires-review" REVIEW_DANGLING_LEGAL_FILE = "review-dangling-legal-file" NOT_DEPLOYED = "not-deployed" +LICENSE_ISSUE = "license-mismatch-declared-vs-detected" GENERATED = "generated-file" @@ -138,3 +139,8 @@ def flag_mapped_resources(project): """Flag all codebase resources that were mapped during the d2d pipeline.""" resources = project.codebaseresources.has_relation().no_status() return resources.update(status=MAPPED) + + +def clear_status(resource_qs): + """Clear the status of given codebase resources.""" + return resource_qs.update(status="") diff --git a/scanpipe/pipes/nix.py b/scanpipe/pipes/nix.py new file mode 100644 index 0000000000..0e5df025b8 --- /dev/null +++ b/scanpipe/pipes/nix.py @@ -0,0 +1,1321 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# http://nexb.com and https://github.com/aboutcode-org/scancode.io +# The ScanCode.io software is licensed under the Apache License version 2.0. +# Data generated with ScanCode.io is provided as-is without warranties. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode.io should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# +# ScanCode.io is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/aboutcode-org/scancode.io for support and download. + +import atexit +import hashlib +import logging +import os +import shutil +import subprocess +import tempfile +import time +import uuid +from collections import namedtuple +from pathlib import Path + +import requests +from fetchcode import fetch_json_response +from packageurl import PackageURL + +from scanpipe.pipes import utils + +logger = logging.getLogger(__name__) + + +# Result of `get_patched_source_with_docker`: +# - `path`: extracted source tree, or "" when nothing could be produced +# - `used_fallback`: True when the patched-source build failed and we fell +# back to the raw upstream `pkg.src` (unpatched) +# - `fallback_reason`: reason for the fallback, or "" +# - `failure_detail`: reason the fetch failed, or "" +PatchedSourceResult = namedtuple( + "PatchedSourceResult", + ["path", "used_fallback", "fallback_reason", "failure_detail"], +) + +FALLBACK_REASON_PREFIX = "PATCHED_SOURCE_FALLBACK_REASON=" + + +_DOCKER_BUILD_TIMEOUT = 1800 +_DOCKER_IO_TIMEOUT = 600 +_DOCKER_EVAL_TIMEOUT = 300 + + +# Markers to identify meaningful failure lines in `nix-build` stderr. +_NIX_ERROR_MARKERS = ( + "Encountered missing or private dependencies:", + "error: Cannot build", + "error: builder failed", +) + + +def _summarize_nix_build_error(stderr, max_lines=10): + """ + Extract the meaningful failure from `nix-build` stderr. + + nix-build writes hundreds of lines of progress output ("copying + path...", phase names) around the one or two lines that explain + why the build failed. Return a short excerpt so the project's error + message is actionable. + """ + if not stderr: + return "" + + lines = stderr.strip().splitlines() + for i, line in enumerate(lines): + if any(marker in line for marker in _NIX_ERROR_MARKERS): + summary = "\n".join(lines[i : i + max_lines]) + if "Encountered missing or private dependencies:" in line: + summary += ( + "\n\nHint: this Haskell package targets an older compiler than " + "the one in this nixpkgs commit. Try an older commit, or a " + "Haskell package set that uses a matching compiler." + ) + return summary + + return lines[-1] if lines else "" + + +# Each scan runs in its own RQ worker process, so several scans can run at +# once. They all share three things on the host: +# +# 1. The custom Nix Docker image. +# 2. The nixpkgs git repo and its worktrees. +# 3. The `nix-eval-cache` Docker volume (the Nix store). +# +# We use file locks (`fcntl.flock`) so only one process touches any of +# these at a time. The kernel drops the lock when the holder exits, even on +# a crash, so a dead worker can't leave a lock stuck. We don't want to set +# a timeout as the first nixpkgs clone can take minutes, and we don't want +# to treat a slow scan as a deadlock. +# +# The timestamp file is not a lock: it records when the Nix store GC +# (Garbage Collector) last ran, so GC runs at most once per interval +# instead of on every scan. +_NIXPKGS_CACHE_DIR = Path( + os.environ.get( + "SCANCODE_NIXPKGS_CACHE_DIR", + str(Path(os.path.expanduser("~")) / ".cache" / "scancode-nix"), + ) +) +_NIXPKGS_BARE_REPO = _NIXPKGS_CACHE_DIR / "nixpkgs.git" +_NIXPKGS_REPO_URL = os.environ.get( + "SCANCODE_NIXPKGS_REPO_URL", + "https://github.com/NixOS/nixpkgs.git", +) +_NIXPKGS_CONTAINER_PATH = "/nixpkgs-local" +_NIXPKGS_WORKTREE_CACHE = {} + +_NIX_IMAGE_LOCK_PATH = _NIXPKGS_CACHE_DIR / "nix-image.lock" +_NIXPKGS_REPO_LOCK_PATH = _NIXPKGS_CACHE_DIR / "nixpkgs-repo.lock" +_NIX_GC_LOCK_PATH = _NIXPKGS_CACHE_DIR / "nix-gc.lock" +_NIX_GC_TIMESTAMP_PATH = _NIXPKGS_CACHE_DIR / "nix-gc.timestamp" + + +# `extract_nar_archive()` needs zstd and xz to decompress `.nar.zst` or +# `.nar.xz`. The stock `nixos/nix` image doesn't ship them, so the +# extraction step fetches them via `nix-shell -p` on every scan from +# `cache.nixos.org`. The Nix cache server throttles these repeated requests +# by slowing the response or returning 429/503 errors, causing the +# extraction to time out. +# +# To avoid that, we build a custom Nix image with zstd, xz, bzip2, and gzip +# installed. +# +# The Dockerfile also registers the image's base tooling (bash and the +# nix-* commands) as GC roots. Without this, `nix-collect-garbage` running +# against the shared `nix-eval-cache` volume would delete the image's own +# dependencies and leave `/bin/sh` pointing at a removed store path. +_NIX_IMAGE_DOCKERFILE = """\ +FROM nixos/nix +RUN nix-channel --add https://nixos.org/channels/nixos-24.05 nixpkgs \\ + && nix-channel --update \\ + && nix-env -iA nixpkgs.zstd nixpkgs.xz nixpkgs.bzip2 nixpkgs.gzip \\ + && GCROOTS=/nix/var/nix/gcroots/base \\ + && mkdir -p "$GCROOTS" \\ + && ln -sf "$(readlink -f /bin/sh)" "$GCROOTS/sh" \\ + && for tool in nix-build nix-store nix-env nix-instantiate nix-collect-garbage; do \\ + ln -sf "$(readlink -f "$(command -v $tool)")" "$GCROOTS/$tool"; \\ + done +""" + +# Generate a unique image tag from a hash of the Dockerfile contents. +# This ensures that modifying the Dockerfile automatically changes the tag, +# forcing Docker to rebuild the image instead of reusing a stale cached version +_NIX_IMAGE_TAG = hashlib.sha256(_NIX_IMAGE_DOCKERFILE.encode()).hexdigest()[:8] +NIX_IMAGE = f"scancode-nix:{_NIX_IMAGE_TAG}" + +_resolved_nix_image = None + + +def _ensure_nix_image(): + """ + Ensure the custom Nix image exists locally, building it if necessary. + Return the image name to use for `docker run`. + Uses a file lock to safely support concurrent calls. + """ + global _resolved_nix_image + if _resolved_nix_image is not None: + return _resolved_nix_image + + if utils.docker_image_exists(NIX_IMAGE): + _resolved_nix_image = NIX_IMAGE + return _resolved_nix_image + + # Lock the file to avoid crashing during concurrent runs. + with utils.file_lock(_NIX_IMAGE_LOCK_PATH): + # Another process may have built it while we waited for the lock. + if utils.docker_image_exists(NIX_IMAGE): + _resolved_nix_image = NIX_IMAGE + return _resolved_nix_image + + logger.info(f"Custom Nix image '{NIX_IMAGE}' not found. Building it now.") + + with tempfile.TemporaryDirectory() as tmpdir: + dockerfile_path = Path(tmpdir) / "Dockerfile" + dockerfile_path.write_text(_NIX_IMAGE_DOCKERFILE) + build_cmd = [ + "docker", + "build", + "-t", + NIX_IMAGE, + "-f", + str(dockerfile_path), + tmpdir, + ] + try: + subprocess.run( # noqa: S603 + build_cmd, + capture_output=True, + text=True, + check=True, + timeout=_DOCKER_BUILD_TIMEOUT, + ) + except subprocess.CalledProcessError as e: + raise RuntimeError( + f"Failed to build {NIX_IMAGE}: {e.stderr.strip()}" + ) from e + except subprocess.TimeoutExpired as e: + raise RuntimeError( + f"Timeout building {NIX_IMAGE} after {_DOCKER_BUILD_TIMEOUT}s." + ) from e + + _resolved_nix_image = NIX_IMAGE + return _resolved_nix_image + + +# The `nix-eval-cache` volume holds the Nix store, which caches the +# packages and dependencies that scans fetch and build. Its disk usage +# grows as more distinct packages are scanned. To keep it from growing +# forever, we periodically run: +# +# nix-collect-garbage --delete-older-than +# +# inside the custom image, against the shared volume. This removes store +# paths that have not been read or written for the defined period. +# +# A timestamp file limits how often GC runs: at most once every +# `SCANCODE_NIX_GC_INTERVAL_HOURS` hours, and only for store paths older +# than `SCANCODE_NIX_GC_OLDER_THAN`. Defaults: 3 hours and 12 hours. +# +# The GC does NOT touch the custom Nix image. The image is built once and +# survives GC, host reboots, and even full volume deletion. +# +# SCANCODE_NIX_GC_INTERVAL_HOURS — run GC at most this often. 0 disables. +# SCANCODE_NIX_GC_OLDER_THAN — passed to `--delete-older-than`. + +_NIX_GC_INTERVAL_HOURS = int(os.environ.get("SCANCODE_NIX_GC_INTERVAL_HOURS", "3")) +_NIX_GC_OLDER_THAN = os.environ.get("SCANCODE_NIX_GC_OLDER_THAN", "12h") + + +def _gc_due(now): + """Return True if GC has not run for at least the configured interval.""" + if not _NIX_GC_TIMESTAMP_PATH.exists(): + return True + try: + last_run = float(_NIX_GC_TIMESTAMP_PATH.read_text().strip()) + except (ValueError, OSError): + # Corrupt or unreadable timestamp; treat as "due". + return True + return (now - last_run) >= _NIX_GC_INTERVAL_HOURS * 3600 + + +def _run_nix_gc_if_due(): + """ + Run `nix-collect-garbage --delete-older-than + ` on the shared Nix store cache, but at + most once per `SCANCODE_NIX_GC_INTERVAL_HOURS`. + """ + if _NIX_GC_INTERVAL_HOURS <= 0: + return + + now = time.time() + if not _gc_due(now): + return + + nix_image = _ensure_nix_image() + + with utils.file_lock(_NIX_GC_LOCK_PATH): + if not _gc_due(now): + return + + logger.info( + f"Running nix-collect-garbage --delete-older-than " + f"{_NIX_GC_OLDER_THAN} on the shared Nix store cache." + ) + cmd = [ + "docker", + "run", + "--rm", + "--init", + "-v", + "nix-eval-cache:/nix", + nix_image, + "nix-collect-garbage", + "--delete-older-than", + _NIX_GC_OLDER_THAN, + ] + try: + subprocess.run( # noqa: S603 + cmd, + capture_output=True, + text=True, + timeout=_DOCKER_IO_TIMEOUT, + ) + except subprocess.TimeoutExpired: + logger.warning("nix-collect-garbage timed out; skipping.") + return + + try: + _NIX_GC_TIMESTAMP_PATH.write_text(str(now)) + except OSError as e: + logger.debug(f"Could not write GC timestamp: {e}") + + +def prepare_nix_environment(): + """Ensure the custom Nix image exists and run the periodic store GC.""" + _ensure_nix_image() + _run_nix_gc_if_due() + + +# Nixpkgs is a huge repo, and evaluating a package needs the full tree at +# the exact commit referenced by the PURL. Without a local clone, every scan +# fetches a tarball from GitHub. At scale, GitHub rate-limits these requests +# and the pipeline stalls. +# +# So we keep one bare git clone of nixpkgs on the host. Each scan +# shallow fetches its commit, creates a temporary worktree, and bind-mounts +# it into the container read-only. The Nix expression imports from that path +# instead of `fetchTarball`, so no network access is needed inside the +# container. +def _clone_nixpkgs_bare_repo(): + """ + Clone the nixpkgs bare repo. + Return the repo path, or "" on failure. + """ + _NIXPKGS_CACHE_DIR.mkdir(parents=True, exist_ok=True) + logger.info( + f"First run: creating a local bare clone of nixpkgs at {_NIXPKGS_BARE_REPO}." + ) + + cmd = [ + "git", + "clone", + "--bare", + "--filter=blob:none", + "--no-tags", + _NIXPKGS_REPO_URL, + str(_NIXPKGS_BARE_REPO), + ] + try: + subprocess.run( # noqa: S603 + cmd, + check=True, + capture_output=True, + text=True, + timeout=3600, + ) + return str(_NIXPKGS_BARE_REPO) + except subprocess.CalledProcessError as e: + logger.error( + f"Failed to create nixpkgs bare clone: {e.stderr.strip()}. " + f"Falling back to fetchTarball." + ) + shutil.rmtree(_NIXPKGS_BARE_REPO, ignore_errors=True) + except subprocess.TimeoutExpired: + logger.error( + "Timeout creating nixpkgs bare clone; falling back to fetchTarball." + ) + shutil.rmtree(_NIXPKGS_BARE_REPO, ignore_errors=True) + return "" + + +def _ensure_nixpkgs_bare_repo(): + """Ensure a local bare clone of nixpkgs exists.""" + if os.environ.get("SCANCODE_NIXPKGS_USE_LOCAL_CACHE", "1") == "0": + return "" + + if not shutil.which("git"): + logger.warning( + "git is not available; falling back to fetchTarball for nixpkgs. " + "Install git to enable the local cache and avoid GitHub rate limits." + ) + return "" + + if _NIXPKGS_BARE_REPO.exists(): + return str(_NIXPKGS_BARE_REPO) + + with utils.file_lock(_NIXPKGS_REPO_LOCK_PATH): + # Re-check inside the lock: another process may have cloned while + # we were waiting. + if _NIXPKGS_BARE_REPO.exists(): + return str(_NIXPKGS_BARE_REPO) + return _clone_nixpkgs_bare_repo() + + +def _prepare_nixpkgs_worktree(commit_hash): + """ + Fetch `commit_hash` into the bare repo and create a worktree + for it. Return the worktree's host path, or "" on failure. + """ + bare_repo = _ensure_nixpkgs_bare_repo() + if not bare_repo: + return "" + + worktree_dir = _NIXPKGS_CACHE_DIR / "worktrees" + worktree_dir.mkdir(parents=True, exist_ok=True) + worktree_path = worktree_dir / f"{commit_hash}-{uuid.uuid4().hex[:8]}" + + with utils.file_lock(_NIXPKGS_REPO_LOCK_PATH): + try: + fetch_cmd = [ + "git", + "-C", + bare_repo, + "fetch", + "--depth", + "1", + "--no-tags", + "origin", + commit_hash, + ] + subprocess.run( # noqa: S603 + fetch_cmd, + check=True, + capture_output=True, + text=True, + timeout=600, + ) + worktree_cmd = [ + "git", + "-C", + bare_repo, + "worktree", + "add", + "--detach", + str(worktree_path), + commit_hash, + ] + subprocess.run( # noqa: S603 + worktree_cmd, + check=True, + capture_output=True, + text=True, + timeout=180, + ) + return str(worktree_path) + except subprocess.CalledProcessError as e: + logger.warning( + f"Failed to prepare nixpkgs worktree for commit {commit_hash}: " + f"{e.stderr.strip()}. Falling back to fetchTarball." + ) + shutil.rmtree(worktree_path, ignore_errors=True) + except subprocess.TimeoutExpired: + logger.warning( + f"Timeout preparing nixpkgs worktree for commit {commit_hash}; " + f"falling back to fetchTarball." + ) + shutil.rmtree(worktree_path, ignore_errors=True) + return "" + + +def _get_nixpkgs_worktree(commit_hash): + """ + Return a worktree path for `commit_hash`, cached per process. Fetched + once, reused until `cleanup_worktrees` removes it. + """ + if commit_hash in _NIXPKGS_WORKTREE_CACHE: + return _NIXPKGS_WORKTREE_CACHE[commit_hash] + + worktree_path = _prepare_nixpkgs_worktree(commit_hash) + if worktree_path: + _NIXPKGS_WORKTREE_CACHE[commit_hash] = worktree_path + return worktree_path + + +def _remove_nixpkgs_worktree(worktree_path): + """Remove a worktree created by `_prepare_nixpkgs_worktree`.""" + if not worktree_path: + return + + bare_repo = str(_NIXPKGS_BARE_REPO) + if not Path(bare_repo).exists(): + shutil.rmtree(worktree_path, ignore_errors=True) + return + + with utils.file_lock(_NIXPKGS_REPO_LOCK_PATH): + try: + cmd = [ + "git", + "-C", + bare_repo, + "worktree", + "remove", + "--force", + worktree_path, + ] + subprocess.run( # noqa: S603 + cmd, + check=True, + capture_output=True, + text=True, + timeout=60, + ) + except (subprocess.CalledProcessError, subprocess.TimeoutExpired): + # Remove the directory directly, then prune the now-stale metadata. + shutil.rmtree(worktree_path, ignore_errors=True) + try: + prune_cmd = ["git", "-C", bare_repo, "worktree", "prune"] + subprocess.run( # noqa: S603 + prune_cmd, + capture_output=True, + text=True, + timeout=30, + ) + except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as e: + logger.debug(f"Failed to prune git worktrees: {e}") + + +def cleanup_nixpkgs_worktrees(): + """ + Remove every nixpkgs worktree created during this process's lifetime + and clear the in-process cache. + """ + for path in list(_NIXPKGS_WORKTREE_CACHE.values()): + _remove_nixpkgs_worktree(path) + _NIXPKGS_WORKTREE_CACHE.clear() + + +atexit.register(cleanup_nixpkgs_worktrees) + + +def _nixpkgs_mount_args(worktree_path): + """ + Return the `-v` arguments that mount a worktree into the + container read-only, or `[]` if no worktree is available. + """ + if not worktree_path: + return [] + return ["-v", f"{worktree_path}:{_NIXPKGS_CONTAINER_PATH}:ro"] + + +def _nixpkgs_import_expr(commit_hash, worktree_path, system_config, config_str): + """ + Build the Nix expression that imports nixpkgs at `commit_hash`. + + With a worktree mounted at `_NIXPKGS_CONTAINER_PATH`, the expression + imports from there (no network). Otherwise it falls back to + `fetchTarball` from GitHub, which might subject to rate-limiting. + """ + config = f"{{ {system_config} {config_str} }}" + if worktree_path: + return f"import {_NIXPKGS_CONTAINER_PATH} {config}" + return ( + f'import (fetchTarball "https://github.com/NixOS/nixpkgs/archive/' + f'{commit_hash}.tar.gz") {config}' + ) + + +def _is_non_linux_system(system): + """Return True if the target system is not a Linux variant.""" + target_os = system.split("-")[-1] if "-" in system else system + return bool(target_os) and target_os != "linux" + + +def _system_barrier_message(system): + """ + Return the warning about cross-system evaluation, or "" + when the target system is Linux. + """ + if not _is_non_linux_system(system): + return "" + return ( + f"Target system '{system}' requires OS-specific SDKs that cannot be " + f"evaluated inside the Linux-based Nix Docker container. The source " + f"tree was evaluated in the container's native Linux environment, so " + f"it will contain Linux-specific patches instead of {system} patches. " + f"Impact on deployment-to-development mapping is expected to be " + f"minimal, but a small number of unmapped files may appear due to " + f"missing OS-specific structural patches." + ) + + +def check_input_and_return_purl(project): + """Validate the input and return a Nix PURL.""" + input_sources = project.inputsources.all() + if len(input_sources) != 1: + error_msg = "Only 1 nix purl is accepted." + raise ValueError(error_msg) + + project_input = str(input_sources[0]) + input_purl = PackageURL.from_string(project_input) + if input_purl.type != "nix": + error_msg = "Only nix purl is supported." + raise ValueError(error_msg) + + namespace = input_purl.namespace + if not namespace or namespace.lower() != "nixpkgs": + raise ValueError( + "Only official nixpkgs repository is supported (i.e. namespace=nixpkgs)." + ) + + qualifiers = input_purl.qualifiers or {} + if not input_purl.version and "commit" not in qualifiers: + raise ValueError("Version or a 'commit' qualifier is required.") + + if "system" not in qualifiers: + raise ValueError( + "The 'system' qualifier is required to resolve system-specific binaries." + ) + + return input_purl + + +def fetch_inputs(purl, output_dir): + """ + Fetch the system specific binary and the exact source tree with the + patches and configurations applied for the given input purl. Return a + tuple of (source_path, binary_path, output_format, error_messages, + warning_messages). + """ + data = get_package_data(purl) + name = purl.name + version = purl.version + + commit_hash = purl.qualifiers.get("commit", "") + system = purl.qualifiers.get("system", "") + user_output = purl.qualifiers.get("output", "") + error_messages = [] + warning_messages = [] + + barrier_warning = _system_barrier_message(system) + if barrier_warning: + logger.warning(barrier_warning) + warning_messages.append(barrier_warning) + + output_format, path, release_commit_hash = get_nix_store_path( + data, name, version, system, commit_hash, user_output + ) + + concluded_commit_hash = release_commit_hash or commit_hash + + bin_path = "" + nix_bin_download_url = get_nix_download_url(path) if path else "" + if nix_bin_download_url: + bin_path = utils.fetch_path(nix_bin_download_url) + + if bin_path: + logger.info(f"Downloaded binary for {purl} to {bin_path}") + else: + if concluded_commit_hash: + logger.info( + f"Binary not found in cache for {purl}. Attempting local Nix build..." + ) + bin_path, error_message = build_binary_with_docker( + name, output_dir, system, concluded_commit_hash, output_format + ) + if error_message: + error_messages.append(error_message) + if bin_path: + logger.info(f"Successfully built binary for {purl} to {bin_path}") + warning_message = ( + f"Binary not found in cache for {purl}. Built locally using " + f"commit {concluded_commit_hash} with a Linux-based Nix " + f"Docker container." + ) + logger.warning(warning_message) + warning_messages.append(warning_message) + else: + error_message = f"Failed to fetch or build the binary for {purl}" + logger.error(error_message) + error_messages.append(error_message) + + patched_source_path = "" + if concluded_commit_hash: + source_result = get_patched_source_with_docker( + name, output_dir, system, concluded_commit_hash + ) + patched_source_path = source_result.path + + if not source_result.path: + detail = ( + f" Reason: {source_result.failure_detail}" + if source_result.failure_detail + else "" + ) + source_error = ( + f"Failed to fetch the patched source for {purl} " + f"(commit={concluded_commit_hash}, system={system}).{detail} " + f"D2D scan will be disabled." + ) + logger.error(source_error) + error_messages.append(source_error) + + if source_result.used_fallback: + detail = ( + f" Reason: {source_result.fallback_reason}." + if source_result.fallback_reason + else "" + ) + fallback_warning = ( + f"The patched source build for {name} failed; D2D will run " + f"against the raw upstream source (pkg.src) without nixpkgs " + f"patches.{detail} Mismatches between the source and binary " + f"trees may include files that were only added or modified by " + f"patches." + ) + logger.warning(fallback_warning) + warning_messages.append(fallback_warning) + + return ( + patched_source_path, + bin_path, + output_format, + error_messages, + warning_messages, + ) + + +def build_binary_with_docker(name, output_dir, system, commit_hash, output_format): + """ + Fetch a Nix package and build its binary from source using Docker. + Exports the resulting store path as a .nar file for standard extraction. + + Return a tuple of (path, error_msg), where `path` is the path to the + exported `.nar` file or an empty string on failure. + """ + nar_filename = f"{name}-bin.nar" + extracted_path = Path(output_dir) / nar_filename + absolute_out_dir = str(Path(output_dir).resolve()) + error_msg = "" + + # Handle architecture and system incompatibilities. + if _is_non_linux_system(system): + system_config = "" + else: + system_config = ( + f'localSystem = builtins.currentSystem; crossSystem = "{system}";' + ) + + config_str = ( + "config = { " + "allowBroken = true; " + "allowUnfree = true; " + "allowUnsupportedSystem = true; " + "};" + ) + + worktree_path = _get_nixpkgs_worktree(commit_hash) + nixpkgs_import = _nixpkgs_import_expr( + commit_hash, worktree_path, system_config, config_str + ) + + # Defaulting to 'debug' if none is specified. + effective_output = output_format or "debug" + + # Fall back to the default target if the effective_output is not + # defined in the recipe for this package. + nix_expression = ( + f"let " + f" pkgs = {nixpkgs_import}; " + f" target = pkgs.{name}; " + f" hasIt = builtins.isAttrs target && " + f'builtins.hasAttr "{effective_output}" target; ' + f"in if hasIt then target.{effective_output} else target" + ) + + # Build the Nix package, verify it succeeded, and export the output as + # a .nar file. + container_script = f""" + OUT_PATH=$(nix-build --no-out-link -E '{nix_expression}') + if [ -z "$OUT_PATH" ] || [ ! -e "$OUT_PATH" ]; then + echo "Error: nix-build failed to return a valid store path." >&2 + exit 1 + fi + nix-store --dump "$OUT_PATH" > /build_output/{nar_filename} + """ + + container_name = f"nix-bin-build-{uuid.uuid4().hex[:12]}" + nix_image = _ensure_nix_image() + cmd = [ + "docker", + "run", + "--rm", + "--init", + "--name", + container_name, + "-v", + "nix-eval-cache:/nix", + *_nixpkgs_mount_args(worktree_path), + "-v", + f"{absolute_out_dir}:/build_output", + nix_image, + "/bin/sh", + "-c", + container_script, + ] + + task_description = f"Building ({name} for {system})" + + try: + utils.run_docker_container( + cmd, container_name, _DOCKER_BUILD_TIMEOUT, task_description + ) + if extracted_path.exists(): + return str(extracted_path), "" + error_msg = f"Failed: {task_description} did not produce {nar_filename}" + logger.error(error_msg) + except subprocess.CalledProcessError as e: + summary = _summarize_nix_build_error(e.stderr) + error_msg = f"Failed: {task_description}: {summary}" + logger.error(error_msg) + except subprocess.TimeoutExpired: + error_msg = f"Failed: {task_description} with error: Process timed out" + logger.error(error_msg) + return "", error_msg + + +def get_nix_store_path(data, name, version, system, commit_hash, user_output): + """Get the Nix store path and release commit hash.""" + outputs_to_try = [user_output] if user_output else ["debug", "out"] + path = "" + release_commit_hash = "" + output_format = "" + + for output in outputs_to_try: + if data: + release_commit_hash, path = get_commit_hash_nix_store_path( + data, system, output, version, commit_hash + ) + + if not data or not path: + if commit_hash: + path = get_nix_store_path_with_nix(name, system, output, commit_hash) + + if path: + output_format = output + break + + if not path: + if not commit_hash: + raise Exception( + "Please provide a 'commit' qualifier in the PURL " + "for Nix to determine the download URL or build it locally." + ) + output_format = user_output or "debug" + + return output_format, path, release_commit_hash + + +def get_commit_hash_nix_store_path(data, system, output, version, commit_hash=""): + """ + Find and return the commit_hash and store path (/nix/store/) + based on the qualifiers. + """ + releases = data.get("releases") or [] + releases = [r for r in releases if r.get("version") == version] + + for release in releases: + release_version = release.get("version", "") + if version and release_version != version: + continue + for platform in release.get("platforms", []): + release_commit_hash = platform.get("commit_hash", "") + if platform.get("system") != system: + continue + if commit_hash and release_commit_hash != commit_hash: + continue + for out in platform.get("outputs", []): + out_path = out.get("path") + if out.get("name") == output and out_path: + return release_commit_hash, out_path + return "", "" + + +def get_package_data(purl): + """Fetch package data from https://search.devbox.sh/.""" + api_url = f"https://search.devbox.sh/v2/pkg?name={purl.name}" + try: + return fetch_json_response(api_url) + except Exception as e: + logger.warning(f"Failed to fetch package data for {purl}: {e}") + return None + + +def get_nix_store_path_with_nix(name, system, output, commit_hash): + """Find and return the store path using `nix`.""" + system_config = f'system = "{system}";' if system else "" + config_str = "config = { allowBroken = true; allowUnfree = true; };" + + worktree_path = _get_nixpkgs_worktree(commit_hash) + nixpkgs_import = _nixpkgs_import_expr( + commit_hash, worktree_path, system_config, config_str + ) + + nix_expression = ( + "let " + f" pkgs = {nixpkgs_import}; " + f" target = pkgs.{name}; " + f' hasIt = builtins.isAttrs target && builtins.hasAttr "{output}" target; ' + f'in if hasIt then target.{output}.outPath else ""' + ) + + container_name = f"nix-eval-{uuid.uuid4().hex[:12]}" + nix_image = _ensure_nix_image() + cmd = [ + "docker", + "run", + "--rm", + "--init", + "--name", + container_name, + "-v", + "nix-eval-cache:/nix", + *_nixpkgs_mount_args(worktree_path), + nix_image, + "nix-instantiate", + "--eval", + "--raw", + "-E", + nix_expression, + ] + + task_description = f"Evaluating store path for {name} ({output})" + + try: + result = utils.run_docker_container( + cmd, container_name, _DOCKER_EVAL_TIMEOUT, task_description + ) + return result.stdout.strip() + except subprocess.CalledProcessError as e: + logger.error(f"Error evaluating attribute for package '{name}': {e.stderr}") + except subprocess.TimeoutExpired: + logger.error(f"Timeout evaluating attribute for package '{name}'") + return "" + + +def get_nix_download_url(path): + """Construct a download URL from cache.nixos.org based on store path.""" + base_name = path.rstrip("/").split("/")[-1] + narinfo_hash = base_name.split("-")[0] + + narinfo_url = f"https://cache.nixos.org/{narinfo_hash}.narinfo" + url_path = get_narinfo_url(narinfo_url) + + if not url_path: + logger.warning(f"{narinfo_url} is not accessible.") + return "" + + return f"https://cache.nixos.org/{url_path}" + + +def get_narinfo_url(narinfo_url, retries=3, timeout=10): + """ + Visit the narinfo url and return the URL value, retrying on transient + failures. + + "cache.nixos.org" and its CDN occasionally return 429, 503, or a + network error for a single request. A failed lookup forces the caller + into a local Nix build, which is much slower and, for non-Linux + targets, may fail on a system barrier. Retrying with backoff recovers + from these transient failures at low cost. + """ + last_error = "" + for attempt in range(retries): + try: + response = requests.get(narinfo_url, timeout=timeout) + except requests.exceptions.RequestException as e: + last_error = str(e) + if attempt < retries - 1: + time.sleep(2**attempt) + continue + logger.debug(f"{narinfo_url}: {last_error} after {retries} attempts") + return "" + + if response.status_code == 200: + for line in response.text.splitlines(): + if line.startswith("URL:"): + return line.split(":", 1)[1].strip() + return "" + + # Retrying will not help with these permanent errors + if response.status_code in (400, 403, 404): + logger.debug(f"{narinfo_url}: HTTP {response.status_code}, not retrying") + return "" + + # Retry with backoff + last_error = f"HTTP {response.status_code}" + if attempt < retries - 1: + retry_after = response.headers.get("Retry-After") + if retry_after and retry_after.isdigit(): + delay = min(int(retry_after), 30) + else: + delay = 2**attempt + time.sleep(delay) + continue + + logger.debug(f"{narinfo_url}: {last_error} after {retries} attempts") + return "" + + +def _stage_archive(archive_path, output_dir): + """ + Ensure the archive lives inside output_dir so it is visible to the + Docker daemon that resolves the `-v` mount source. + + Return (staged_path, staged). `staged` is True when a new copy was + created inside `output_dir`, False when the archive was already there. + """ + target = output_dir / archive_path.name + staged = False + if archive_path == target: + return target, staged + + is_present = ( + target.exists() and target.stat().st_size == archive_path.stat().st_size + ) + if not is_present: + shutil.copy2(archive_path, target) + staged = True + return target, staged + + +def _decompress_pipeline_for(archive_name): + """ + Return the shell pipeline that decompresses `archive_name` to stdout. + + The custom Nix image has zstd, xz, bzip2, and gzip on `$PATH`, so + the pipeline is a direct call to the appropriate decompressor. No + `nix-shell -p ` is involved, so no fetch from + "cache.nixos.org" happens at extraction time. + """ + if archive_name.endswith(".zst"): + return f"zstdcat /input/{archive_name}" + if archive_name.endswith(".xz"): + return f"xzcat /input/{archive_name}" + if archive_name.endswith(".bz2"): + return f"bzcat /input/{archive_name}" + if archive_name.endswith(".gz"): + return f"zcat /input/{archive_name}" + return f"cat /input/{archive_name}" + + +def extract_nar_archive(archive_path, output_dir, output): + """Extract a compressed Nix NAR archive.""" + archive_path = Path(archive_path).resolve() + output_dir = Path(output_dir).resolve() + output_dir.mkdir(parents=True, exist_ok=True) + + # Docker mounts are resolved by the daemon, not the client. To make the + # archive visible to the daemon that runs the container, it must live + # in `output_dir` — the one path this project shares with that daemon. + archive_path, staged = _stage_archive(archive_path, output_dir) + + archive_dir = str(archive_path.parent) + archive_name = archive_path.name + extracted_path = output_dir / "to" / output + + decompress_cmd = _decompress_pipeline_for(archive_name) + restore_pipeline = f"{decompress_cmd} | nix-store --restore /output/to/{output}" + + # nix-store --restore runs as root inside the container and preserves the + # NAR's ownership metadata, so the extracted tree ends up owned by root. + # Chown it back to the calling user so ScanCode can extract nested archives, + # read the files, and clean up afterwards. + host_uid = os.getuid() + host_gid = os.getgid() + + container_script = ( + f"rm -rf /output/to/{output} " + f"&& mkdir -p /output/to " + f"&& {restore_pipeline} " + f"&& chown -R {host_uid}:{host_gid} /output/to " + f"&& chmod -R u+w /output/to" + ) + + container_name = f"nix-nar-extract-{uuid.uuid4().hex[:12]}" + nix_image = _ensure_nix_image() + cmd = [ + "docker", + "run", + "--rm", + "--init", + "--name", + container_name, + "-v", + f"{archive_dir}:/input:ro", + "-v", + f"{output_dir}:/output", + nix_image, + "/bin/sh", + "-c", + container_script, + ] + + task_description = f"Extracting {archive_name}" + + try: + utils.run_docker_container( + cmd, container_name, _DOCKER_IO_TIMEOUT, task_description + ) + return str(extracted_path) + except subprocess.CalledProcessError as e: + logger.error(f"Failed to extract {archive_name} with error: {e.stderr.strip()}") + except subprocess.TimeoutExpired: + logger.error(f"Failed to extract {archive_name}: Process timed out") + finally: + if staged: + try: + archive_path.unlink(missing_ok=True) + except OSError as e: + logger.debug(f"Could not remove staged archive {archive_path}: {e}") + return "" + + +def get_patched_source_with_docker(name, output_dir, system, commit_hash): + """ + Fetch a Nix package source and apply its official patches, falling back + to raw archives if package source cannot be built. + """ + extracted_path = Path(output_dir) / "from" + extracted_path.mkdir(parents=True, exist_ok=True) + absolute_out_dir = str(extracted_path.resolve()) + + host_uid = os.getuid() + host_gid = os.getgid() + + if _is_non_linux_system(system): + system_config = "" + else: + system_config = ( + f'localSystem = builtins.currentSystem; crossSystem = "{system}";' + ) + + config_str = ( + "config = { " + "allowBroken = true; " + "allowUnfree = true; " + "allowUnsupportedSystem = true; " + "};" + ) + + worktree_path = _get_nixpkgs_worktree(commit_hash) + nixpkgs_import = _nixpkgs_import_expr( + commit_hash, worktree_path, system_config, config_str + ) + + # `applyPatches` unpacks `pkg.src`, applies `pkg.patches`, and copies + # the resulting tree to `$out`. + nix_expression = f""" + let + pkgs = {nixpkgs_import}; + pkg = pkgs.{name}; + in + if !(pkg ? src) then pkg + else pkgs.applyPatches {{ + name = "{name}-patched-src"; + src = pkg.src; + patches = pkg.patches or []; + prePatch = pkg.prePatch or ""; + postPatch = pkg.postPatch or ""; + }}""" + + fallback_expression = f""" + let + pkgs = {nixpkgs_import}; + pkg = pkgs.{name}; + in + if pkg ? gemFile then pkg.gemFile + else if pkg ? src then pkg.src + else pkg + """ + + # This bash script must NOT be indented in Python. + # If EOF has spaces before it, bash will fail to parse it. + # The following script first attempts a standard patched build; if that + # fails (or yields no files), it falls back to fetching the raw source + # archive. The result is copied to the mounted `from/` directory with + # correct ownership so the host can extract and process it. + container_script = f""" +set -e + +cat << 'EOF' > /tmp/expr.nix +{nix_expression} +EOF + +cat << 'EOF' > /tmp/fallback.nix +{fallback_expression} +EOF + +# Try standard patched build +OUT_PATH=$(nix-build --no-out-link /tmp/expr.nix || true) + +VALID_FILES=0 +if [ -n "$OUT_PATH" ] && [ -d "$OUT_PATH" ]; then + VALID_FILES=$(ls -A1 "$OUT_PATH" 2>/dev/null | grep -v "^env-vars$" | wc -l) +fi + +if [ -z "$OUT_PATH" ]; then + FALLBACK_REASON="primary nix-build returned no store path" +elif [ ! -d "$OUT_PATH" ]; then + FALLBACK_REASON="primary store path is not a directory" +elif [ "$VALID_FILES" -eq 0 ]; then + FALLBACK_REASON="primary output contained only env-vars" +fi + +if [ -n "$FALLBACK_REASON" ]; then + echo "PATCHED_SOURCE_FALLBACK_REASON=$FALLBACK_REASON" >&2 + OUT_PATH=$(nix-build --no-out-link /tmp/fallback.nix || true) +fi + +if [ -z "$OUT_PATH" ] || [ ! -e "$OUT_PATH" ]; then + echo "Error: nix-build failed to return a valid store path." >&2 + rm -f /tmp/expr.nix /tmp/fallback.nix + exit 1 +fi + +if [ -d "$OUT_PATH" ]; then + cp -a "$OUT_PATH/." /build_output/ +else + cp -L "$OUT_PATH" /build_output/ +fi + +chown -R $HOST_UID:$HOST_GID /build_output/ +chmod -R u+w /build_output/ + +rm -f /tmp/expr.nix /tmp/fallback.nix +""" + + container_name = f"nix-patched-src-{uuid.uuid4().hex[:12]}" + nix_image = _ensure_nix_image() + cmd = [ + "docker", + "run", + "--rm", + "--init", + "--name", + container_name, + "-e", + f"HOST_UID={host_uid}", + "-e", + f"HOST_GID={host_gid}", + "-v", + "nix-eval-cache:/nix", + *_nixpkgs_mount_args(worktree_path), + "-v", + f"{absolute_out_dir}:/build_output", + nix_image, + "/bin/sh", + "-c", + container_script, + ] + + task_description = f"Fetching patched source for {name} ({system})" + + failure_detail = "" + try: + result = utils.run_docker_container( + cmd, container_name, _DOCKER_BUILD_TIMEOUT, task_description + ) + if any(extracted_path.iterdir()): + used_fallback = False + fallback_reason = "" + for line in result.stderr.splitlines(): + if line.startswith(FALLBACK_REASON_PREFIX): + used_fallback = True + fallback_reason = line[len(FALLBACK_REASON_PREFIX) :].strip() + break + if used_fallback: + logger.warning( + f"Primary patched-source build failed for {name}: {fallback_reason}" + ) + return PatchedSourceResult( + path=str(extracted_path), + used_fallback=used_fallback, + fallback_reason=fallback_reason, + failure_detail="", + ) + failure_detail = "Container produced no output" + logger.warning( + f"Patched-source extraction produced no files for {name} " + f"(system={system}, commit={commit_hash})." + ) + except subprocess.CalledProcessError as e: + failure_detail = _summarize_nix_build_error(e.stderr) or "docker run failed" + logger.error(f"Failed: {failure_detail}") + except subprocess.TimeoutExpired: + failure_detail = "Container timed out" + logger.error(failure_detail) + + shutil.rmtree(extracted_path, ignore_errors=True) + return PatchedSourceResult( + path="", + used_fallback=False, + fallback_reason="", + failure_detail=failure_detail, + ) + + +def ensure_multiarch_emulation(): + """ + Configure Docker host with binfmt emulators to support + multi-architecture execution and builds. + """ + cmd = [ + "docker", + "run", + "--privileged", + "--rm", + "--init", + "tonistiigi/binfmt", + "--install", + "all", + ] + try: + subprocess.run(cmd, capture_output=True, text=True, check=True, timeout=60) # noqa: S603 + return True + except subprocess.CalledProcessError as e: + logger.warning(f"Could not install binfmt multi-arch emulators: {e.stderr}") + return False + except subprocess.TimeoutExpired: + logger.warning("Timeout trying to setup binfmt emulators. Skipping.") + return False diff --git a/scanpipe/pipes/utils.py b/scanpipe/pipes/utils.py new file mode 100644 index 0000000000..e88e6c5bf2 --- /dev/null +++ b/scanpipe/pipes/utils.py @@ -0,0 +1,401 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# http://nexb.com and https://github.com/aboutcode-org/scancode.io +# The ScanCode.io software is licensed under the Apache License version 2.0. +# Data generated with ScanCode.io is provided as-is without warranties. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode.io should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# +# ScanCode.io is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/aboutcode-org/scancode.io for support and download. + +import fcntl +import logging +import shutil +import subprocess +import time +import uuid +from contextlib import contextmanager +from fnmatch import fnmatch + +import requests +from license_expression import Licensing + +from scanpipe.pipes import fetch +from scanpipe.pipes import flag + +logger = logging.getLogger(__name__) + +_GENERIC_FAILURE = 1 + + +def validate_package_license_integrity(project): + """Validate the correctness of the package license.""" + # Patterns to ignore certain resources during license validation + ignore_patterns = [ + "*test*", + "*.sh", + ] + + for package in project.discoveredpackages.all(): + package_lic = package.get_declared_license_expression() + if package_lic: + if package.type == "cargo": + # A single cargo package only has one Cargo.toml file + # meaning only one package is defined. Therefore, we don't + # need to check for the package_uid + # In addition, the package_uid is not populated to source files: + # https://github.com/aboutcode-org/scancode.io/issues/2169 + # so we set package_uid to None to consider all resources + # in the codebase for license validation. + package_uid = None + else: + package_uid = package.package_uid + resources = project.codebaseresources.has_license_expression() + detected_lic_list = collect_detected_licenses( + resources, ignore_patterns, package_uid + ) + + if detected_lic_list: + lic_exp = " AND ".join(detected_lic_list) + detected_lic_exp = str(Licensing().dedup(lic_exp)) + + if detected_lic_exp != package_lic: + package_issues = package.extra_data.get("issues", []) + + package_issues.append( + { + "issue_type": "License Mismatch", + "declared_license": package_lic, + "detected_codebase_license": detected_lic_exp, + } + ) + + package.update_extra_data({"issues": package_issues}) + + for datafile_path in package.datafile_paths: + if not datafile_path.startswith("https://"): + data_path = project.codebaseresources.get( + path=datafile_path + ) + data_path.update(status=flag.LICENSE_ISSUE) + + resource_issues = data_path.extra_data.get("issues", []) + resource_issues.append( + { + "issue_type": "License Mismatch", + "declared_license": package_lic, + "detected_codebase_license": detected_lic_exp, + } + ) + + data_path.update_extra_data({"issues": resource_issues}) + + +def contains_ignore_pattern(resource_path, ignore_patterns): + """Check if the resource path matches any of the ignore patterns.""" + for pattern in ignore_patterns: + if fnmatch(resource_path, pattern): + return True + return False + + +def filter_ignored_licenses(license_expression, licensing): + """Filter out ignored licenses from a license expression.""" + # Some licenses are not useful for validating package license + # integrity, so we ignore them. + ignored_licenses = [ + "free-unknown", + "unknown", + "unknown-license-reference", + "unknown-spdx", + ] + + if license_expression is None: + return None + + if isinstance(license_expression, licensing.Symbol): + if ( + hasattr(license_expression, "key") + and license_expression.key in ignored_licenses + ): + return None + return license_expression + + # Handle AND operations + if isinstance(license_expression, licensing.AND): + return handle_operator_expression(license_expression, licensing, licensing.AND) + + # Handle OR operations + if isinstance(license_expression, licensing.OR): + return handle_operator_expression(license_expression, licensing, licensing.OR) + + return license_expression + + +def handle_operator_expression(expression, licensing, operator): + """ + Process AND/OR operations in a license expression, filtering out + ignored licenses. + """ + args = [] + for arg in expression.args: + filtered_arg = filter_ignored_licenses(arg, licensing) + if filtered_arg is not None: + args.append(filtered_arg) + if not args: + return None + if len(args) == 1: + return args[0] + + return operator(*args) + + +def collect_detected_licenses(resources, ignore_patterns, package_uid=None): + """Collect detected licenses from resources, ignoring defined patterns.""" + licensing = Licensing() + detected_lic_list = [] + + for resource in resources: + if contains_ignore_pattern(resource.path, ignore_patterns): + continue + + # If a package_uid is provided, only consider resources linked to it + if package_uid and package_uid not in resource.for_packages: + continue + + license_str = resource.detected_license_expression + if not license_str: + continue + try: + parsed_lic = licensing.parse(license_str) + + # Filter out the ignored keys + filtered_license = filter_ignored_licenses(parsed_lic, licensing) + + if filtered_license is not None: + final_lic = str(filtered_license) + + if final_lic not in detected_lic_list: + # Apply parentheses so that the 'OR' expression will + # not be filtered out when doing deduplication later. + detected_lic_list.append(f"({final_lic})") + + except Exception: + logger.warning( + "Failed to parse the license expression: %s at %s", + license_str, + resource.path, + ) + return detected_lic_list + + +def fetch_path(purl): + """Fetch the purl and return the location of the fetched tarball""" + try: + return fetch.fetch_url(url=purl).path + except (ValueError, requests.RequestException) as e: + logger.warning("Failed to fetch package: %s - %s", purl, e) + return None + + +def check_docker_command(): + """Check if the Docker command is available and the daemon is running.""" + docker_path = shutil.which("docker") + if not docker_path: + return False + + try: + subprocess.run([docker_path, "info"], capture_output=True, check=True) # noqa: S603 + return True + except (subprocess.SubprocessError, FileNotFoundError): + return False + + +@contextmanager +def file_lock(lock_path): + """ + Exclusive cross-process lock backed by `fcntl.flock`. + + Blocks until the lock is free. Released when the `with` block exits or + the process dies. + """ + lock_path.parent.mkdir(parents=True, exist_ok=True) + # "a" mode avoids truncating the lock file on every acquisition. + with open(lock_path, "a") as lock_file: + fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX) + try: + yield + finally: + fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN) + + +def docker_image_exists(image): + """Return True if the "image" is present in the local Docker image store.""" + cmd = ["docker", "image", "inspect", image] + result = subprocess.run(cmd, capture_output=True, check=False) # noqa: S603 + return result.returncode == 0 + + +def docker_rm_force(container_name): + """Remove a container left behind after a timeout.""" + cmd = ["docker", "rm", "-f", container_name] + subprocess.run( # noqa: S603 + cmd, + capture_output=True, + check=False, + ) + + +def container_state(container_name): + """Return the container's state string, e.g. "created", or "" if unknown.""" + cmd = ["docker", "inspect", "--format", "{{.State.Status}}", container_name] + try: + result = subprocess.run( # noqa: S603 + cmd, + capture_output=True, + text=True, + timeout=5, + check=True, + ) + return result.stdout.strip() + except ( + subprocess.CalledProcessError, + subprocess.TimeoutExpired, + FileNotFoundError, + ): + return "" + + +def rename_container(cmd, new_name): + """Return a copy of a `docker run` command with its `--name` value replaced.""" + cmd = list(cmd) + try: + idx = cmd.index("--name") + except ValueError: + return cmd + if idx + 1 < len(cmd): + cmd[idx + 1] = new_name + return cmd + + +def is_startup_race(returncode, stderr): + """ + Return True if `docker run` failed with a startup race unique to + Docker Desktop. + + A startup race is when two things happen at once and the wrong one + wins. Docker Desktop exposes host directories to its container VM + through a bridge that can lag a few milliseconds behind the host, so + the daemon sometimes tries to use a path or rootfs before it has + propagated. The failure is transient; a retry usually succeeds. + + Two scenarios: + + - Exit code 125 with `error mounting`: the bind-mount source was + not visible yet. + - `[FATAL tini`: the container started before the rootfs was fully + mounted, so `/bin/sh` could not be resolved. + + Native Linux has no bridge, so neither occurs there. + """ + if not stderr: + return False + if returncode == 125 and "error mounting" in stderr: + return True + if "[FATAL tini" in stderr: + return True + return False + + +def run_docker_container(cmd, container_name, timeout, task_description, retries=1): + """ + Run `docker run` with a timeout and grace window, retrying automatically + if a transient Docker Desktop startup race occurs or the daemon wedges. + + Return a CompletedProcess on success. Raise CalledProcessError on a + real failure, or TimeoutExpired on a genuine timeout. + """ + grace = 15 + + for attempt in range(retries + 1): + if attempt: + logger.warning(f"{task_description}: retrying container in 5s") + time.sleep(5) + prefix = container_name.rsplit("-", 1)[0] + container_name = f"{prefix}-{uuid.uuid4().hex[:12]}" + cmd = rename_container(cmd, container_name) + + # Popen (not run) so we can inspect the daemon state after the grace + # window, while the container may still be running. + proc = subprocess.Popen( # noqa: S603 + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + + try: + # Wait up to `grace` seconds for docker run to finish, + # capturing its stdout and stderr. The timeout is what lets us + # detect a wedged daemon instead of blocking forever. + stdout, stderr = proc.communicate(timeout=grace) + except subprocess.TimeoutExpired: + # Still running past the grace window. If the daemon never + # actually started it, the state will be 'created' and no + # amount of waiting will help. + if container_state(container_name) == "created": + proc.kill() + proc.communicate() + docker_rm_force(container_name) + if attempt >= retries: + raise subprocess.CalledProcessError( + _GENERIC_FAILURE, + cmd, + stderr=( + f"container {container_name} stuck in " + f"'Created' state after {grace}s" + ), + ) + continue + + # The container is actually running. Wait out the rest of + # the timeout. + try: + stdout, stderr = proc.communicate(timeout=max(1, timeout - grace)) + except subprocess.TimeoutExpired: + proc.kill() + proc.communicate() + state = container_state(container_name) + logger.error( + f"{task_description}: timeout after {timeout}s; " + f"container state={state!r}" + ) + docker_rm_force(container_name) + raise + + if proc.returncode == 0: + return subprocess.CompletedProcess(cmd, 0, stdout, stderr) + + if attempt < retries and is_startup_race(proc.returncode, stderr): + logger.warning( + f"{task_description}: startup race for {container_name}; retrying in 5s" + ) + docker_rm_force(container_name) + continue + + docker_rm_force(container_name) + raise subprocess.CalledProcessError( + proc.returncode, cmd, output=stdout, stderr=stderr + ) diff --git a/scanpipe/templates/scanpipe/package_list.html b/scanpipe/templates/scanpipe/package_list.html index 90f917c245..206a66526e 100644 --- a/scanpipe/templates/scanpipe/package_list.html +++ b/scanpipe/templates/scanpipe/package_list.html @@ -34,6 +34,11 @@ {% endif %} + {% if package.extra_data.issues %} + + + + {% endif %} @@ -75,4 +80,4 @@ {% include 'scanpipe/includes/pagination.html' with page_obj=page_obj %} {% endif %} -{% endblock %} \ No newline at end of file +{% endblock %} diff --git a/scanpipe/tests/pipes/test_nix.py b/scanpipe/tests/pipes/test_nix.py new file mode 100644 index 0000000000..5ec2099039 --- /dev/null +++ b/scanpipe/tests/pipes/test_nix.py @@ -0,0 +1,525 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# http://nexb.com and https://github.com/nexB/scancode.io +# The ScanCode.io software is licensed under the Apache License version 2.0. +# Data generated with ScanCode.io is provided as-is without warranties. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode.io should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# +# ScanCode.io is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/nexB/scancode.io for support and download. + +import tempfile +from pathlib import Path +from unittest import mock + +from django.test import TestCase + +from packageurl import PackageURL + +from scanpipe.pipes import nix + + +class ScanPipeNixPipesTest(TestCase): + data = Path(__file__).parent.parent / "data" + + def test_scanpipe_nix_check_input_and_return_purl(self): + project = mock.Mock() + project.inputsources.all.return_value = [ + "pkg:nix/nixpkgs/hello@2.12.1?system=x86_64-linux&commit=1234abcd" + ] + + expected = PackageURL.from_string( + "pkg:nix/nixpkgs/hello@2.12.1?system=x86_64-linux&commit=1234abcd" + ) + result = nix.check_input_and_return_purl(project) + self.assertEqual(result, expected) + + def test_scanpipe_nix_check_input_and_return_purl_no_input(self): + project = mock.Mock() + project.inputsources.all.return_value = [] + with self.assertRaisesMessage(ValueError, "Only 1 nix purl is accepted."): + nix.check_input_and_return_purl(project) + + def test_scanpipe_nix_check_input_and_return_purl_multi_input(self): + project = mock.Mock() + project.inputsources.all.return_value = [ + "pkg:nix/nixpkgs/hello@2.12.1?system=x86_64-linux", + "pkg:nix/nixpkgs/world@2.40.0?system=x86_64-linux", + ] + with self.assertRaisesMessage(ValueError, "Only 1 nix purl is accepted."): + nix.check_input_and_return_purl(project) + + def test_scanpipe_nix_check_input_and_return_purl_non_supported_type(self): + project = mock.Mock() + project.inputsources.all.return_value = ["pkg:npm/test@1.0"] + with self.assertRaisesMessage(ValueError, "Only nix purl is supported."): + nix.check_input_and_return_purl(project) + + def test_scanpipe_nix_check_input_and_return_purl_invalid_namespace(self): + project = mock.Mock() + project.inputsources.all.return_value = [ + "pkg:nix/namespace/hello@2.12.1?system=x86_64-linux" + ] + with self.assertRaisesMessage( + ValueError, "Only official nixpkgs repository is supported" + ): + nix.check_input_and_return_purl(project) + + def test_scanpipe_nix_check_input_and_return_purl_missing_version_and_commit(self): + project = mock.Mock() + project.inputsources.all.return_value = [ + "pkg:nix/nixpkgs/hello?system=x86_64-linux" + ] + with self.assertRaisesMessage( + ValueError, "Version or a 'commit' qualifier is required." + ): + nix.check_input_and_return_purl(project) + + def test_scanpipe_nix_check_input_and_return_purl_missing_system(self): + project = mock.Mock() + project.inputsources.all.return_value = ["pkg:nix/nixpkgs/hello@2.12.1"] + with self.assertRaisesMessage( + ValueError, + "The 'system' qualifier is required to resolve system-specific binaries.", + ): + nix.check_input_and_return_purl(project) + + @mock.patch("scanpipe.pipes.nix.fetch_json_response") + def test_scanpipe_nix_get_package_data(self, mock_fetch_json): + mock_fetch_json.return_value = { + "releases": [ + { + "version": "2.12.1", + "platforms": [ + { + "arch": "x86-64", + "os": "Linux", + "system": "x86_64-linux", + "commit_hash": "1234abcd", + "outputs": [ + { + "name": "out", + "path": "/nix/store/aaaaaaa-hello-2.12.1", + } + ], + } + ], + "platforms_summary": "Linux only", + "outputs_summary": "out", + } + ] + } + purl = PackageURL.from_string( + "pkg:nix/nixpkgs/hello@2.12.1?system=x86_64-linux" + ) + + result = nix.get_package_data(purl) + self.assertEqual( + result, + { + "releases": [ + { + "version": "2.12.1", + "platforms": [ + { + "arch": "x86-64", + "os": "Linux", + "system": "x86_64-linux", + "commit_hash": "1234abcd", + "outputs": [ + { + "name": "out", + "path": "/nix/store/aaaaaaa-hello-2.12.1", + } + ], + } + ], + "platforms_summary": "Linux only", + "outputs_summary": "out", + } + ] + }, + ) + mock_fetch_json.assert_called_once_with( + "https://search.devbox.sh/v2/pkg?name=hello" + ) + + def test_scanpipe_nix_get_commit_hash_nix_store_path(self): + data = { + "releases": [ + { + "version": "2.12.1", + "platforms": [ + { + "system": "x86_64-linux", + "commit_hash": "1234abcd", + "outputs": [ + { + "name": "out", + "path": "/nix/store/aaaaaaa-hello-2.12.1", + }, + { + "name": "debug", + "path": "/nix/store/aaaaaaa-hello-2.12.1-debug", + }, + ], + } + ], + } + ] + } + + commit, store_path = nix.get_commit_hash_nix_store_path( + data, "x86_64-linux", "out", "2.12.1", "1234abcd" + ) + self.assertEqual(commit, "1234abcd") + self.assertEqual(store_path, "/nix/store/aaaaaaa-hello-2.12.1") + + @mock.patch("scanpipe.pipes.nix._ensure_nix_image") + @mock.patch("scanpipe.pipes.nix._get_nixpkgs_worktree") + @mock.patch("scanpipe.pipes.utils.run_docker_container") + def test_scanpipe_nix_get_nix_store_path_with_nix( + self, mock_run_container, mock_get_worktree, mock_ensure_image + ): + mock_ensure_image.return_value = "nixos/nix:test" + mock_get_worktree.return_value = "" + + mock_result = mock.Mock() + mock_result.stdout = "/nix/store/evaluated-path-out" + mock_run_container.return_value = mock_result + + path = nix.get_nix_store_path_with_nix( + "hello", "x86_64-linux", "out", "1234abcd" + ) + self.assertEqual(path, "/nix/store/evaluated-path-out") + + @mock.patch("scanpipe.pipes.nix.get_narinfo_url") + def test_scanpipe_nix_get_nix_download_url(self, mock_get_narinfo): + mock_get_narinfo.return_value = "nar/abc.nar.xz" + store_path = "/nix/store/aaaaaaaaaaaaa-hello-2.12.1" + + url = nix.get_nix_download_url(store_path) + self.assertEqual(url, "https://cache.nixos.org/nar/abc.nar.xz") + + @mock.patch("scanpipe.pipes.nix.requests.get") + def test_scanpipe_nix_get_narinfo_url(self, mock_requests_get): + mock_response = mock.Mock() + mock_response.status_code = 200 + mock_response.text = "StorePath: /nix/store/xyz\nURL: nar/123.nar.xz" + mock_requests_get.return_value = mock_response + + url_path = nix.get_narinfo_url("https://cache.nixos.org/aaaaaaaaaaa.narinfo") + self.assertEqual(url_path, "nar/123.nar.xz") + + @mock.patch("scanpipe.pipes.nix.get_package_data") + @mock.patch("scanpipe.pipes.nix.get_nix_store_path_with_nix") + @mock.patch("scanpipe.pipes.nix.get_nix_download_url") + @mock.patch("scanpipe.pipes.nix.get_patched_source_with_docker") + @mock.patch("scanpipe.pipes.utils.fetch_path") + def test_scanpipe_nix_fetch_inputs( + self, + mock_fetch_path, + mock_get_patched_source, + mock_get_download_url, + mock_get_store_path_with_nix, + mock_get_package_data, + ): + mock_get_package_data.return_value = None + mock_get_store_path_with_nix.return_value = "/nix/store/aaaaaaaaaa" + + mock_get_download_url.return_value = "https://cache.nixos.org/nar/hello.nar.xz" + mock_get_patched_source.return_value = nix.PatchedSourceResult( + path="/path/extracted/from", + used_fallback=False, + fallback_reason="", + failure_detail="", + ) + mock_fetch_path.return_value = "/path/debug/to" + + purl = PackageURL.from_string( + "pkg:nix/nixpkgs/hello@2.12.1?system=x86_64-linux&commit=1234abcd" + ) + + with tempfile.TemporaryDirectory() as temp_dir: + src_path, bin_path, output_fmt, error_msgs, warning_msgs = nix.fetch_inputs( + purl, temp_dir + ) + + self.assertEqual(src_path, "/path/extracted/from") + self.assertEqual(bin_path, "/path/debug/to") + self.assertEqual(output_fmt, "debug") + self.assertEqual(error_msgs, []) + self.assertEqual(warning_msgs, []) + + mock_get_store_path_with_nix.assert_called_once() + + @mock.patch("scanpipe.pipes.nix.get_package_data") + @mock.patch("scanpipe.pipes.nix.get_nix_store_path_with_nix") + @mock.patch("scanpipe.pipes.nix.get_nix_download_url") + @mock.patch("scanpipe.pipes.nix.get_patched_source_with_docker") + @mock.patch("scanpipe.pipes.nix.build_binary_with_docker") + @mock.patch("scanpipe.pipes.utils.fetch_path") + def test_scanpipe_nix_fetch_inputs_fallback_build( + self, + mock_fetch_path, + mock_build_binary, + mock_get_patched_source, + mock_get_download_url, + mock_get_store_path_with_nix, + mock_get_package_data, + ): + """Test that fetch_inputs falls back to local build if download fails.""" + mock_get_package_data.return_value = None + mock_get_store_path_with_nix.return_value = "/nix/store/aaaaaaaaaa" + + # Simulate a missing/failed cache download + mock_get_download_url.return_value = "" + mock_fetch_path.return_value = "" + + # Simulate a successful local build and source extraction + mock_build_binary.return_value = ("/path/built/locally/to", "") + mock_get_patched_source.return_value = nix.PatchedSourceResult( + path="/path/extracted/from", + used_fallback=False, + fallback_reason="", + failure_detail="", + ) + + purl = PackageURL.from_string( + "pkg:nix/nixpkgs/hello@2.12.1?system=x86_64-linux&commit=1234abcd" + ) + + with tempfile.TemporaryDirectory() as temp_dir: + src_path, bin_path, output_fmt, error_msgs, warning_msgs = nix.fetch_inputs( + purl, temp_dir + ) + + self.assertEqual(src_path, "/path/extracted/from") + self.assertEqual(bin_path, "/path/built/locally/to") + self.assertEqual(output_fmt, "debug") + self.assertEqual(error_msgs, []) + self.assertTrue( + any("Built locally using commit" in msg for msg in warning_msgs) + ) + + mock_build_binary.assert_called_once() + mock_get_store_path_with_nix.assert_called_once() + + @mock.patch("scanpipe.pipes.nix.get_commit_hash_nix_store_path") + def test_scanpipe_nix_get_nix_store_path_success( + self, mock_get_commit_hash_nix_store_path + ): + mock_get_commit_hash_nix_store_path.return_value = ( + "1234abcd", + "/nix/store/hello-path", + ) + + output_fmt, path, commit = nix.get_nix_store_path( + data={"releases": []}, + name="hello", + version="2.12.1", + system="x86_64-linux", + commit_hash="1234abcd", + user_output="", + ) + + self.assertEqual(output_fmt, "debug") + self.assertEqual(path, "/nix/store/hello-path") + self.assertEqual(commit, "1234abcd") + + @mock.patch("scanpipe.pipes.nix._ensure_nix_image") + @mock.patch("scanpipe.pipes.nix._get_nixpkgs_worktree") + @mock.patch("scanpipe.pipes.utils.run_docker_container") + def test_scanpipe_nix_get_patched_source_with_docker_success( + self, mock_run_container, mock_get_worktree, mock_ensure_image + ): + """Test successful fetching and patching of source using Docker.""" + mock_ensure_image.return_value = "nixos/nix:test" + mock_get_worktree.return_value = "" + mock_run_container.return_value = mock.Mock(stderr="", returncode=0) + + with tempfile.TemporaryDirectory() as temp_dir: + from_dir = Path(temp_dir) / "from" + from_dir.mkdir() + (from_dir / "somefile").touch() + + result = nix.get_patched_source_with_docker( + name="hello", + output_dir=temp_dir, + system="x86_64-linux", + commit_hash="1234abcd", + ) + + self.assertEqual(result.path, str(from_dir)) + self.assertFalse(result.used_fallback) + self.assertEqual(result.fallback_reason, "") + mock_run_container.assert_called_once() + + @mock.patch("scanpipe.pipes.nix._ensure_nix_image") + @mock.patch("scanpipe.pipes.nix._get_nixpkgs_worktree") + @mock.patch("scanpipe.pipes.utils.run_docker_container") + def test_scanpipe_nix_get_patched_source_with_docker_fallback( + self, mock_run_container, mock_get_worktree, mock_ensure_image + ): + """The fallback line on stderr flips used_fallback and carries the reason.""" + mock_ensure_image.return_value = "nixos/nix:test" + mock_get_worktree.return_value = "" + mock_run_container.return_value = mock.Mock( + stderr=( + "some nix output\n" + "PATCHED_SOURCE_FALLBACK_REASON=" + "primary output contained only env-vars\n" + ), + returncode=0, + ) + + with tempfile.TemporaryDirectory() as temp_dir: + from_dir = Path(temp_dir) / "from" + from_dir.mkdir() + (from_dir / "somefile").touch() + + result = nix.get_patched_source_with_docker( + name="hello", + output_dir=temp_dir, + system="x86_64-linux", + commit_hash="1234abcd", + ) + + self.assertEqual(result.path, str(from_dir)) + self.assertTrue(result.used_fallback) + self.assertEqual( + result.fallback_reason, "primary output contained only env-vars" + ) + + @mock.patch("scanpipe.pipes.nix._ensure_nix_image") + @mock.patch("scanpipe.pipes.utils.run_docker_container") + def test_scanpipe_nix_extract_nar_archive_success( + self, mock_run_container, mock_ensure_image + ): + """Test extracting a .nar archive via Docker.""" + mock_ensure_image.return_value = "nixos/nix:test" + mock_run_container.return_value = mock.Mock(returncode=0) + + with tempfile.TemporaryDirectory() as temp_dir: + # We don't actually need the file to exist for the mocked test + archive_path = Path(temp_dir) / "hello-bin.nar.xz" + + result = nix.extract_nar_archive( + archive_path=str(archive_path), output_dir=temp_dir, output="debug" + ) + + expected_extracted_path = str(Path(temp_dir).resolve() / "to" / "debug") + self.assertEqual(result, expected_extracted_path) + + @mock.patch("scanpipe.pipes.nix._ensure_nix_image") + @mock.patch("scanpipe.pipes.nix.shutil.copy2") + @mock.patch("scanpipe.pipes.utils.run_docker_container") + def test_scanpipe_nix_extract_nar_archive_stages_from_tmp( + self, mock_run_container, mock_copy2, mock_ensure_image + ): + """Archive outside output_dir is staged into it before docker run.""" + mock_ensure_image.return_value = "nixos/nix:test" + mock_run_container.return_value = mock.Mock(returncode=0) + + with ( + tempfile.TemporaryDirectory() as source_dir, + tempfile.TemporaryDirectory() as output_dir, + ): + archive_path = Path(source_dir) / "hello-bin.nar.xz" + + result = nix.extract_nar_archive( + archive_path=str(archive_path), output_dir=output_dir, output="debug" + ) + + expected_extracted_path = str(Path(output_dir).resolve() / "to" / "debug") + self.assertEqual(result, expected_extracted_path) + + # Staging must have happened exactly once + mock_copy2.assert_called_once() + src, dst = mock_copy2.call_args[0] + self.assertEqual(Path(src), Path(source_dir).resolve() / "hello-bin.nar.xz") + self.assertEqual(Path(dst), Path(output_dir).resolve() / "hello-bin.nar.xz") + + # The docker mount source must be output_dir, not the /tmp source + cmd = mock_run_container.call_args[0][0] + volume_mounts = [cmd[i + 1] for i, a in enumerate(cmd) if a == "-v"] + self.assertTrue( + any(str(Path(output_dir).resolve()) in v for v in volume_mounts), + f"expected staged mount in {volume_mounts}", + ) + self.assertFalse( + any(str(Path(source_dir).resolve()) in v for v in volume_mounts), + f"unexpected source mount in {volume_mounts}", + ) + + def test_scanpipe_nix_decompress_pipeline_for(self): + cases = [ + ("foo.nar.xz", "xzcat /input/foo.nar.xz"), + ("foo.nar.zst", "zstdcat /input/foo.nar.zst"), + ("foo.nar.bz2", "bzcat /input/foo.nar.bz2"), + ("foo.nar.gz", "zcat /input/foo.nar.gz"), + ("foo.nar", "cat /input/foo.nar"), + ] + for name, expected_cmd in cases: + cmd = nix._decompress_pipeline_for(name) + self.assertEqual(cmd, expected_cmd) + + def test_scanpipe_nix_stage_archive_already_in_output_dir(self): + with tempfile.TemporaryDirectory() as temp_dir: + output_dir = Path(temp_dir).resolve() + archive_path = output_dir / "hello-bin.nar.xz" + + with mock.patch("scanpipe.pipes.nix.shutil.copy2") as mock_copy2: + result, staged = nix._stage_archive(archive_path, output_dir) + + self.assertEqual(result, archive_path) + self.assertFalse(staged) + mock_copy2.assert_not_called() + + @mock.patch("scanpipe.pipes.nix.shutil.copy2") + def test_scanpipe_nix_stage_archive_from_elsewhere(self, mock_copy2): + with ( + tempfile.TemporaryDirectory() as source_dir, + tempfile.TemporaryDirectory() as output_dir, + ): + archive_path = Path(source_dir).resolve() / "hello-bin.nar.xz" + output_path = Path(output_dir).resolve() + + result, staged = nix._stage_archive(archive_path, output_path) + + self.assertEqual(result, output_path / "hello-bin.nar.xz") + self.assertTrue(staged) + mock_copy2.assert_called_once_with( + archive_path, output_path / "hello-bin.nar.xz" + ) + + @mock.patch("scanpipe.pipes.nix.shutil.copy2") + def test_scanpipe_nix_stage_archive_skips_copy_when_sizes_match(self, mock_copy2): + with ( + tempfile.TemporaryDirectory() as source_dir, + tempfile.TemporaryDirectory() as output_dir, + ): + archive_path = Path(source_dir).resolve() / "hello-bin.nar.xz" + archive_path.write_bytes(b"payload") + target = Path(output_dir).resolve() / "hello-bin.nar.xz" + target.write_bytes(b"payload") # same size + + result, staged = nix._stage_archive( + archive_path, Path(output_dir).resolve() + ) + + self.assertEqual(result, target) + self.assertFalse(staged) + mock_copy2.assert_not_called() diff --git a/scanpipe/tests/pipes/test_utils.py b/scanpipe/tests/pipes/test_utils.py new file mode 100644 index 0000000000..153f936e19 --- /dev/null +++ b/scanpipe/tests/pipes/test_utils.py @@ -0,0 +1,297 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# http://nexb.com and https://github.com/nexB/scancode.io +# The ScanCode.io software is licensed under the Apache License version 2.0. +# Data generated with ScanCode.io is provided as-is without warranties. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode.io should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# +# ScanCode.io is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/nexB/scancode.io for support and download. + +import subprocess +from unittest import mock + +from django.test import TestCase + +from license_expression import Licensing + +from scanpipe.pipes import flag +from scanpipe.pipes import utils + + +class ScanPipeUtilsTest(TestCase): + def setUp(self): + self.licensing = Licensing() + + @mock.patch("scanpipe.models.CodebaseResource") + @mock.patch("scanpipe.models.DiscoveredPackage") + @mock.patch("scanpipe.models.Project") + def test_validate_package_license_integrity_mismatch( + self, mock_project_class, mock_package_class, mock_resource_class + ): + mock_project = mock_project_class() + mock_package = mock_package_class() + + mock_package.type = "pypi" + mock_package.package_uid = "pkg:pypi/test@1.0" + mock_package.get_declared_license_expression.return_value = "mit" + mock_package.datafile_paths = ["src/main.py"] + mock_package.extra_data = {} + + mock_project.discoveredpackages.all.return_value = [mock_package] + + mock_resource = mock_resource_class() + mock_resource.path = "src/main.py" + mock_resource.for_packages = ["pkg:pypi/test@1.0"] + mock_resource.detected_license_expression = "gpl-3.0" + + mock_project.codebaseresources.has_license_expression.return_value = [ + mock_resource + ] + + mock_data_path = mock_resource_class() + mock_data_path.extra_data = {} + mock_project.codebaseresources.get.return_value = mock_data_path + + utils.validate_package_license_integrity(mock_project) + + package_update_args = mock_package.update_extra_data.call_args.args[0] + self.assertEqual( + package_update_args["issues"][0]["issue_type"], "License Mismatch" + ) + self.assertEqual( + package_update_args["issues"][0]["detected_codebase_license"], "gpl-3.0" + ) + + mock_data_path.update.assert_called_once_with(status=flag.LICENSE_ISSUE) + + def test_contains_ignore_pattern(self): + ignore_patterns = ["*test*", "*.sh"] + self.assertTrue( + utils.contains_ignore_pattern("src/test_main.py", ignore_patterns) + ) + self.assertTrue( + utils.contains_ignore_pattern("scripts/build.sh", ignore_patterns) + ) + self.assertFalse(utils.contains_ignore_pattern("src/main.py", ignore_patterns)) + + def test_filter_ignored_licenses(self): + exp1 = self.licensing.parse("mit") + self.assertEqual( + str(utils.filter_ignored_licenses(exp1, self.licensing)), "mit" + ) + + exp2 = self.licensing.parse("unknown") + self.assertIsNone(utils.filter_ignored_licenses(exp2, self.licensing)) + + exp3 = self.licensing.parse("mit AND unknown") + self.assertEqual( + str(utils.filter_ignored_licenses(exp3, self.licensing)), "mit" + ) + + exp4 = self.licensing.parse("unknown-spdx OR free-unknown") + self.assertIsNone(utils.filter_ignored_licenses(exp4, self.licensing)) + + def test_collect_detected_licenses(self): + mock_resource1 = mock.Mock() + mock_resource1.path = "src/main.py" + mock_resource1.for_packages = ["pkg:pypi/test@1.0"] + mock_resource1.detected_license_expression = "mit AND unknown" + + mock_resource2 = mock.Mock() + mock_resource2.path = "test/test_main.py" + mock_resource2.for_packages = ["pkg:pypi/test@1.0"] + mock_resource2.detected_license_expression = "gpl-3.0" + + mock_resource3 = mock.Mock() + mock_resource3.path = "src/other.py" + mock_resource3.for_packages = ["pkg:pypi/test@2.0"] + mock_resource3.detected_license_expression = "apache-2.0" + + resources = [mock_resource1, mock_resource2, mock_resource3] + ignore_patterns = ["*test*"] + + result = utils.collect_detected_licenses( + resources, ignore_patterns, package_uid="pkg:pypi/test@1.0" + ) + + self.assertEqual(result, ["(mit)"]) + + def test_handle_operator_expression_and(self): + expr = self.licensing.parse("mit AND apache-2.0") + result = utils.handle_operator_expression( + expr, self.licensing, self.licensing.AND + ) + self.assertEqual(str(result), "mit AND apache-2.0") + + def test_handle_operator_expression_or(self): + expr = self.licensing.parse("mit OR bsd-3-clause") + result = utils.handle_operator_expression( + expr, self.licensing, self.licensing.OR + ) + self.assertEqual(str(result), "mit OR bsd-3-clause") + + def test_handle_operator_expression_filters_to_single_arg(self): + # 'unknown' gets filtered out to None, leaving only 'mit' (len == 1) + expr = self.licensing.parse("mit AND unknown") + result = utils.handle_operator_expression( + expr, self.licensing, self.licensing.AND + ) + self.assertEqual(str(result), "mit") + + def test_handle_operator_expression_all_filtered_out(self): + # Both 'unknown' and 'free-unknown' get filtered out, leaving empty args + expr = self.licensing.parse("unknown AND free-unknown") + result = utils.handle_operator_expression( + expr, self.licensing, self.licensing.AND + ) + self.assertIsNone(result) + + @mock.patch("scanpipe.pipes.utils.shutil.which") + @mock.patch("scanpipe.pipes.utils.subprocess.run") + def test_check_docker_command_success(self, mock_subprocess_run, mock_shutil_which): + mock_shutil_which.return_value = "/usr/bin/docker" + mock_subprocess_run.return_value = mock.Mock(returncode=0) + + self.assertTrue(utils.check_docker_command()) + + @mock.patch("scanpipe.pipes.utils.shutil.which") + def test_check_docker_command_not_found(self, mock_shutil_which): + mock_shutil_which.return_value = None + + self.assertFalse(utils.check_docker_command()) + + @mock.patch("scanpipe.pipes.utils.subprocess.run") + def test_docker_image_exists(self, mock_subprocess_run): + mock_subprocess_run.return_value = mock.Mock(returncode=0) + self.assertTrue(utils.docker_image_exists("scancode:test")) + + mock_subprocess_run.return_value = mock.Mock(returncode=1) + self.assertFalse(utils.docker_image_exists("scancode:test")) + + @mock.patch("scanpipe.pipes.utils.subprocess.run") + def test_container_state(self, mock_subprocess_run): + mock_subprocess_run.return_value = mock.Mock(stdout="running\n") + self.assertEqual(utils.container_state("test-container"), "running") + + mock_subprocess_run.side_effect = subprocess.CalledProcessError(1, []) + self.assertEqual(utils.container_state("test-container"), "") + + def test_rename_container(self): + cmd = ["docker", "run", "--name", "old-name", "image"] + result = utils.rename_container(cmd, "new-name") + self.assertEqual(result, ["docker", "run", "--name", "new-name", "image"]) + + def test_is_startup_race(self): + self.assertTrue(utils.is_startup_race(125, "some error mounting things")) + self.assertTrue(utils.is_startup_race(1, "log [FATAL tini log")) + + self.assertFalse(utils.is_startup_race(1, "regular failure")) + self.assertFalse(utils.is_startup_race(125, "different error")) + self.assertFalse(utils.is_startup_race(1, "")) + + @mock.patch("scanpipe.pipes.utils.subprocess.Popen") + def test_run_docker_container_success(self, mock_popen): + mock_proc = mock.Mock() + mock_proc.communicate.return_value = ("stdout_data", "stderr_data") + mock_proc.returncode = 0 + mock_popen.return_value = mock_proc + + result = utils.run_docker_container( + ["docker", "run", "img"], "test-container", 100, "Testing container" + ) + self.assertEqual(result.returncode, 0) + self.assertEqual(result.stdout, "stdout_data") + + @mock.patch("scanpipe.pipes.utils.docker_rm_force") + @mock.patch("scanpipe.pipes.utils.subprocess.Popen") + def test_run_docker_container_failure(self, mock_popen, mock_docker_rm): + mock_proc = mock.Mock() + mock_proc.communicate.return_value = ("", "standard error message") + mock_proc.returncode = 1 + mock_popen.return_value = mock_proc + + with self.assertRaises(subprocess.CalledProcessError) as cm: + utils.run_docker_container( + ["docker", "run", "img"], + "test-container", + 100, + "Testing container", + retries=0, + ) + + self.assertEqual(cm.exception.stderr, "standard error message") + mock_docker_rm.assert_called_once_with("test-container") + + @mock.patch("scanpipe.pipes.utils.container_state") + @mock.patch("scanpipe.pipes.utils.docker_rm_force") + @mock.patch("scanpipe.pipes.utils.subprocess.Popen") + def test_run_docker_container_timeout_stuck_in_created( + self, mock_popen, mock_docker_rm, mock_container_state + ): + mock_proc = mock.Mock() + # Raise TimeoutExpired on the first call, return empty strings on + # the cleanup call + mock_proc.communicate.side_effect = [ + subprocess.TimeoutExpired("cmd", 15), + ("", ""), + ] + mock_popen.return_value = mock_proc + mock_container_state.return_value = "created" + + with self.assertRaises(subprocess.CalledProcessError) as cm: + utils.run_docker_container( + ["docker", "run", "img"], + "test-container", + 100, + "Testing container", + retries=0, + ) + + self.assertIn("stuck in 'Created' state after 15s", cm.exception.stderr) + mock_proc.kill.assert_called_once() + mock_docker_rm.assert_called_once_with("test-container") + + @mock.patch("scanpipe.pipes.utils.time.sleep") + @mock.patch("scanpipe.pipes.utils.docker_rm_force") + @mock.patch("scanpipe.pipes.utils.subprocess.Popen") + def test_run_docker_container_startup_race_retry( + self, mock_popen, mock_docker_rm, mock_sleep + ): + # First attempt: simulated race condition + mock_proc_fail = mock.Mock() + mock_proc_fail.communicate.return_value = ("", "error mounting") + mock_proc_fail.returncode = 125 + + # Second attempt: success + mock_proc_success = mock.Mock() + mock_proc_success.communicate.return_value = ("success_data", "") + mock_proc_success.returncode = 0 + + mock_popen.side_effect = [mock_proc_fail, mock_proc_success] + + result = utils.run_docker_container( + ["docker", "run", "--name", "test-container", "img"], + "test-container", + 100, + "Testing container", + retries=1, + ) + + self.assertEqual(result.returncode, 0) + self.assertEqual(result.stdout, "success_data") + mock_sleep.assert_called_once_with(5) + mock_docker_rm.assert_called_once_with("test-container") + self.assertEqual(mock_popen.call_count, 2) diff --git a/scanpipe/views.py b/scanpipe/views.py index 7981f2a7d7..a22278220c 100644 --- a/scanpipe/views.py +++ b/scanpipe/views.py @@ -1774,6 +1774,7 @@ def get_queryset(self): "compliance_alert", "copyright", "affected_by_vulnerabilities", + "extra_data", ) .with_resources_count() .order_by_package_url()