diff --git a/docs/testing.md b/docs/testing.md index 80a622b..45c9579 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -104,6 +104,15 @@ directly on the host under `packages//tmp` (the checkout is bind-mounte container), and use `docker cp linux-pkg-: ` for anything that lives only inside the container. +**Gotcha:** for a package with `SBOM_DEEP_SCAN="true"` whose `.deb` ships an npm lock file +(`yarn.lock`, `package-lock.json`, `npm-shrinkwrap.json` or `pnpm-lock.yaml`) next to a +`package.json`, outside `node_modules` — virtualization ships one per browser UI project — +`generate_sbom()` asks an npm registry for the license of every runtime dependency in it. That +needs network access to `SBOM_NPM_REGISTRY` (default: Artifactory's `dlpx-npm`, which needs no +credentials) and takes a few minutes for virtualization. A registry that cannot be reached is not +an error: the components are still listed, without licenses, so check the SBOM's licenses rather +than the build's exit status. Set `SBOM_NPM_REGISTRY` to point the lookups at another registry. + ### 1.3 Testing changes to an existing package (upstream source, not linux-pkg itself) If you changed the upstream source of a package `linux-pkg` already builds (not `linux-pkg`'s own diff --git a/lib/common.sh b/lib/common.sh index 2746f3c..f72591f 100644 --- a/lib/common.sh +++ b/lib/common.sh @@ -1581,6 +1581,78 @@ function sanitize_sbom() { logmust mv "$tmp" "$sbom_file" } +# +# Registry Syft asks for the license of each npm package it lists from a lock +# file. A lock file records only names and versions, and every Delphix npm +# build already resolves through this registry, so it can answer for internal +# @delphix packages as well as public ones. +# +SBOM_NPM_REGISTRY="${SBOM_NPM_REGISTRY:-https://artifactory.delphix.com/artifactory/api/npm/dlpx-npm/}" + +# +# Replace every npm lock file shipped in an extracted .deb payload with a +# CycloneDX document of that project's runtime dependencies, for the main scan +# in generate_sbom() to pick up through Syft's sbom-cataloger. +# +# A package whose browser UI is shipped as minified bundles can ship the UI +# project's package.json and lock file alongside it, so that what went into +# those bundles is still on record. Syft reads a lock file directly, but it +# can only tell runtime from development dependencies by reading the +# package.json next to it, and Syft 1.46 does so only when that lock file is at +# the root of the scan. Anywhere deeper it lists every entry, so a UI's whole +# build toolchain (compilers, bundlers, test runners) is reported as shipped +# code: 1,847 components rather than 445 for one project in dlpx-app-gate. +# Scanning each lock file's directory on its own puts it at the root. +# +# Lock files under node_modules belong to installed packages, which the main +# scan already lists from their own package.json, so they are left alone. +# +function catalog_npm_lockfiles() { + local root="$1" + local dirs=() dir lock pids=() i + + while IFS= read -r -d '' lock; do + dir="$(dirname "$lock")" + [[ -f "$dir/package.json" ]] || continue + [[ " ${dirs[*]} " == *" $dir "* ]] || dirs+=("$dir") + done < <(find "$root" -path '*/node_modules' -prune -o \ + \( -name yarn.lock -o -name package-lock.json -o \ + -name npm-shrinkwrap.json -o -name pnpm-lock.yaml \) \ + -type f -print0) + + [[ ${#dirs[@]} -eq 0 ]] && return 0 + + # + # Each scan looks up the license of every package with one request at + # a time, which takes minutes for a large UI project, so the projects + # are scanned in parallel. + # + for dir in "${dirs[@]}"; do + echo "Cataloging npm runtime dependencies of ${dir#"$root"}" + SYFT_FILE_METADATA_SELECTION=none \ + SYFT_JAVASCRIPT_INCLUDE_DEV_DEPENDENCIES=false \ + SYFT_JAVASCRIPT_SEARCH_REMOTE_LICENSES=true \ + SYFT_JAVASCRIPT_NPM_BASE_URL="$SBOM_NPM_REGISTRY" \ + syft scan "dir:$dir" -q \ + --override-default-catalogers javascript-lock-cataloger \ + -o "cyclonedx-json@1.6=$dir.npm.cdx.json" & + pids+=($!) + done + for i in "${!pids[@]}"; do + wait "${pids[$i]}" || + die "Failed to catalog the npm lock file in '${dirs[$i]}'." + done + + # + # The lock files and package.json go, so that the main scan neither + # lists every entry again nor reports the project itself as a package. + # + for dir in "${dirs[@]}"; do + logmust rm -f "$dir"/{package.json,yarn.lock,package-lock.json,npm-shrinkwrap.json,pnpm-lock.yaml} + logmust mv "$dir.npm.cdx.json" "$dir/npm-runtime.cdx.json" + done +} + function generate_sbom() { if [[ "$SBOM_DEEP_SCAN" != "true" ]]; then return 0 @@ -1661,6 +1733,7 @@ function generate_sbom() { # extract_dir="$(logmust mktemp -d)" logmust dpkg-deb -x "$deb" "$extract_dir" + logmust catalog_npm_lockfiles "$extract_dir" # # Full catalogers here, deliberately unlike appliance-build's @@ -1671,12 +1744,22 @@ function generate_sbom() { # found belongs to that package by construction, which is the # entire point of scanning here rather than at image level. # + # Syft's default set for a directory scan is not quite all of + # them: it leaves out the catalogers it tags as "image" only. + # Two of those matter here. javascript-package-cataloger lists + # installed node_modules packages from their package.json; + # without it a package that ships a node_modules tree (such as + # virtualization's CLI) lists none of it. sbom-cataloger reads + # SBOMs shipped in the payload, which is how the documents + # catalog_npm_lockfiles() leaves are merged in. + # # SYFT_FILE_METADATA_SELECTION=none suppresses Syft's default # per-file "file" component (with SHA-1/SHA-256 hashes and the # scanned path baked in) -- with a whole extracted payload to # walk that would otherwise emit an entry per file. # SYFT_FILE_METADATA_SELECTION=none logmust syft scan "dir:$extract_dir" \ + --select-catalogers +javascript-package-cataloger,+sbom-cataloger \ --source-name "$PACKAGE" \ --source-version "$deb_version" \ -o "cyclonedx-json@1.6=$sbom_file"