From 90f2124da5241f7ea86967c8030da82e8d2f5b1c Mon Sep 17 00:00:00 2001 From: Sameeh Jubran Date: Tue, 3 Nov 2026 09:59:41 +0200 Subject: [PATCH] fix(frontends): handle other drives, compiler launchers and app-target modules Copilot found these on wolfSSL/wolfTPM#658, in the vendored copies. - compdb: on Windows, commonpath() raises ValueError for a source and a --root on two drives. Treat that source as outside --root, as included_sources already does. - zephyr: strip ccache, sccache, distcc and icecc from the compile command, so the settings capture runs the real compiler. - zephyr: add --target. A module that calls target_sources(app ...), as wolfTPM does, has no modules// object directory, so the default selection found no sources. --target app takes the objects of that target under --root. The default does not change. --- share/README.md | 3 ++- share/frontends/compdb_sbom.py | 6 ++++- share/frontends/zephyr_sbom.py | 41 +++++++++++++++++++++++++--------- tests/test_compdb_sbom.py | 32 ++++++++++++++++++++++++++ tests/test_zephyr_sbom.py | 38 +++++++++++++++++++++++++++++++ 5 files changed, 108 insertions(+), 12 deletions(-) diff --git a/share/README.md b/share/README.md index b88d114..c4c33b4 100644 --- a/share/README.md +++ b/share/README.md @@ -90,7 +90,8 @@ A product does not copy logic. It describes itself: actually built. Without that hook, run `zephyr_sbom.py` on the build's `compile_commands.json` with `--root` set to the module, and pass `--user-settings` with the module's settings header to record its - configuration. + configuration. A module that adds its sources with `target_sources(app + ...)`, as wolfTPM does, needs `--target app`. Keep only true product knowledge in the product: the route-through script, the module extractor, and the HAL source selector. diff --git a/share/frontends/compdb_sbom.py b/share/frontends/compdb_sbom.py index 11eaddf..1bbd2fb 100755 --- a/share/frontends/compdb_sbom.py +++ b/share/frontends/compdb_sbom.py @@ -194,7 +194,11 @@ def main(): continue if root_abs: real = os.path.realpath(fpath) - if os.path.commonpath([real, root_abs]) != root_abs: + try: + inside = os.path.commonpath([real, root_abs]) == root_abs + except ValueError: + inside = False + if not inside: sys.stderr.write( f"WARNING: skipping source outside --root: {fpath}\n") continue diff --git a/share/frontends/zephyr_sbom.py b/share/frontends/zephyr_sbom.py index c06c566..b7bc6c0 100755 --- a/share/frontends/zephyr_sbom.py +++ b/share/frontends/zephyr_sbom.py @@ -14,6 +14,9 @@ under `--root` whose object Zephyr placed in the module's library directory, `/modules//`. That set follows Kconfig and any CMake logic the module uses, and it leaves out application files inside the module tree. +A module that adds its sources to another target, for example with +`target_sources(app ...)`, has no library directory. For that module, pass +`--target app` to take the objects of that target under `--root`. The configuration is Kconfig-driven, so by default this produces a source-inventory SBOM. `--user-settings` records the module's -D flags and the @@ -37,6 +40,8 @@ SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) DEFAULT_DRIVER = os.path.join(os.path.dirname(SCRIPT_DIR), "sbom-driver") +sys.path.insert(0, SCRIPT_DIR) +from included_sources import _compiler_index # noqa: E402 SRC_EXTS = ('.c', '.cc', '.cpp', '.cxx', '.s', '.asm') _ARG_OPTS = ('-D', '-U', '-I', '-iquote', '-isystem', '-idirafter', @@ -68,15 +73,20 @@ def module_name(root): return os.path.basename(os.path.normpath(root)) -def compdb_entries(compdb, root, name): +def compdb_entries(compdb, root, name, target=None): """Return the module library's compile commands as [(source, entry)] - for sources under root, plus the library sources outside it.""" + for sources under root, plus the library sources outside it. + + With target, take the objects of that CMake target instead, for a + module that adds its sources to another target such as app.""" try: with open(compdb) as f: db = json.load(f) except (OSError, json.JSONDecodeError) as e: raise ParseError(f"cannot read {compdb}: {e}") root = os.path.realpath(root) + os.sep + target_dir = (os.sep + os.path.join('CMakeFiles', target + '.dir') + os.sep + if target else None) picked, outside, seen = [], [], set() for e in db: if 'output' not in e: @@ -85,12 +95,15 @@ def compdb_entries(compdb, root, name): if not e.get('file'): continue directory = e.get('directory', '') - lib_dir = os.path.normpath( - os.path.join(directory, 'modules', name)) + os.sep out = os.path.normpath(os.path.join(directory, e['output'])) src = os.path.normpath(os.path.join(directory, e['file'])) - if (not out.startswith(lib_dir) - or not src.lower().endswith(SRC_EXTS)): + if target_dir: + in_target = target_dir in out + else: + lib_dir = os.path.normpath( + os.path.join(directory, 'modules', name)) + os.sep + in_target = out.startswith(lib_dir) + if not in_target or not src.lower().endswith(SRC_EXTS): continue if os.path.realpath(src).startswith(root): picked.append((src, e)) @@ -144,7 +157,7 @@ def _entry_tokens(entry): f"{entry.get('file')}: {e}") if not toks: raise ParseError(f"empty compile command for {entry.get('file')}") - return toks + return toks[_compiler_index(toks):] def _dump_macros(cc, flags, directory): @@ -267,6 +280,11 @@ def main(): ap.add_argument('--module-name', default=None, help='Zephyr module name. Default: the name in ' 'zephyr/module.yml, else the --root directory name.') + ap.add_argument('--target', default=None, + help='CMake target that compiles the module sources, ' + 'e.g. app for a module that calls ' + 'target_sources(app ...). Default: the module ' + 'library.') ap.add_argument('--user-settings', default=None, help="The module's settings header, captured with the " "module's compile flags.") @@ -315,11 +333,14 @@ def main(): settings = None try: name = args.module_name or module_name(args.root) - entries, outside = compdb_entries(args.compdb, args.root, name) + entries, outside = compdb_entries(args.compdb, args.root, name, + args.target) if not entries: + what = (f"target '{args.target}'" if args.target + else f"module '{name}'") sys.exit(f"ERROR: {args.compdb} has no compile commands for " - f"module '{name}' from sources under {args.root}; " - f"check --root and --module-name.") + f"{what} from sources under {args.root}; " + f"check --root, --module-name and --target.") if args.user_settings: settings = capture_user_settings(entries, args.user_settings, args.hostcc) diff --git a/tests/test_compdb_sbom.py b/tests/test_compdb_sbom.py index 702f75a..ba7680e 100644 --- a/tests/test_compdb_sbom.py +++ b/tests/test_compdb_sbom.py @@ -1,12 +1,16 @@ #!/usr/bin/env python3 """Tests for the compilation-database frontend (share/frontends/compdb_sbom.py).""" +import contextlib +import importlib.util +import io import json import os import subprocess import sys import tempfile import unittest +from unittest import mock REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) COMPDB = os.path.join(REPO, "share", "frontends", "compdb_sbom.py") @@ -179,6 +183,34 @@ def test_selected_source_outside_root_is_warned(self): self.assertIn("outside --root", r.stderr) self.assertNotIn("app.c", r.stdout) + def test_source_on_another_drive_is_warned(self): + # Windows commonpath() raises ValueError for paths on two drives. + root = self.write_outside() + spec = importlib.util.spec_from_file_location("compdb_sbom", COMPDB) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + real_commonpath = os.path.commonpath + + def commonpath(paths): + if any(os.path.basename(p) == "app.c" for p in paths): + raise ValueError("Paths don't have the same drive") + return real_commonpath(paths) + + out, err = io.StringIO(), io.StringIO() + argv = [COMPDB, self.db, "--name", "t", "--root", root, + "--print-only"] + with mock.patch.object(sys, "argv", argv), \ + mock.patch.object(mod.os.path, "commonpath", commonpath), \ + contextlib.redirect_stdout(out), \ + contextlib.redirect_stderr(err): + try: + mod.main() + except SystemExit as e: + self.assertIn(e.code, (None, 0), err.getvalue()) + self.assertIn("outside --root", err.getvalue()) + self.assertIn("in.c", out.getvalue()) + self.assertNotIn("app.c", out.getvalue()) + if __name__ == "__main__": unittest.main(verbosity=2) diff --git a/tests/test_zephyr_sbom.py b/tests/test_zephyr_sbom.py index 85c6824..512ac11 100644 --- a/tests/test_zephyr_sbom.py +++ b/tests/test_zephyr_sbom.py @@ -146,6 +146,27 @@ def test_no_module_entries_fails(self): self.assertNotEqual(r.returncode, 0) self.assertIn("--module-name", r.stderr) + def test_target_selects_app_sources(self): + compdb = self.write_compdb([ + ("src/core.c", APP), ("src/est.c", APP), ("src/scep.c", LIB), + ("src/core.c", "CMakeFiles/app_extra.dir/"), + (self.outside, APP)]) + entries, outside = zephyr.compdb_entries(compdb, self.mod, "mod", + "app") + self.assertEqual(self.rels(entries), ["src/core.c", "src/est.c"]) + self.assertEqual(outside, [self.outside]) + r = self.run_frontend("--compdb", compdb, "--target", "app", + "--print-only") + self.assertEqual(r.returncode, 0, r.stderr) + self.assertIn("# mod: 2 sources", r.stdout) + + def test_target_without_entries_fails(self): + compdb = self.write_compdb([("src/core.c", LIB)]) + r = self.run_frontend("--compdb", compdb, "--target", "app", + "--print-only") + self.assertNotEqual(r.returncode, 0) + self.assertIn("target 'app'", r.stderr) + def test_source_outside_root_is_warned(self): compdb = self.write_compdb([("src/core.c", LIB), (self.outside, LIB)]) r = self.run_frontend("--compdb", compdb, "--print-only") @@ -320,6 +341,23 @@ def test_asm_entry_first_is_skipped(self): self.assertEqual(r.returncode, 0, r.stderr) self.assertIn("#define HAVE_X 1", r.stdout) + def test_compiler_launcher_is_skipped(self): + launcher = os.path.join(self.tmp.name, "ccache") + with open(launcher, "w") as f: + f.write("#!/bin/sh\nexit 1\n") + os.chmod(launcher, 0o755) + compdb = self.write_compdb([("src/core.c", LIB)], self.flags) + with open(compdb) as f: + db = json.load(f) + db[0]["arguments"].insert(0, launcher) + with open(compdb, "w") as f: + json.dump(db, f) + r = self.run_frontend("--compdb", compdb, "--print-only", + "--user-settings", self.settings) + self.assertEqual(r.returncode, 0, r.stderr) + self.assertIn("#define HAVE_X 1", r.stdout) + self.assertIn("#define MOD_DEBUG 1", r.stdout) + def test_hostcc_needs_user_settings(self): compdb = self.write_compdb([("src/core.c", LIB)]) r = self.run_frontend("--compdb", compdb, "--hostcc", "cc")