From 2a11b2ec08497de7d4951d99e3af3a0e1bb259a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Frauenschl=C3=A4ger?= Date: Tue, 20 Oct 2026 12:01:34 +0200 Subject: [PATCH] feat(compdb): optionally add source files the compile commands #include A source-set SBOM hashes the files listed as compile_commands.json entries. A product can also compile a .c file by including it from another one: wolfSSL compiles 29 such files, among them src/ssl_api_rw.c, src/x509.c and, when inlining is on, wolfcrypt/src/misc.c. None of those were listed, so editing one did not change the source-set hash or the SBOM. With the new --included-sources, compdb_sbom.py reruns every selected C, C++ and .S compile command with -M and adds each file the compiler read that has a source extension, lies under --root (else under the common directory of the listed sources), passes --include/--exclude and is not already listed. Headers stay out, as before. Every entry for a file is rerun, since a second target can compile it with other -D flags. Files are matched by device and inode, so a case-variant #include on macOS or Windows or a hardlink is not listed twice, paths through a symlinked directory resolve to the file the compiler read, and an included file whose basename clashes with another source is named in the error. The rerun is ` -M ` with only the flags that change what the preprocessor reads: -D, -U, -I and friends, -include, -imacros, -Wp, minus its -M and -o parts, --sysroot, --specs, -std, -m, -O, and the -f flags that set predefined macros (char signedness, -fshort-wchar, -ffreestanding, -fno-builtin, -fPIC, -fstack-protector, -fsanitize=, -fopenmp and the like). Every other -f flag is dropped, because several write files (SARIF, module caches, -fdeps-file). Options that take a separate argument are kept or dropped as a pair, so -mllvm, -framework and -undefined cannot split; preprocessor options passed through -Xclang or -Xpreprocessor (-D, -I, -include, -include-pch) are forwarded as they are, so a CMake precompiled header still applies, and -M sits in front so a misparsed option takes the source and fails instead of linking. Response files are expanded, nested ones too, and a missing one is an error; a ccache or distcc launcher is skipped; a universal build is scanned once per -arch, with that arch's -Xarch_ flags, and the results merged. Assembler entries (GNU as reads -M as --mri) are not rerun, and a -M run whose output does not list its own source is an error, so a compiler that gives -M another meaning cannot pass silently. The rerun never writes a file. It uses the build's own compiler, because only that resolves the target's include paths; a compiler that fails stops the run. The scan is opt-in. It runs the compilers compile_commands.json names, which the frontend otherwise treats as untrusted input, and the TI, IAR and CCRX compilers compdb also serves have no -M. On a CMake build of wolfSSL master the scan adds those 29 files to the 38 library sources in src/ and wolfcrypt/src/, in about 1.5 seconds. The scan lives in share/frontends/included_sources.py, imported only when the option is given, so the Zephyr frontend can use it too and loading compdb_sbom.py as a module does not need frontends/ on sys.path. New tests/test_included_sources.py covers the command rewrite, the -M output parser, and the frontend end to end (34 tests); selftest.yml runs it. --- .github/workflows/selftest.yml | 5 + share/README.md | 1 + share/frontends/compdb_sbom.py | 74 +++++- share/frontends/included_sources.py | 200 ++++++++++++++++ tests/test_included_sources.py | 343 ++++++++++++++++++++++++++++ 5 files changed, 619 insertions(+), 4 deletions(-) create mode 100644 share/frontends/included_sources.py create mode 100644 tests/test_included_sources.py diff --git a/.github/workflows/selftest.yml b/.github/workflows/selftest.yml index aa7db34..5980f10 100644 --- a/.github/workflows/selftest.yml +++ b/.github/workflows/selftest.yml @@ -41,6 +41,7 @@ jobs: share/validate_sbom.py \ share/frontends/compdb_sbom.py \ share/frontends/iar_sbom.py \ + share/frontends/included_sources.py \ share/frontends/zephyr_sbom.py \ central/gen-advisory \ central/test_gen_advisory.py \ @@ -59,6 +60,7 @@ jobs: tests/test_sbom_identity.py \ tests/test_low_findings.py \ tests/test_iar_sbom.py \ + tests/test_included_sources.py \ tests/test_sbom_cmake_install.py \ tests/test_wolfssl_compliance_tooling_sync.py \ tests/test_sbom_autotools.py @@ -84,6 +86,9 @@ jobs: - name: Run CMake SBOM install tests run: python -m unittest tests/test_sbom_cmake_install.py + - name: Run included-sources tests + run: python -m unittest tests/test_included_sources.py + - name: Run wolfssl-compliance-tooling-sync tests run: python -m unittest tests/test_wolfssl_compliance_tooling_sync.py diff --git a/share/README.md b/share/README.md index ebc1e89..9313c1a 100644 --- a/share/README.md +++ b/share/README.md @@ -13,6 +13,7 @@ into a product at `tools/sbom/`, together with the pin files (`VERSION` and | `validate_sbom.py` | Structural validator for CI (`--name-prefix`). | | `frontends/compdb_sbom.py` | Extractor for any `compile_commands.json`. | | `frontends/iar_sbom.py` | Extractor for an IAR Embedded Workbench `.ewp`. | +| `frontends/included_sources.py` | Finds the source files a compile command compiles in through `#include` (`compdb_sbom.py --included-sources`). | | `frontends/zephyr_sbom.py` | Extractor for a Zephyr module `CMakeLists.txt`. | | `build/sbom.mk` | Shared plain-Make fragment and `wolfssl_compliance_tooling_sbom_rule` macro. | | `build/sbom.cmake` | Shared CMake helper: `wolfssl_compliance_tooling_add_sbom()`. | diff --git a/share/frontends/compdb_sbom.py b/share/frontends/compdb_sbom.py index c744ab9..b4548c6 100755 --- a/share/frontends/compdb_sbom.py +++ b/share/frontends/compdb_sbom.py @@ -13,6 +13,11 @@ * Renesas e2studio / CCRX (via a build wrapper that records commands) * Xilinx SDK / Vitis (via `bear -- make ...`) +A source a file compiles in through #include, such as wolfSSL's +src/ssl_api_*.c, is not an entry of its own. `--included-sources` adds those +by rerunning the selected commands with -M, which needs a gcc- or clang-style +compiler and runs the compilers the database names. + The extracted sources + defines are handed to the shared driver (sbom-driver), so the resulting CycloneDX 1.6 / SPDX 2.3 SBOM is identical in shape to the Make, CMake, and IAR paths. @@ -31,6 +36,8 @@ import tempfile SRC_EXTS = ('.c', '.cc', '.cpp', '.cxx', '.s', '.asm') +# Only these are preprocessed, so only these are rerun with -M. +SCAN_EXTS = ('.c', '.cc', '.cpp', '.cxx', '.S') SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) # The shared driver is the sibling of this frontends/ directory. @@ -48,6 +55,11 @@ def entry_tokens(entry): return [] +def file_id(path): + st = os.stat(path) + return st.st_dev, st.st_ino + + def macro_name(d): """Macro name of a -D/-U operand: the text before the first '=' or '(', so a function-like 'F(a)=a' is keyed as 'F'.""" @@ -117,6 +129,10 @@ def main(): ap.add_argument('--spdx-out', default=None) ap.add_argument('--srcs-out', default=None) ap.add_argument('--print-only', action='store_true') + ap.add_argument('--included-sources', action='store_true', + help='Rerun the compile commands with -M and add the ' + 'source files they #include. Runs the compilers ' + 'the database names; needs one that supports -M.') args = ap.parse_args() if not os.path.isfile(args.compdb): @@ -135,7 +151,7 @@ def main(): # product must not pull an unrelated file (and its path) into the SBOM. root_abs = os.path.realpath(args.root) if args.root else None - srcs, defines, seen = [], set(), set() + srcs, defines, seen, selected = [], set(), set(), [] for entry in db: fpath = entry.get('file') if not fpath: @@ -158,18 +174,67 @@ def main(): continue for d in extract_defines(entry_tokens(entry)): defines.add(d) - if fpath not in seen and os.path.isfile(fpath): + if not os.path.isfile(fpath): + continue + if fpath not in seen: seen.add(fpath) srcs.append(fpath) + ext = os.path.splitext(fpath)[1] + if ext in SCAN_EXTS or ext.lower() in SCAN_EXTS: + selected.append((fpath, entry)) if not srcs: sys.exit("ERROR: no matching source files found in compilation database") + included = [] + if args.included_sources: + sys.path.insert(0, SCRIPT_DIR) + from included_sources import ScanError, included_sources + try: + scan_root = root_abs or os.path.commonpath( + [os.path.dirname(s) for s in srcs]) + except ValueError: + sys.exit("ERROR: sources share no common directory; pass --root") + known = {file_id(s) for s in srcs} + names = {os.path.basename(s): s for s in srcs} + scanned = set() + for fpath, entry in selected: + tokens = entry_tokens(entry) + key = (fpath, entry.get('directory'), tuple(tokens)) + if key in scanned: + continue + scanned.add(key) + if not tokens: + sys.stderr.write(f"WARNING: no compile command for {fpath}; " + f"its #include'd sources are not scanned\n") + continue + try: + found = included_sources( + tokens, entry.get('directory', os.getcwd()), fpath, + scan_root, SRC_EXTS) + except ScanError as e: + sys.exit(f"ERROR: {e}") + for path in found: + if (not os.path.isfile(path) or file_id(path) in known + or (inc and not any(r.search(path) for r in inc)) + or (exc and any(r.search(path) for r in exc))): + continue + name = os.path.basename(path) + if name in names: + sys.exit(f"ERROR: {path}, #include'd by {fpath}, has the " + f"same basename as {names[name]}; --exclude " + f"one of them") + known.add(file_id(path)) + names[name] = path + included.append(path) + srcs += included + defines = sorted(defines) cflags = ' '.join(f'-D{d}' for d in defines) if args.print_only: - print(f"# {len(srcs)} sources, {len(defines)} defines") + print(f"# {len(srcs)} sources ({len(included)} included), " + f"{len(defines)} defines") print("\n[defines]") for d in defines: print(f" -D{d}") @@ -204,7 +269,8 @@ def main(): for inc in args.include_dir: cmd += ['--include-dir', inc] - print(f"compdb SBOM: {len(srcs)} sources, {len(defines)} defines") + print(f"compdb SBOM: {len(srcs)} sources ({len(included)} included), " + f"{len(defines)} defines") try: rc = subprocess.call(cmd) finally: diff --git a/share/frontends/included_sources.py b/share/frontends/included_sources.py new file mode 100644 index 0000000..2097a20 --- /dev/null +++ b/share/frontends/included_sources.py @@ -0,0 +1,200 @@ +"""Find the source files a compile command compiles in through #include, +such as wolfSSL's src/ssl_api_*.c, which are not compile_commands.json +entries of their own.""" + +import os +import re +import shlex +import subprocess + +# Options whose next token is their argument, kept for the -M run. +_KEEP_PAIRS = ('-D', '-U', '-I', '-iquote', '-isystem', '-isystem-after', + '-idirafter', '-imacros', '-include', '-isysroot', '-iprefix', + '-iwithprefix', '-iwithprefixbefore', '-iwithsysroot', + '-iframework', '-iframeworkwithsysroot', '-F', '-target', + '-arch', '--sysroot', '-x', '-specs', '--specs', + '--gcc-toolchain', '-gcc-toolchain', '-meabi', '-include-pch') +# Options whose next token is their argument, dropped with it. +_DROP_PAIRS = ('-o', '-MF', '-MT', '-MQ', '-MJ', '-Xclang', '-Xpreprocessor', + '-Xassembler', '-Xlinker', '-mllvm', '--param', '--output', + '-framework', '-undefined', '-force_load', '-filelist', + '-multiply_defined', '-mthread-model', '-arch_only', '-mmlir', + '-module-dependency-dir', '-multiply_defined_unused', + '-target-arch-variant') +_KEEP_EXACT = ('-ansi', '-undef', '-pthread', '-nostdinc', '-nostdinc++', + '-nobuiltininc') +# Joined forms of the pairs, and flags that set target or language macros. +_KEEP = tuple(o for o in _KEEP_PAIRS if o != '-arch') + ( + '--sysroot=', '--target=', '-std=', '--std=', '-stdlib=', '-specs=', + '--specs=', '--gcc-toolchain=', '-m', '-O') +# -f flags that change predefined macros; any other -f may write a file. +_F_KEEP = ('signed-char', 'unsigned-char', 'short-wchar', 'short-enums', + 'freestanding', 'hosted', 'builtin', 'pic', 'PIC', 'pie', 'PIE', + 'stack-protector', 'sanitize=', 'openmp', 'openacc', 'exceptions', + 'cxx-exceptions', 'rtti', 'ms-extensions', 'ms-compatibility', + 'declspec', 'blocks', 'gnu89-inline', 'asm', 'wrapv', 'trapv', + 'char8_t', 'gnu-keywords', 'coroutines', 'fast-math', 'math-errno', + 'finite-math-only', 'inline', 'cf-protection', 'input-charset=', + 'exec-charset=', 'dollars-in-identifiers', 'threadsafe-statics', + 'sized-deallocation', 'aligned-new', 'unwind-tables', + 'leading-underscore', 'single-precision-constant', 'abi-version=') +# Preprocessor options worth forwarding from -Xclang and -Xpreprocessor. +_WRAPPED_PAIRS = ('-D', '-U', '-I', '-include', '-imacros', '-include-pch', + '-isystem', '-iquote', '-idirafter') +_LAUNCHERS = ('ccache', 'sccache', 'distcc', 'icecc') +# GNU as takes -M as --mri. +_ASSEMBLER = re.compile(r'(^|-)as(\.exe)?$') + + +class ScanError(Exception): + pass + + +def _wp_option(t): + """A -Wp, option without its -M and -o parts, or None if none is left.""" + parts, kept = t.split(',')[1:], [] + while parts: + p = parts.pop(0) + if p in ('-MD', '-MMD', '-MF', '-MT', '-MQ', '-o', '--output') \ + and parts: + parts.pop(0) + elif not p.startswith(('-M', '-o', '--output')): + kept.append(p) + return '-Wp,' + ','.join(kept) if kept else None + + +def _keep_flag(t): + """t if it is a single-token flag the -M run needs, else None.""" + if t.startswith('-Wp,'): + return _wp_option(t) + if t.startswith('-f'): + name = t[5:] if t.startswith('-fno-') else t[2:] + return t if name.startswith(_F_KEEP) else None + if t in _KEEP_EXACT or (t.startswith(_KEEP) and t not in _DROP_PAIRS): + return t + return None + + +def _expand_response_files(tokens, directory, depth=0): + out = [] + for t in tokens: + if not t.startswith('@') or t == '@': + out.append(t) + continue + path = os.path.join(directory or '', t[1:]) + if depth > 8: + raise ScanError(f"response files nested too deep at {path}") + try: + with open(path, errors='surrogateescape') as f: + inner = shlex.split(f.read()) + except (OSError, ValueError) as e: + raise ScanError(f"cannot read response file {path}: {e}") + out += _expand_response_files(inner, directory, depth + 1) + return out + + +def _compiler_index(tokens): + i = 0 + while (i < len(tokens) - 1 and os.path.splitext( + os.path.basename(tokens[i]))[0] in _LAUNCHERS): + i += 1 + return i + + +def _arches(tokens): + return list(dict.fromkeys(tokens[j + 1] for j in range(len(tokens) - 1) + if tokens[j] == '-arch')) + + +def dependency_command(tokens, source, arch=None): + """The preprocessor flags of a compile command for one -arch (the + first by default), as a -M run on source.""" + i = _compiler_index(tokens) + rest = tokens[i + 1:] + arch = arch or next(iter(_arches(rest)), None) + # -M goes first so a misparsed option swallows the source and fails. + cmd, i = [tokens[i], '-M'], 0 + while i < len(rest): + t = rest[i] + pair = i + 1 < len(rest) + if t in ('-Xclang', '-Xpreprocessor') and pair: + inner = rest[i + 1] + if (inner in _WRAPPED_PAIRS and i + 3 < len(rest) + and rest[i + 2] == t): + cmd += rest[i:i + 4] + i += 4 + continue + if inner.startswith(('-D', '-U', '-I')): + cmd += [t, inner] + i += 2 + elif t == f'-Xarch_{arch}' and pair: + kept = _keep_flag(rest[i + 1]) + cmd += [kept] if kept else [] + i += 2 + elif (t in _DROP_PAIRS or t.startswith('-Xarch_')) and pair: + i += 2 + elif t in _KEEP_PAIRS and pair: + if t != '-arch' or rest[i + 1] == arch: + cmd += [t, rest[i + 1]] + i += 2 + else: + kept = _keep_flag(t) + cmd += [kept] if kept else [] + i += 1 + return cmd + [source] + + +def parse_make_deps(text): + """Return the prerequisites of the make rules printed by -M.""" + text = text.replace('\\\r\n', ' ').replace('\\\n', ' ') + deps = [] + for rule in text.splitlines(): + prereqs = re.split(r':(?:\s|$)', rule, maxsplit=1) + if len(prereqs) < 2: + continue + deps += [d.replace('\\ ', ' ').replace('\\#', '#').replace('$$', '$') + for d in re.split(r'(?&2\nexit 1\n") + os.chmod(cc, 0o755) + self.write_db(cc) + r = self.run_compdb() + self.assertNotEqual(r.returncode, 0) + self.assertIn("nope", r.stderr) + + def test_module_import_without_frontends_on_path(self): + code = ("import importlib.util as u; s = u.spec_from_file_location(" + f"'c', {COMPDB!r}); m = u.module_from_spec(s); " + "s.loader.exec_module(m); print(m.SRC_EXTS)") + r = subprocess.run([sys.executable, "-I", "-c", code], + capture_output=True, text=True, cwd="/") + self.assertEqual(r.returncode, 0, r.stderr) + + def test_off_by_default(self): + r = subprocess.run( + [sys.executable, COMPDB, self.db, "--name", "t", + "--root", self.root, "--print-only"], + capture_output=True, text=True) + self.assertEqual(r.returncode, 0, r.stderr) + self.assertNotIn("ssl_api.c", r.stdout) + + def test_every_entry_for_a_file_is_scanned(self): + self.write("src/ssl.c", '#ifdef INL\n#include "misc.c"\n#endif\n') + self.write("src/misc.c", "int misc;\n") + self.write_entries(["cc", "-c", "src/ssl.c"], + ["cc", "-DINL", "-c", "src/ssl.c"]) + r = self.run_compdb() + self.assertEqual(r.returncode, 0, r.stderr) + self.assertIn("misc.c", r.stdout) + + def test_response_file(self): + os.makedirs(os.path.join(self.root, "inc")) + self.write("inc/extra.c", "int extra;\n") + self.write("src/ssl.c", "#include \n") + self.write("flags.rsp", "-Iinc -o ssl.o\n") + self.write_entries(["cc", "@flags.rsp", "-c", "src/ssl.c"]) + r = self.run_compdb() + self.assertEqual(r.returncode, 0, r.stderr) + self.assertIn("extra.c", r.stdout) + self.assertFalse(os.path.exists(os.path.join(self.root, "ssl.o"))) + + def test_nested_response_file(self): + os.makedirs(os.path.join(self.root, "inc")) + self.write("inc/extra.c", "int extra;\n") + self.write("src/ssl.c", "#include \n") + self.write("outer.rsp", "@inner.rsp\n") + self.write("inner.rsp", "-Iinc\n") + self.write_entries(["cc", "@outer.rsp", "-c", "src/ssl.c"]) + r = self.run_compdb() + self.assertEqual(r.returncode, 0, r.stderr) + self.assertIn("extra.c", r.stdout) + + def test_missing_response_file_fails(self): + self.write_entries(["cc", "@gone.rsp", "-c", "src/ssl.c"]) + r = self.run_compdb() + self.assertNotEqual(r.returncode, 0) + self.assertIn("gone.rsp", r.stderr) + + def test_hardlink_of_listed_source_is_not_added(self): + os.link(os.path.join(self.root, "src", "ssl_api.c"), + os.path.join(self.root, "src", "alias.c")) + self.write("src/ssl.c", '#include "alias.c"\n') + self.write_entries(["cc", "-c", "src/ssl.c"], + ["cc", "-c", "src/ssl_api.c"]) + r = self.run_compdb() + self.assertEqual(r.returncode, 0, r.stderr) + self.assertIn("(0 included)", r.stdout) + + def test_path_through_symlink_is_resolved(self): + os.makedirs(os.path.join(self.root, "real", "sub")) + self.write("real/y.c", "int real_y;\n") + os.symlink(os.path.join(self.root, "real", "sub"), + os.path.join(self.root, "src", "link")) + self.write("src/ssl.c", '#include "link/../y.c"\n') + self.write_entries(["cc", "-c", "src/ssl.c"]) + r = self.run_compdb() + self.assertEqual(r.returncode, 0, r.stderr) + self.assertIn(os.path.join("real", "y.c"), r.stdout) + + def test_case_variant_include_is_listed_once(self): + if not os.path.exists(os.path.join(self.root, "SRC")): + self.skipTest("case-sensitive filesystem") + self.write("src/ssl.c", '#include "SSL_API.c"\n') + self.write_entries(["cc", "-c", "src/ssl.c"], + ["cc", "-c", "src/ssl_api.c"]) + r = self.run_compdb() + self.assertEqual(r.returncode, 0, r.stderr) + self.assertIn("(0 included)", r.stdout) + + def test_assembly_entry_is_not_scanned(self): + self.write("src/a.s", "nop\n") + self.write_entries(["/nonexistent/as", "-c", "src/a.s"]) + r = self.run_compdb() + self.assertEqual(r.returncode, 0, r.stderr) + + def test_basename_clash_names_the_included_file(self): + os.makedirs(os.path.join(self.root, "other")) + self.write("other/ssl_api.c", "int other;\n") + self.write_db("cc", ["cc", "-c", "other/ssl_api.c"]) + r = self.run_compdb() + self.assertNotEqual(r.returncode, 0) + self.assertIn("same basename", r.stderr) + self.assertIn(os.path.join("src", "ssl_api.c") + ", #include'd", + r.stderr) + + def test_every_arch_is_scanned(self): + self.write("src/ssl.c", '#ifdef __aarch64__\n#include "arm.c"\n' + '#else\n#include "x86.c"\n#endif\n') + self.write("src/arm.c", "int arm;\n") + self.write("src/x86.c", "int x86;\n") + cc = os.path.join(self.tmp.name, "archcc") + with open(cc, "w") as f: + f.write('#!/bin/sh\nfor a; do case "$p" in -arch) arch=$a;; ' + 'esac; p=$a; src=$a; done\n' + 'if [ "$arch" = arm64 ]; then d=arm.c; else d=x86.c; fi\n' + 'echo "ssl.o: $src src/$d"\n') + os.chmod(cc, 0o755) + self.write_entries([cc, "-arch", "x86_64", "-arch", "arm64", "-c", + "src/ssl.c"]) + r = self.run_compdb() + self.assertEqual(r.returncode, 0, r.stderr) + self.assertIn("arm.c", r.stdout) + self.assertIn("x86.c", r.stdout) + + def test_assembler_entry_is_not_run(self): + self.write("src/s.S", "nop\n") + self.write_entries(["/nonexistent/arm-none-eabi-as", "-c", "src/s.S"]) + r = self.run_compdb() + self.assertEqual(r.returncode, 0, r.stderr) + + def test_empty_dependency_output_fails(self): + cc = os.path.join(self.tmp.name, "quietcc") + with open(cc, "w") as f: + f.write("#!/bin/sh\nexit 0\n") + os.chmod(cc, 0o755) + self.write_db(cc) + r = self.run_compdb() + self.assertNotEqual(r.returncode, 0) + self.assertIn("printed no dependencies", r.stderr) + + def test_missing_compiler_fails(self): + self.write_db("/nonexistent/cc") + r = self.run_compdb() + self.assertNotEqual(r.returncode, 0) + self.assertIn("cannot run", r.stderr) + + +if __name__ == "__main__": + unittest.main(verbosity=2)