feat(compdb): optionally add source files the compile commands #include

.github/workflows/selftest.yml

@@ -41,6 +41,7 @@ jobs:
             share/validate_sbom.py \
             share/frontends/compdb_sbom.py \
             share/frontends/iar_sbom.py \
+            share/frontends/included_sources.py \
             share/frontends/zephyr_sbom.py \
             central/gen-advisory \
             central/test_gen_advisory.py \
@@ -59,6 +60,7 @@ jobs:
             tests/test_sbom_identity.py \
             tests/test_low_findings.py \
             tests/test_iar_sbom.py \
+            tests/test_included_sources.py \
             tests/test_sbom_cmake_install.py \
             tests/test_wolfssl_compliance_tooling_sync.py \
             tests/test_sbom_autotools.py
@@ -84,6 +86,9 @@ jobs:
       - name: Run CMake SBOM install tests
         run: python -m unittest tests/test_sbom_cmake_install.py
 
+      - name: Run included-sources tests
+        run: python -m unittest tests/test_included_sources.py
+
       - name: Run wolfssl-compliance-tooling-sync tests
         run: python -m unittest tests/test_wolfssl_compliance_tooling_sync.py
 

share/README.md

@@ -13,6 +13,7 @@ into a product at `tools/sbom/`, together with the pin files (`VERSION` and
 | `validate_sbom.py` | Structural validator for CI (`--name-prefix`). |
 | `frontends/compdb_sbom.py` | Extractor for any `compile_commands.json`. |
 | `frontends/iar_sbom.py` | Extractor for an IAR Embedded Workbench `.ewp`. |
+| `frontends/included_sources.py` | Finds the source files a compile command compiles in through `#include` (`compdb_sbom.py --included-sources`). |
 | `frontends/zephyr_sbom.py` | Extractor for a Zephyr module `CMakeLists.txt`. |
 | `build/sbom.mk` | Shared plain-Make fragment and `wolfssl_compliance_tooling_sbom_rule` macro. |
 | `build/sbom.cmake` | Shared CMake helper: `wolfssl_compliance_tooling_add_sbom()`. |

share/frontends/compdb_sbom.py

@@ -13,6 +13,11 @@
   * Renesas e2studio / CCRX (via a build wrapper that records commands)
   * Xilinx SDK / Vitis      (via `bear -- make ...`)
 
+A source a file compiles in through #include, such as wolfSSL's
+src/ssl_api_*.c, is not an entry of its own. `--included-sources` adds those
+by rerunning the selected commands with -M, which needs a gcc- or clang-style
+compiler and runs the compilers the database names.
+
 The extracted sources + defines are handed to the shared driver (sbom-driver),
 so the resulting CycloneDX 1.6 / SPDX 2.3 SBOM is identical in shape to the Make,
 CMake, and IAR paths.
@@ -31,6 +36,8 @@
 import tempfile
 
 SRC_EXTS = ('.c', '.cc', '.cpp', '.cxx', '.s', '.asm')
+# Only these are preprocessed, so only these are rerun with -M.
+SCAN_EXTS = ('.c', '.cc', '.cpp', '.cxx', '.S')
 
 SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
 # The shared driver is the sibling of this frontends/ directory.
@@ -48,6 +55,11 @@ def entry_tokens(entry):
     return []
 
 
+def file_id(path):
+    st = os.stat(path)
+    return st.st_dev, st.st_ino
+
+
 def macro_name(d):
     """Macro name of a -D/-U operand: the text before the first '=' or '(',
     so a function-like 'F(a)=a' is keyed as 'F'."""
@@ -117,6 +129,10 @@ def main():
     ap.add_argument('--spdx-out', default=None)
     ap.add_argument('--srcs-out', default=None)
     ap.add_argument('--print-only', action='store_true')
+    ap.add_argument('--included-sources', action='store_true',
+                    help='Rerun the compile commands with -M and add the '
+                         'source files they #include. Runs the compilers '
+                         'the database names; needs one that supports -M.')
     args = ap.parse_args()
 
     if not os.path.isfile(args.compdb):
@@ -135,7 +151,7 @@ def main():
     # product must not pull an unrelated file (and its path) into the SBOM.
     root_abs = os.path.realpath(args.root) if args.root else None
 
-    srcs, defines, seen = [], set(), set()
+    srcs, defines, seen, selected = [], set(), set(), []
     for entry in db:
         fpath = entry.get('file')
         if not fpath:
@@ -158,18 +174,67 @@ def main():
             continue
         for d in extract_defines(entry_tokens(entry)):
             defines.add(d)
-        if fpath not in seen and os.path.isfile(fpath):
+        if not os.path.isfile(fpath):
+            continue
+        if fpath not in seen:
             seen.add(fpath)
             srcs.append(fpath)
+        ext = os.path.splitext(fpath)[1]
+        if ext in SCAN_EXTS or ext.lower() in SCAN_EXTS:
+            selected.append((fpath, entry))
 
     if not srcs:
         sys.exit("ERROR: no matching source files found in compilation database")
 
+    included = []
+    if args.included_sources:
+        sys.path.insert(0, SCRIPT_DIR)
+        from included_sources import ScanError, included_sources
+        try:
+            scan_root = root_abs or os.path.commonpath(
+                [os.path.dirname(s) for s in srcs])
+        except ValueError:
+            sys.exit("ERROR: sources share no common directory; pass --root")
+        known = {file_id(s) for s in srcs}
+        names = {os.path.basename(s): s for s in srcs}
+        scanned = set()
+        for fpath, entry in selected:
+            tokens = entry_tokens(entry)
+            key = (fpath, entry.get('directory'), tuple(tokens))
+            if key in scanned:
+                continue
+            scanned.add(key)
+            if not tokens:
+                sys.stderr.write(f"WARNING: no compile command for {fpath}; "
+                                 f"its #include'd sources are not scanned\n")
+                continue
+            try:
+                found = included_sources(
+                    tokens, entry.get('directory', os.getcwd()), fpath,
+                    scan_root, SRC_EXTS)
+            except ScanError as e:
+                sys.exit(f"ERROR: {e}")
+            for path in found:
+                if (not os.path.isfile(path) or file_id(path) in known
+                        or (inc and not any(r.search(path) for r in inc))
+                        or (exc and any(r.search(path) for r in exc))):
+                    continue
+                name = os.path.basename(path)
+                if name in names:
+                    sys.exit(f"ERROR: {path}, #include'd by {fpath}, has the "
+                             f"same basename as {names[name]}; --exclude "
+                             f"one of them")
+                known.add(file_id(path))
+                names[name] = path
+                included.append(path)
+        srcs += included
+
     defines = sorted(defines)
     cflags = ' '.join(f'-D{d}' for d in defines)
 
     if args.print_only:
-        print(f"# {len(srcs)} sources, {len(defines)} defines")
+        print(f"# {len(srcs)} sources ({len(included)} included), "
+              f"{len(defines)} defines")
         print("\n[defines]")
         for d in defines:
             print(f"  -D{d}")
@@ -204,7 +269,8 @@ def main():
     for inc in args.include_dir:
         cmd += ['--include-dir', inc]
 
-    print(f"compdb SBOM: {len(srcs)} sources, {len(defines)} defines")
+    print(f"compdb SBOM: {len(srcs)} sources ({len(included)} included), "
+          f"{len(defines)} defines")
     try:
         rc = subprocess.call(cmd)
     finally:

share/frontends/included_sources.py

@@ -0,0 +1,200 @@
+"""Find the source files a compile command compiles in through #include,
+such as wolfSSL's src/ssl_api_*.c, which are not compile_commands.json
+entries of their own."""
+
+import os
+import re
+import shlex
+import subprocess
+
+# Options whose next token is their argument, kept for the -M run.
+_KEEP_PAIRS = ('-D', '-U', '-I', '-iquote', '-isystem', '-isystem-after',
+               '-idirafter', '-imacros', '-include', '-isysroot', '-iprefix',
+               '-iwithprefix', '-iwithprefixbefore', '-iwithsysroot',
+               '-iframework', '-iframeworkwithsysroot', '-F', '-target',
+               '-arch', '--sysroot', '-x', '-specs', '--specs',
+               '--gcc-toolchain', '-gcc-toolchain', '-meabi', '-include-pch')
+# Options whose next token is their argument, dropped with it.
+_DROP_PAIRS = ('-o', '-MF', '-MT', '-MQ', '-MJ', '-Xclang', '-Xpreprocessor',
+               '-Xassembler', '-Xlinker', '-mllvm', '--param', '--output',
+               '-framework', '-undefined', '-force_load', '-filelist',
+               '-multiply_defined', '-mthread-model', '-arch_only', '-mmlir',
+               '-module-dependency-dir', '-multiply_defined_unused',
+               '-target-arch-variant')
+_KEEP_EXACT = ('-ansi', '-undef', '-pthread', '-nostdinc', '-nostdinc++',
+               '-nobuiltininc')
+# Joined forms of the pairs, and flags that set target or language macros.
+_KEEP = tuple(o for o in _KEEP_PAIRS if o != '-arch') + (
+    '--sysroot=', '--target=', '-std=', '--std=', '-stdlib=', '-specs=',
+    '--specs=', '--gcc-toolchain=', '-m', '-O')
+# -f flags that change predefined macros; any other -f may write a file.
+_F_KEEP = ('signed-char', 'unsigned-char', 'short-wchar', 'short-enums',
+           'freestanding', 'hosted', 'builtin', 'pic', 'PIC', 'pie', 'PIE',
+           'stack-protector', 'sanitize=', 'openmp', 'openacc', 'exceptions',
+           'cxx-exceptions', 'rtti', 'ms-extensions', 'ms-compatibility',
+           'declspec', 'blocks', 'gnu89-inline', 'asm', 'wrapv', 'trapv',
+           'char8_t', 'gnu-keywords', 'coroutines', 'fast-math', 'math-errno',
+           'finite-math-only', 'inline', 'cf-protection', 'input-charset=',
+           'exec-charset=', 'dollars-in-identifiers', 'threadsafe-statics',
+           'sized-deallocation', 'aligned-new', 'unwind-tables',
+           'leading-underscore', 'single-precision-constant', 'abi-version=')
+# Preprocessor options worth forwarding from -Xclang and -Xpreprocessor.
+_WRAPPED_PAIRS = ('-D', '-U', '-I', '-include', '-imacros', '-include-pch',
+                  '-isystem', '-iquote', '-idirafter')
+_LAUNCHERS = ('ccache', 'sccache', 'distcc', 'icecc')
+# GNU as takes -M as --mri.
+_ASSEMBLER = re.compile(r'(^|-)as(\.exe)?$')
+
+
+class ScanError(Exception):
+    pass
+
+
+def _wp_option(t):
+    """A -Wp, option without its -M and -o parts, or None if none is left."""
+    parts, kept = t.split(',')[1:], []
+    while parts:
+        p = parts.pop(0)
+        if p in ('-MD', '-MMD', '-MF', '-MT', '-MQ', '-o', '--output') \
+                and parts:
+            parts.pop(0)
+        elif not p.startswith(('-M', '-o', '--output')):
+            kept.append(p)
+    return '-Wp,' + ','.join(kept) if kept else None
+
+
+def _keep_flag(t):
+    """t if it is a single-token flag the -M run needs, else None."""
+    if t.startswith('-Wp,'):
+        return _wp_option(t)
+    if t.startswith('-f'):
+        name = t[5:] if t.startswith('-fno-') else t[2:]
+        return t if name.startswith(_F_KEEP) else None
+    if t in _KEEP_EXACT or (t.startswith(_KEEP) and t not in _DROP_PAIRS):
+        return t
+    return None
+
+
+def _expand_response_files(tokens, directory, depth=0):
+    out = []
+    for t in tokens:
+        if not t.startswith('@') or t == '@':
+            out.append(t)
+            continue
+        path = os.path.join(directory or '', t[1:])
+        if depth > 8:
+            raise ScanError(f"response files nested too deep at {path}")
+        try:
+            with open(path, errors='surrogateescape') as f:
+                inner = shlex.split(f.read())
+        except (OSError, ValueError) as e:
+            raise ScanError(f"cannot read response file {path}: {e}")
+        out += _expand_response_files(inner, directory, depth + 1)
+    return out
+
+
+def _compiler_index(tokens):
+    i = 0
+    while (i < len(tokens) - 1 and os.path.splitext(
+            os.path.basename(tokens[i]))[0] in _LAUNCHERS):
+        i += 1
+    return i
+
+
+def _arches(tokens):
+    return list(dict.fromkeys(tokens[j + 1] for j in range(len(tokens) - 1)
+                              if tokens[j] == '-arch'))
+
+
+def dependency_command(tokens, source, arch=None):
+    """The preprocessor flags of a compile command for one -arch (the
+    first by default), as a -M run on source."""
+    i = _compiler_index(tokens)
+    rest = tokens[i + 1:]
+    arch = arch or next(iter(_arches(rest)), None)
+    # -M goes first so a misparsed option swallows the source and fails.
+    cmd, i = [tokens[i], '-M'], 0
+    while i < len(rest):
+        t = rest[i]
+        pair = i + 1 < len(rest)
+        if t in ('-Xclang', '-Xpreprocessor') and pair:
+            inner = rest[i + 1]
+            if (inner in _WRAPPED_PAIRS and i + 3 < len(rest)
+                    and rest[i + 2] == t):
+                cmd += rest[i:i + 4]
+                i += 4
+                continue
+            if inner.startswith(('-D', '-U', '-I')):
+                cmd += [t, inner]
+            i += 2
+        elif t == f'-Xarch_{arch}' and pair:
+            kept = _keep_flag(rest[i + 1])
+            cmd += [kept] if kept else []
+            i += 2
+        elif (t in _DROP_PAIRS or t.startswith('-Xarch_')) and pair:
+            i += 2
+        elif t in _KEEP_PAIRS and pair:
+            if t != '-arch' or rest[i + 1] == arch:
+                cmd += [t, rest[i + 1]]
+            i += 2
+        else:
+            kept = _keep_flag(t)
+            cmd += [kept] if kept else []
+            i += 1
+    return cmd + [source]
+
+
+def parse_make_deps(text):
+    """Return the prerequisites of the make rules printed by -M."""
+    text = text.replace('\\\r\n', ' ').replace('\\\n', ' ')
+    deps = []
+    for rule in text.splitlines():
+        prereqs = re.split(r':(?:\s|$)', rule, maxsplit=1)
+        if len(prereqs) < 2:
+            continue
+        deps += [d.replace('\\ ', ' ').replace('\\#', '#').replace('$$', '$')
+                 for d in re.split(r'(?<!\\)\s+', prereqs[1].strip()) if d]
+    return deps
+
+
+def included_sources(tokens, directory, source, root, exts):
+    """Return the files under root with an extension in exts that the
+    command for source reads, other than source itself."""
+    if _ASSEMBLER.search(os.path.basename(tokens[_compiler_index(tokens)])):
+        return []
+    tokens = _expand_response_files(tokens, directory)
+    found = []
+    for arch in _arches(tokens) or [None]:
+        cmd = dependency_command(tokens, source, arch)
+        for path in _scan(cmd, directory, source, root, exts):
+            if path not in found:
+                found.append(path)
+    return found
+
+
+def _scan(cmd, directory, source, root, exts):
+    try:
+        r = subprocess.run(cmd, cwd=directory or None, capture_output=True,
+                           text=True, errors='surrogateescape')
+    except OSError as e:
+        raise ScanError(f"cannot run '{cmd[0]}': {e}")
+    if r.returncode:
+        raise ScanError(f"'{cmd[0]} -M' failed for {source}:\n"
+                        f"{r.stderr.strip()}")
+    root, own = os.path.realpath(root), os.path.realpath(source)
+    found, listed = [], False
+    for dep in parse_make_deps(r.stdout):
+        raw = os.path.join(directory or '', dep)
+        real, path = os.path.realpath(raw), os.path.normpath(raw)
+        if os.path.realpath(path) != real:
+            path = real
+        try:
+            inside = os.path.commonpath([real, root]) == root
+        except ValueError:
+            inside = False
+        listed = listed or real == own
+        if inside and real != own and path.lower().endswith(exts):
+            found.append(path)
+    if not listed:
+        raise ScanError(f"'{cmd[0]} -M' printed no dependencies for {source}")
+    return found