fix(compdb): reject conflicting -D values and forward the driver flags

.github/workflows/selftest.yml

@@ -61,6 +61,7 @@ jobs:
             tests/test_low_findings.py \
             tests/test_iar_sbom.py \
             tests/test_included_sources.py \
+            tests/test_compdb_sbom.py \
             tests/test_sbom_cmake_install.py \
             tests/test_wolfssl_compliance_tooling_sync.py \
             tests/test_sbom_autotools.py
@@ -89,6 +90,9 @@ jobs:
       - name: Run included-sources tests
         run: python -m unittest tests/test_included_sources.py
 
+      - name: Run compdb frontend tests
+        run: python -m unittest tests/test_compdb_sbom.py
+
       - name: Run wolfssl-compliance-tooling-sync tests
         run: python -m unittest tests/test_wolfssl_compliance_tooling_sync.py
 

share/frontends/compdb_sbom.py

@@ -66,6 +66,11 @@ def macro_name(d):
     return re.split(r'[=(]', d, maxsplit=1)[0]
 
 
+def define_value(d):
+    """A -D operand with the implicit '=1' of a bare -DNAME made explicit."""
+    return d if '=' in d else d + '=1'
+
+
 def extract_defines(tokens):
     """Net -D macros for one compile command, honouring a later -U that cancels
     an earlier -D of the same name (the compiler applies them in order, so a
@@ -102,8 +107,12 @@ def main():
         formatter_class=argparse.RawDescriptionHelpFormatter)
     ap.add_argument('compdb', help='Path to compile_commands.json')
     ap.add_argument('--name', required=True, help='Package name for the SBOM.')
-    ap.add_argument('--include', action='append', default=[])
-    ap.add_argument('--exclude', action='append', default=[])
+    ap.add_argument('--include', action='append', default=[],
+                    help='Keep sources whose path matches this regex '
+                         '(re.search, repeatable).')
+    ap.add_argument('--exclude', action='append', default=[],
+                    help='Drop sources whose path matches this regex '
+                         '(re.search, repeatable).')
     ap.add_argument('--driver', default=DEFAULT_DRIVER,
                     help='Path to the shared sbom-driver.')
     ap.add_argument('--gen-sbom', default=None)
@@ -113,6 +122,18 @@ def main():
     ap.add_argument('--root', default=None)
     ap.add_argument('--license-file', default=None)
     ap.add_argument('--copyright-start', default=None)
+    ap.add_argument('--license-override', default=None)
+    ap.add_argument('--license-text', default=None)
+    ap.add_argument('--supplier', default=None)
+    ap.add_argument('--component-type', default=None)
+    ap.add_argument('--dep-wolfssl', default=None)
+    ap.add_argument('--dep-wolfcrypt', default=None)
+    ap.add_argument('--dep-openssl', default=None)
+    ap.add_argument('--dep-version', action='append', default=[],
+                    help='Dependency version, e.g. wolfssl=5.9.2 (repeatable).')
+    ap.add_argument('--options-h', default=None,
+                    help='Pre-expanded #define header recorded instead of the '
+                         '-D set from the database.')
     # A compilation database lists the -D tokens the compiler was given, not
     # the macros a settings header derives from them. Without these two the
     # capture reads an empty translation unit and the recorded configuration
@@ -135,6 +156,9 @@ def main():
                          'the database names; needs one that supports -M.')
     args = ap.parse_args()
 
+    if args.options_h and (args.settings_h or args.include_dir):
+        sys.exit("ERROR: --settings-h and --include-dir only feed the -D "
+                 "capture; drop them when passing --options-h.")
     if not os.path.isfile(args.compdb):
         sys.exit(f"ERROR: compilation database not found: {args.compdb}")
     with open(args.compdb) as f:
@@ -151,7 +175,9 @@ def main():
     # product must not pull an unrelated file (and its path) into the SBOM.
     root_abs = os.path.realpath(args.root) if args.root else None
 
-    srcs, defines, seen, selected = [], set(), set(), []
+    srcs, seen, selected = [], set(), []
+    defined = {}     # macro name -> (token, first file that set it)
+    conflicts = {}   # macro name -> {token: example file}
     for entry in db:
         fpath = entry.get('file')
         if not fpath:
@@ -162,18 +188,24 @@ def main():
         fpath = os.path.normpath(fpath)
         if not fpath.lower().endswith(SRC_EXTS):
             continue
+        if inc and not any(r.search(fpath) for r in inc):
+            continue
+        if exc and any(r.search(fpath) for r in exc):
+            continue
         if root_abs:
             real = os.path.realpath(fpath)
             if os.path.commonpath([real, root_abs]) != root_abs:
                 sys.stderr.write(
                     f"WARNING: skipping source outside --root: {fpath}\n")
                 continue
-        if inc and not any(r.search(fpath) for r in inc):
-            continue
-        if exc and any(r.search(fpath) for r in exc):
-            continue
         for d in extract_defines(entry_tokens(entry)):
-            defines.add(d)
+            name = macro_name(d)
+            if name not in defined:
+                defined[name] = (d, fpath)
+            elif define_value(defined[name][0]) != define_value(d):
+                vals = conflicts.setdefault(
+                    name, {defined[name][0]: defined[name][1]})
+                vals.setdefault(d, fpath)
         if not os.path.isfile(fpath):
             continue
         if fpath not in seen:
@@ -186,6 +218,17 @@ def main():
     if not srcs:
         sys.exit("ERROR: no matching source files found in compilation database")
 
+    if conflicts and not args.options_h:
+        lines = ["ERROR: the selected compile commands disagree on these "
+                 "macros:"]
+        for name in sorted(conflicts):
+            lines.append(f"  {name}:")
+            for tok, src in sorted(conflicts[name].items()):
+                lines.append(f"    -D{tok}  (e.g. {src})")
+        lines.append("Narrow the set with --include/--exclude so one SBOM "
+                     "records one configuration.")
+        sys.exit("\n".join(lines))
+
     included = []
     if args.included_sources:
         sys.path.insert(0, SCRIPT_DIR)
@@ -229,7 +272,7 @@ def main():
                 included.append(path)
         srcs += included
 
-    defines = sorted(defines)
+    defines = sorted(tok for tok, _src in defined.values())
     cflags = ' '.join(f'-D{d}' for d in defines)
 
     if args.print_only:
@@ -251,14 +294,23 @@ def main():
     with open(srcs_out, 'w') as f:
         f.write('\n'.join(srcs) + '\n')
 
-    cmd = [args.driver, '--srcs-file', srcs_out, f'--cflags={cflags}',
+    config = (['--options-h', args.options_h] if args.options_h
+              else [f'--cflags={cflags}'])
+    cmd = [args.driver, '--srcs-file', srcs_out, *config,
            '--name', args.name]
     for flag, val in (('--version', args.version),
                       ('--version-file', args.version_file),
                       ('--version-macro', args.version_macro),
                       ('--root', args.root),
                       ('--license-file', args.license_file),
                       ('--copyright-start', args.copyright_start),
+                      ('--license-override', args.license_override),
+                      ('--license-text', args.license_text),
+                      ('--supplier', args.supplier),
+                      ('--component-type', args.component_type),
+                      ('--dep-wolfssl', args.dep_wolfssl),
+                      ('--dep-wolfcrypt', args.dep_wolfcrypt),
+                      ('--dep-openssl', args.dep_openssl),
                       ('--settings-h', args.settings_h),
                       ('--crypto-only', args.crypto_only),
                       ('--gen-sbom', args.gen_sbom),
@@ -268,6 +320,8 @@ def main():
             cmd += [flag, val]
     for inc in args.include_dir:
         cmd += ['--include-dir', inc]
+    for dv in args.dep_version:
+        cmd += ['--dep-version', dv]
 
     print(f"compdb SBOM: {len(srcs)} sources ({len(included)} included), "
           f"{len(defines)} defines")

share/sbom-driver.py

@@ -435,7 +435,9 @@ def main():
                 with open(args.options_h, encoding="utf-8", errors="replace") as f:
                     defines_text = f.read()
             elif args.cflags:
-                if args.settings_h and not os.path.isfile(args.settings_h):
+                # A relative path is left to the compiler's -include search.
+                if (os.path.isabs(args.settings_h)
+                        and not os.path.isfile(args.settings_h)):
                     sys.exit(f"ERROR: --settings-h '{args.settings_h}' "
                              f"does not exist.")
                 defines_text = capture_macros(

tests/test_compdb_sbom.py

@@ -0,0 +1,184 @@
+#!/usr/bin/env python3
+"""Tests for the compilation-database frontend (share/frontends/compdb_sbom.py)."""
+
+import json
+import os
+import subprocess
+import sys
+import tempfile
+import unittest
+
+REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+COMPDB = os.path.join(REPO, "share", "frontends", "compdb_sbom.py")
+
+# Stands in for sbom-driver and writes its argv, one per line, to STUB_ARGV.
+_STUB = """import os, sys
+with open(os.environ['STUB_ARGV'], 'w') as f:
+    f.write('\\n'.join(sys.argv[1:]))
+"""
+
+
+class CompdbCase(unittest.TestCase):
+    def setUp(self):
+        self.tmp = tempfile.TemporaryDirectory()
+        self.d = self.tmp.name
+        self.db = os.path.join(self.d, "compile_commands.json")
+        self.stub = os.path.join(self.d, "stub-driver")
+        with open(self.stub, "w") as f:
+            f.write("#!" + sys.executable + "\n" + _STUB)
+        os.chmod(self.stub, 0o755)
+        self.argv_path = os.path.join(self.d, "argv.txt")
+
+    def tearDown(self):
+        self.tmp.cleanup()
+
+    def write_db(self, entries):
+        db = []
+        for name, flags in entries:
+            src = os.path.join(self.d, name)
+            open(src, "w").close()
+            db.append({"directory": self.d, "file": name,
+                       "arguments": ["cc", *flags, "-c", name]})
+        with open(self.db, "w") as f:
+            json.dump(db, f)
+
+    def run_compdb(self, *extra):
+        return subprocess.run(
+            [sys.executable, COMPDB, self.db, "--name", "t",
+             "--driver", self.stub, *extra],
+            capture_output=True, text=True,
+            env=dict(os.environ, STUB_ARGV=self.argv_path))
+
+    def driver_argv(self):
+        with open(self.argv_path) as f:
+            return f.read().splitlines()
+
+
+class TestConflictingDefines(CompdbCase):
+    def test_different_values_fail(self):
+        self.write_db([("a.c", ["-DFOO=10"]), ("b.c", ["-DFOO=2"])])
+        r = self.run_compdb("--print-only")
+        self.assertNotEqual(r.returncode, 0)
+        self.assertIn("FOO", r.stderr)
+        self.assertIn("-DFOO=10", r.stderr)
+        self.assertIn("-DFOO=2", r.stderr)
+        self.assertIn("a.c", r.stderr)
+        self.assertIn("b.c", r.stderr)
+        self.assertIn("--include/--exclude", r.stderr)
+
+    def test_bare_define_equals_value_one(self):
+        self.write_db([("a.c", ["-DFOO"]), ("b.c", ["-DFOO=1"])])
+        r = self.run_compdb("--print-only")
+        self.assertEqual(r.returncode, 0, r.stderr)
+
+    def test_empty_value_conflicts_with_bare_define(self):
+        self.write_db([("a.c", ["-DFOO"]), ("b.c", ["-DFOO="])])
+        r = self.run_compdb("--print-only")
+        self.assertNotEqual(r.returncode, 0)
+        self.assertIn("FOO", r.stderr)
+
+    def test_options_h_ignores_conflicting_defines(self):
+        self.write_db([("a.c", ["-DUNIT_ID=1"]), ("b.c", ["-DUNIT_ID=2"])])
+        r = self.run_compdb("--options-h", "options.h")
+        self.assertEqual(r.returncode, 0, r.stderr)
+        self.assertIn("--options-h", self.driver_argv())
+
+    def test_exclude_resolves_conflict(self):
+        self.write_db([("a.c", ["-DFOO=10"]), ("b.c", ["-DFOO=2"])])
+        r = self.run_compdb("--exclude", r"b\.c$", "--print-only")
+        self.assertEqual(r.returncode, 0, r.stderr)
+        self.assertIn("-DFOO=10", r.stdout)
+        self.assertNotIn("-DFOO=2", r.stdout)
+
+    def test_undef_in_one_entry_is_not_a_conflict(self):
+        self.write_db([("a.c", ["-DFOO=1"]), ("b.c", ["-DFOO=1", "-UFOO"])])
+        r = self.run_compdb("--print-only")
+        self.assertEqual(r.returncode, 0, r.stderr)
+        self.assertIn("-DFOO=1", r.stdout)
+
+    def test_partial_defines_are_merged(self):
+        self.write_db([("a.c", ["-DFOO=1", "-DONLY_A"]),
+                       ("b.c", ["-DFOO=1", "-DONLY_B=2"])])
+        r = self.run_compdb("--print-only")
+        self.assertEqual(r.returncode, 0, r.stderr)
+        self.assertIn("# 2 sources (0 included), 3 defines", r.stdout)
+        for tok in ("-DFOO=1", "-DONLY_A", "-DONLY_B=2"):
+            self.assertIn(tok, r.stdout)
+
+
+class TestForwarding(CompdbCase):
+    def test_identity_and_dependency_flags_reach_driver(self):
+        self.write_db([("a.c", ["-DFOO=1"])])
+        r = self.run_compdb(
+            "--license-override", "GPL-3.0-or-later",
+            "--license-text", "/lic.txt",
+            "--supplier", "Example Inc.",
+            "--component-type", "firmware",
+            "--dep-wolfssl", "yes", "--dep-wolfcrypt", "yes",
+            "--dep-openssl", "no",
+            "--dep-version", "wolfssl=5.9.2",
+            "--dep-version", "wolfcrypt=5.9.2")
+        self.assertEqual(r.returncode, 0, r.stderr)
+        argv = self.driver_argv()
+        pairs = list(zip(argv, argv[1:]))
+        for pair in (("--license-override", "GPL-3.0-or-later"),
+                     ("--license-text", "/lic.txt"),
+                     ("--supplier", "Example Inc."),
+                     ("--component-type", "firmware"),
+                     ("--dep-wolfssl", "yes"),
+                     ("--dep-wolfcrypt", "yes"),
+                     ("--dep-openssl", "no"),
+                     ("--dep-version", "wolfssl=5.9.2"),
+                     ("--dep-version", "wolfcrypt=5.9.2")):
+            self.assertIn(pair, pairs)
+        self.assertIn("--cflags=-DFOO=1", argv)
+
+    def test_options_h_replaces_cflags(self):
+        self.write_db([("a.c", ["-DFOO=1"])])
+        opts = os.path.join(self.d, "options.h")
+        open(opts, "w").close()
+        r = self.run_compdb("--options-h", opts)
+        self.assertEqual(r.returncode, 0, r.stderr)
+        argv = self.driver_argv()
+        self.assertIn(("--options-h", opts), list(zip(argv, argv[1:])))
+        self.assertFalse(any(a.startswith("--cflags") for a in argv), argv)
+
+    def test_options_h_with_capture_inputs_fails(self):
+        self.write_db([("a.c", ["-DFOO=1"])])
+        for extra in (["--settings-h", "settings.h"], ["--include-dir", "."]):
+            r = self.run_compdb("--options-h", "options.h", *extra)
+            self.assertNotEqual(r.returncode, 0)
+            self.assertIn("--options-h", r.stderr)
+            self.assertFalse(os.path.exists(self.argv_path))
+
+
+class TestRootFilter(CompdbCase):
+    def write_outside(self):
+        root = os.path.join(self.d, "mod")
+        os.mkdir(root)
+        open(os.path.join(root, "in.c"), "w").close()
+        open(os.path.join(self.d, "app.c"), "w").close()
+        with open(self.db, "w") as f:
+            json.dump([{"directory": self.d, "file": n,
+                        "arguments": ["cc", "-c", n]}
+                       for n in ("mod/in.c", "app.c")], f)
+        return root
+
+    def test_filtered_out_source_is_not_warned(self):
+        root = self.write_outside()
+        r = self.run_compdb("--root", root, "--include", "/mod/",
+                            "--print-only")
+        self.assertEqual(r.returncode, 0, r.stderr)
+        self.assertNotIn("outside --root", r.stderr)
+        self.assertIn("in.c", r.stdout)
+
+    def test_selected_source_outside_root_is_warned(self):
+        root = self.write_outside()
+        r = self.run_compdb("--root", root, "--print-only")
+        self.assertEqual(r.returncode, 0, r.stderr)
+        self.assertIn("outside --root", r.stderr)
+        self.assertNotIn("app.c", r.stdout)
+
+
+if __name__ == "__main__":
+    unittest.main(verbosity=2)