From 1429870c06f1065124f8d9795b25afbb87cbdcbd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Frauenschl=C3=A4ger?= Date: Mon, 19 Oct 2026 19:21:32 +0200 Subject: [PATCH 1/2] fix(compdb): reject conflicting -D values and forward the driver flags compdb_sbom.py merged the -D tokens of every selected compile command into one set. When two entries defined the same macro with different values (-DFOO=10 in one, -DFOO=2 in another), both tokens reached the capture and whichever the compiler saw last won, so the SBOM recorded a configuration that matched neither build. The frontend now keys the merged set by macro name and exits with the macro, each value and an example file when the selected entries disagree, pointing at --include/--exclude to split the database into one configuration per SBOM. A bare -DFOO counts as -DFOO=1, as the compiler defines it. A macro that only some entries define still merges as before, since per-file -D flags are normal. With --options-h the -D tokens are not captured, so their conflicts are not checked. The frontend also dropped flags the driver supports, so a compdb SBOM fell back to the inferred licence, listed no wolfSSL dependency and could not use a pre-expanded header. It now forwards --license-override, --license-text, --supplier, --component-type, --dep-wolfssl, --dep-wolfcrypt, --dep-openssl and --dep-version, and accepts --options-h in place of the -D capture. --options-h with --settings-h or --include-dir is an error, since those only feed the capture. tests/test_compdb_sbom.py covers the conflict, the merge, --exclude as the way out, and the forwarding through a stub driver. It is wired into selftest.yml. --- .github/workflows/selftest.yml | 4 + share/frontends/compdb_sbom.py | 58 +++++++++++- tests/test_compdb_sbom.py | 156 +++++++++++++++++++++++++++++++++ 3 files changed, 214 insertions(+), 4 deletions(-) create mode 100644 tests/test_compdb_sbom.py diff --git a/.github/workflows/selftest.yml b/.github/workflows/selftest.yml index 5980f10..e646523 100644 --- a/.github/workflows/selftest.yml +++ b/.github/workflows/selftest.yml @@ -61,6 +61,7 @@ jobs: tests/test_low_findings.py \ tests/test_iar_sbom.py \ tests/test_included_sources.py \ + tests/test_compdb_sbom.py \ tests/test_sbom_cmake_install.py \ tests/test_wolfssl_compliance_tooling_sync.py \ tests/test_sbom_autotools.py @@ -89,6 +90,9 @@ jobs: - name: Run included-sources tests run: python -m unittest tests/test_included_sources.py + - name: Run compdb frontend tests + run: python -m unittest tests/test_compdb_sbom.py + - name: Run wolfssl-compliance-tooling-sync tests run: python -m unittest tests/test_wolfssl_compliance_tooling_sync.py diff --git a/share/frontends/compdb_sbom.py b/share/frontends/compdb_sbom.py index b4548c6..3c4df35 100755 --- a/share/frontends/compdb_sbom.py +++ b/share/frontends/compdb_sbom.py @@ -66,6 +66,11 @@ def macro_name(d): return re.split(r'[=(]', d, maxsplit=1)[0] +def define_value(d): + """A -D operand with the implicit '=1' of a bare -DNAME made explicit.""" + return d if '=' in d else d + '=1' + + def extract_defines(tokens): """Net -D macros for one compile command, honouring a later -U that cancels an earlier -D of the same name (the compiler applies them in order, so a @@ -113,6 +118,18 @@ def main(): ap.add_argument('--root', default=None) ap.add_argument('--license-file', default=None) ap.add_argument('--copyright-start', default=None) + ap.add_argument('--license-override', default=None) + ap.add_argument('--license-text', default=None) + ap.add_argument('--supplier', default=None) + ap.add_argument('--component-type', default=None) + ap.add_argument('--dep-wolfssl', default=None) + ap.add_argument('--dep-wolfcrypt', default=None) + ap.add_argument('--dep-openssl', default=None) + ap.add_argument('--dep-version', action='append', default=[], + help='Dependency version, e.g. wolfssl=5.9.2 (repeatable).') + ap.add_argument('--options-h', default=None, + help='Pre-expanded #define header recorded instead of the ' + '-D set from the database.') # A compilation database lists the -D tokens the compiler was given, not # the macros a settings header derives from them. Without these two the # capture reads an empty translation unit and the recorded configuration @@ -135,6 +152,9 @@ def main(): 'the database names; needs one that supports -M.') args = ap.parse_args() + if args.options_h and (args.settings_h or args.include_dir): + sys.exit("ERROR: --settings-h and --include-dir only feed the -D " + "capture; drop them when passing --options-h.") if not os.path.isfile(args.compdb): sys.exit(f"ERROR: compilation database not found: {args.compdb}") with open(args.compdb) as f: @@ -151,7 +171,9 @@ def main(): # product must not pull an unrelated file (and its path) into the SBOM. root_abs = os.path.realpath(args.root) if args.root else None - srcs, defines, seen, selected = [], set(), set(), [] + srcs, seen, selected = [], set(), [] + defined = {} # macro name -> (token, first file that set it) + conflicts = {} # macro name -> {token: example file} for entry in db: fpath = entry.get('file') if not fpath: @@ -173,7 +195,13 @@ def main(): if exc and any(r.search(fpath) for r in exc): continue for d in extract_defines(entry_tokens(entry)): - defines.add(d) + name = macro_name(d) + if name not in defined: + defined[name] = (d, fpath) + elif define_value(defined[name][0]) != define_value(d): + vals = conflicts.setdefault( + name, {defined[name][0]: defined[name][1]}) + vals.setdefault(d, fpath) if not os.path.isfile(fpath): continue if fpath not in seen: @@ -186,6 +214,17 @@ def main(): if not srcs: sys.exit("ERROR: no matching source files found in compilation database") + if conflicts and not args.options_h: + lines = ["ERROR: the selected compile commands disagree on these " + "macros:"] + for name in sorted(conflicts): + lines.append(f" {name}:") + for tok, src in sorted(conflicts[name].items()): + lines.append(f" -D{tok} (e.g. {src})") + lines.append("Narrow the set with --include/--exclude so one SBOM " + "records one configuration.") + sys.exit("\n".join(lines)) + included = [] if args.included_sources: sys.path.insert(0, SCRIPT_DIR) @@ -229,7 +268,7 @@ def main(): included.append(path) srcs += included - defines = sorted(defines) + defines = sorted(tok for tok, _src in defined.values()) cflags = ' '.join(f'-D{d}' for d in defines) if args.print_only: @@ -251,7 +290,9 @@ def main(): with open(srcs_out, 'w') as f: f.write('\n'.join(srcs) + '\n') - cmd = [args.driver, '--srcs-file', srcs_out, f'--cflags={cflags}', + config = (['--options-h', args.options_h] if args.options_h + else [f'--cflags={cflags}']) + cmd = [args.driver, '--srcs-file', srcs_out, *config, '--name', args.name] for flag, val in (('--version', args.version), ('--version-file', args.version_file), @@ -259,6 +300,13 @@ def main(): ('--root', args.root), ('--license-file', args.license_file), ('--copyright-start', args.copyright_start), + ('--license-override', args.license_override), + ('--license-text', args.license_text), + ('--supplier', args.supplier), + ('--component-type', args.component_type), + ('--dep-wolfssl', args.dep_wolfssl), + ('--dep-wolfcrypt', args.dep_wolfcrypt), + ('--dep-openssl', args.dep_openssl), ('--settings-h', args.settings_h), ('--crypto-only', args.crypto_only), ('--gen-sbom', args.gen_sbom), @@ -268,6 +316,8 @@ def main(): cmd += [flag, val] for inc in args.include_dir: cmd += ['--include-dir', inc] + for dv in args.dep_version: + cmd += ['--dep-version', dv] print(f"compdb SBOM: {len(srcs)} sources ({len(included)} included), " f"{len(defines)} defines") diff --git a/tests/test_compdb_sbom.py b/tests/test_compdb_sbom.py new file mode 100644 index 0000000..6c6ea4f --- /dev/null +++ b/tests/test_compdb_sbom.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python3 +"""Tests for the compilation-database frontend (share/frontends/compdb_sbom.py).""" + +import json +import os +import subprocess +import sys +import tempfile +import unittest + +REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +COMPDB = os.path.join(REPO, "share", "frontends", "compdb_sbom.py") + +# Stands in for sbom-driver and writes its argv, one per line, to STUB_ARGV. +_STUB = """import os, sys +with open(os.environ['STUB_ARGV'], 'w') as f: + f.write('\\n'.join(sys.argv[1:])) +""" + + +class CompdbCase(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.d = self.tmp.name + self.db = os.path.join(self.d, "compile_commands.json") + self.stub = os.path.join(self.d, "stub-driver") + with open(self.stub, "w") as f: + f.write("#!" + sys.executable + "\n" + _STUB) + os.chmod(self.stub, 0o755) + self.argv_path = os.path.join(self.d, "argv.txt") + + def tearDown(self): + self.tmp.cleanup() + + def write_db(self, entries): + db = [] + for name, flags in entries: + src = os.path.join(self.d, name) + open(src, "w").close() + db.append({"directory": self.d, "file": name, + "arguments": ["cc", *flags, "-c", name]}) + with open(self.db, "w") as f: + json.dump(db, f) + + def run_compdb(self, *extra): + return subprocess.run( + [sys.executable, COMPDB, self.db, "--name", "t", + "--driver", self.stub, *extra], + capture_output=True, text=True, + env=dict(os.environ, STUB_ARGV=self.argv_path)) + + def driver_argv(self): + with open(self.argv_path) as f: + return f.read().splitlines() + + +class TestConflictingDefines(CompdbCase): + def test_different_values_fail(self): + self.write_db([("a.c", ["-DFOO=10"]), ("b.c", ["-DFOO=2"])]) + r = self.run_compdb("--print-only") + self.assertNotEqual(r.returncode, 0) + self.assertIn("FOO", r.stderr) + self.assertIn("-DFOO=10", r.stderr) + self.assertIn("-DFOO=2", r.stderr) + self.assertIn("a.c", r.stderr) + self.assertIn("b.c", r.stderr) + self.assertIn("--include/--exclude", r.stderr) + + def test_bare_define_equals_value_one(self): + self.write_db([("a.c", ["-DFOO"]), ("b.c", ["-DFOO=1"])]) + r = self.run_compdb("--print-only") + self.assertEqual(r.returncode, 0, r.stderr) + + def test_empty_value_conflicts_with_bare_define(self): + self.write_db([("a.c", ["-DFOO"]), ("b.c", ["-DFOO="])]) + r = self.run_compdb("--print-only") + self.assertNotEqual(r.returncode, 0) + self.assertIn("FOO", r.stderr) + + def test_options_h_ignores_conflicting_defines(self): + self.write_db([("a.c", ["-DUNIT_ID=1"]), ("b.c", ["-DUNIT_ID=2"])]) + r = self.run_compdb("--options-h", "options.h") + self.assertEqual(r.returncode, 0, r.stderr) + self.assertIn("--options-h", self.driver_argv()) + + def test_exclude_resolves_conflict(self): + self.write_db([("a.c", ["-DFOO=10"]), ("b.c", ["-DFOO=2"])]) + r = self.run_compdb("--exclude", r"b\.c$", "--print-only") + self.assertEqual(r.returncode, 0, r.stderr) + self.assertIn("-DFOO=10", r.stdout) + self.assertNotIn("-DFOO=2", r.stdout) + + def test_undef_in_one_entry_is_not_a_conflict(self): + self.write_db([("a.c", ["-DFOO=1"]), ("b.c", ["-DFOO=1", "-UFOO"])]) + r = self.run_compdb("--print-only") + self.assertEqual(r.returncode, 0, r.stderr) + self.assertIn("-DFOO=1", r.stdout) + + def test_partial_defines_are_merged(self): + self.write_db([("a.c", ["-DFOO=1", "-DONLY_A"]), + ("b.c", ["-DFOO=1", "-DONLY_B=2"])]) + r = self.run_compdb("--print-only") + self.assertEqual(r.returncode, 0, r.stderr) + self.assertIn("# 2 sources (0 included), 3 defines", r.stdout) + for tok in ("-DFOO=1", "-DONLY_A", "-DONLY_B=2"): + self.assertIn(tok, r.stdout) + + +class TestForwarding(CompdbCase): + def test_identity_and_dependency_flags_reach_driver(self): + self.write_db([("a.c", ["-DFOO=1"])]) + r = self.run_compdb( + "--license-override", "GPL-3.0-or-later", + "--license-text", "/lic.txt", + "--supplier", "Example Inc.", + "--component-type", "firmware", + "--dep-wolfssl", "yes", "--dep-wolfcrypt", "yes", + "--dep-openssl", "no", + "--dep-version", "wolfssl=5.9.2", + "--dep-version", "wolfcrypt=5.9.2") + self.assertEqual(r.returncode, 0, r.stderr) + argv = self.driver_argv() + pairs = list(zip(argv, argv[1:])) + for pair in (("--license-override", "GPL-3.0-or-later"), + ("--license-text", "/lic.txt"), + ("--supplier", "Example Inc."), + ("--component-type", "firmware"), + ("--dep-wolfssl", "yes"), + ("--dep-wolfcrypt", "yes"), + ("--dep-openssl", "no"), + ("--dep-version", "wolfssl=5.9.2"), + ("--dep-version", "wolfcrypt=5.9.2")): + self.assertIn(pair, pairs) + self.assertIn("--cflags=-DFOO=1", argv) + + def test_options_h_replaces_cflags(self): + self.write_db([("a.c", ["-DFOO=1"])]) + opts = os.path.join(self.d, "options.h") + open(opts, "w").close() + r = self.run_compdb("--options-h", opts) + self.assertEqual(r.returncode, 0, r.stderr) + argv = self.driver_argv() + self.assertIn(("--options-h", opts), list(zip(argv, argv[1:]))) + self.assertFalse(any(a.startswith("--cflags") for a in argv), argv) + + def test_options_h_with_capture_inputs_fails(self): + self.write_db([("a.c", ["-DFOO=1"])]) + for extra in (["--settings-h", "settings.h"], ["--include-dir", "."]): + r = self.run_compdb("--options-h", "options.h", *extra) + self.assertNotEqual(r.returncode, 0) + self.assertIn("--options-h", r.stderr) + self.assertFalse(os.path.exists(self.argv_path)) + + +if __name__ == "__main__": + unittest.main(verbosity=2) From 58a93766c448598f12509a06f619e229d83d1af5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Frauenschl=C3=A4ger?= Date: Tue, 20 Oct 2026 09:27:55 +0200 Subject: [PATCH 2/2] fix(compdb): filter before the --root check, resolve --settings-h like -include A Zephyr build's compile_commands.json lists every kernel, driver and application file next to the module's sources. compdb_sbom.py checked --root before --include/--exclude, so each of those files printed a "skipping source outside --root" warning even when the include filter would have dropped it anyway: about 200 lines for wolfssl and 285 for wolfCert. The filters now run first, and only a selected source outside --root is warned about. The driver also refused a relative --settings-h unless it existed under the current directory, although the compdb help gives wolfssl/wolfcrypt/settings.h as the example. It now checks only an absolute path and leaves a relative one to the compiler's -include search, which tries the current directory, then the -I dirs in --cflags, then each --include-dir. --include and --exclude gain help text saying they are re.search patterns, not globs. --- share/frontends/compdb_sbom.py | 16 ++++++++++------ share/sbom-driver.py | 4 +++- tests/test_compdb_sbom.py | 28 ++++++++++++++++++++++++++++ tests/test_sbom.py | 25 +++++++++++++++++++++++++ 4 files changed, 66 insertions(+), 7 deletions(-) diff --git a/share/frontends/compdb_sbom.py b/share/frontends/compdb_sbom.py index 3c4df35..11eaddf 100755 --- a/share/frontends/compdb_sbom.py +++ b/share/frontends/compdb_sbom.py @@ -107,8 +107,12 @@ def main(): formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument('compdb', help='Path to compile_commands.json') ap.add_argument('--name', required=True, help='Package name for the SBOM.') - ap.add_argument('--include', action='append', default=[]) - ap.add_argument('--exclude', action='append', default=[]) + ap.add_argument('--include', action='append', default=[], + help='Keep sources whose path matches this regex ' + '(re.search, repeatable).') + ap.add_argument('--exclude', action='append', default=[], + help='Drop sources whose path matches this regex ' + '(re.search, repeatable).') ap.add_argument('--driver', default=DEFAULT_DRIVER, help='Path to the shared sbom-driver.') ap.add_argument('--gen-sbom', default=None) @@ -184,16 +188,16 @@ def main(): fpath = os.path.normpath(fpath) if not fpath.lower().endswith(SRC_EXTS): continue + if inc and not any(r.search(fpath) for r in inc): + continue + if exc and any(r.search(fpath) for r in exc): + continue if root_abs: real = os.path.realpath(fpath) if os.path.commonpath([real, root_abs]) != root_abs: sys.stderr.write( f"WARNING: skipping source outside --root: {fpath}\n") continue - if inc and not any(r.search(fpath) for r in inc): - continue - if exc and any(r.search(fpath) for r in exc): - continue for d in extract_defines(entry_tokens(entry)): name = macro_name(d) if name not in defined: diff --git a/share/sbom-driver.py b/share/sbom-driver.py index 44d12ad..8837615 100755 --- a/share/sbom-driver.py +++ b/share/sbom-driver.py @@ -435,7 +435,9 @@ def main(): with open(args.options_h, encoding="utf-8", errors="replace") as f: defines_text = f.read() elif args.cflags: - if args.settings_h and not os.path.isfile(args.settings_h): + # A relative path is left to the compiler's -include search. + if (os.path.isabs(args.settings_h) + and not os.path.isfile(args.settings_h)): sys.exit(f"ERROR: --settings-h '{args.settings_h}' " f"does not exist.") defines_text = capture_macros( diff --git a/tests/test_compdb_sbom.py b/tests/test_compdb_sbom.py index 6c6ea4f..702f75a 100644 --- a/tests/test_compdb_sbom.py +++ b/tests/test_compdb_sbom.py @@ -152,5 +152,33 @@ def test_options_h_with_capture_inputs_fails(self): self.assertFalse(os.path.exists(self.argv_path)) +class TestRootFilter(CompdbCase): + def write_outside(self): + root = os.path.join(self.d, "mod") + os.mkdir(root) + open(os.path.join(root, "in.c"), "w").close() + open(os.path.join(self.d, "app.c"), "w").close() + with open(self.db, "w") as f: + json.dump([{"directory": self.d, "file": n, + "arguments": ["cc", "-c", n]} + for n in ("mod/in.c", "app.c")], f) + return root + + def test_filtered_out_source_is_not_warned(self): + root = self.write_outside() + r = self.run_compdb("--root", root, "--include", "/mod/", + "--print-only") + self.assertEqual(r.returncode, 0, r.stderr) + self.assertNotIn("outside --root", r.stderr) + self.assertIn("in.c", r.stdout) + + def test_selected_source_outside_root_is_warned(self): + root = self.write_outside() + r = self.run_compdb("--root", root, "--print-only") + self.assertEqual(r.returncode, 0, r.stderr) + self.assertIn("outside --root", r.stderr) + self.assertNotIn("app.c", r.stdout) + + if __name__ == "__main__": unittest.main(verbosity=2) diff --git a/tests/test_sbom.py b/tests/test_sbom.py index b202c09..8ac9285 100755 --- a/tests/test_sbom.py +++ b/tests/test_sbom.py @@ -353,6 +353,31 @@ def integration_tests(gen_sbom): "--cdx-out", "l.cdx.json", "--spdx-out", "l.spdx.json"]) check(r.returncode == 0, "library generate succeeds") + # A relative --settings-h resolves like -include: cwd, then -I in + # CFLAGS, then --include-dir. + for sub, macro in (("a", "PICK_A"), ("b", "PICK_B")): + os.makedirs(os.path.join(d, sub, "cfg")) + with open(os.path.join(d, sub, "cfg", "s.h"), "w") as f: + f.write(f"#ifdef SEL\n#define {macro} 1\n#endif\n") + for extra, want, label in ( + (["--include-dir", os.path.join(d, "b")], "PICK_B", + "relative --settings-h found under --include-dir"), + ([f"--cflags=-DSEL -I{os.path.join(d, 'a')}", + "--include-dir", os.path.join(d, "b")], "PICK_A", + "relative --settings-h prefers the CFLAGS -I")): + if not any(e.startswith("--cflags") for e in extra): + extra = ["--cflags=-DSEL"] + extra + r = run_driver(d, gen_sbom, [ + "--srcs-file", srcs, "--settings-h", "cfg/s.h", *extra, + "--cdx-out", "s.cdx.json", "--spdx-out", "s.spdx.json"]) + text = open(os.path.join(d, "s.cdx.json")).read() \ + if r.returncode == 0 else r.stderr + check(r.returncode == 0 and want in text, label) + r = run_driver(d, gen_sbom, [ + "--srcs-file", srcs, "--cflags=-DSEL", "--settings-h", "cfg/s.h", + "--cdx-out", "s.cdx.json", "--spdx-out", "s.spdx.json"]) + check(r.returncode != 0, "unresolvable relative --settings-h fails") + # Reproducibility: same inputs + SOURCE_DATE_EPOCH => byte-identical. env = dict(os.environ, SOURCE_DATE_EPOCH="1700000000") outs = []