diff --git a/.github/workflows/selftest.yml b/.github/workflows/selftest.yml index 5980f10..e646523 100644 --- a/.github/workflows/selftest.yml +++ b/.github/workflows/selftest.yml @@ -61,6 +61,7 @@ jobs: tests/test_low_findings.py \ tests/test_iar_sbom.py \ tests/test_included_sources.py \ + tests/test_compdb_sbom.py \ tests/test_sbom_cmake_install.py \ tests/test_wolfssl_compliance_tooling_sync.py \ tests/test_sbom_autotools.py @@ -89,6 +90,9 @@ jobs: - name: Run included-sources tests run: python -m unittest tests/test_included_sources.py + - name: Run compdb frontend tests + run: python -m unittest tests/test_compdb_sbom.py + - name: Run wolfssl-compliance-tooling-sync tests run: python -m unittest tests/test_wolfssl_compliance_tooling_sync.py diff --git a/share/frontends/compdb_sbom.py b/share/frontends/compdb_sbom.py index b4548c6..11eaddf 100755 --- a/share/frontends/compdb_sbom.py +++ b/share/frontends/compdb_sbom.py @@ -66,6 +66,11 @@ def macro_name(d): return re.split(r'[=(]', d, maxsplit=1)[0] +def define_value(d): + """A -D operand with the implicit '=1' of a bare -DNAME made explicit.""" + return d if '=' in d else d + '=1' + + def extract_defines(tokens): """Net -D macros for one compile command, honouring a later -U that cancels an earlier -D of the same name (the compiler applies them in order, so a @@ -102,8 +107,12 @@ def main(): formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument('compdb', help='Path to compile_commands.json') ap.add_argument('--name', required=True, help='Package name for the SBOM.') - ap.add_argument('--include', action='append', default=[]) - ap.add_argument('--exclude', action='append', default=[]) + ap.add_argument('--include', action='append', default=[], + help='Keep sources whose path matches this regex ' + '(re.search, repeatable).') + ap.add_argument('--exclude', action='append', default=[], + help='Drop sources whose path matches this regex ' + '(re.search, repeatable).') ap.add_argument('--driver', default=DEFAULT_DRIVER, help='Path to the shared sbom-driver.') ap.add_argument('--gen-sbom', default=None) @@ -113,6 +122,18 @@ def main(): ap.add_argument('--root', default=None) ap.add_argument('--license-file', default=None) ap.add_argument('--copyright-start', default=None) + ap.add_argument('--license-override', default=None) + ap.add_argument('--license-text', default=None) + ap.add_argument('--supplier', default=None) + ap.add_argument('--component-type', default=None) + ap.add_argument('--dep-wolfssl', default=None) + ap.add_argument('--dep-wolfcrypt', default=None) + ap.add_argument('--dep-openssl', default=None) + ap.add_argument('--dep-version', action='append', default=[], + help='Dependency version, e.g. wolfssl=5.9.2 (repeatable).') + ap.add_argument('--options-h', default=None, + help='Pre-expanded #define header recorded instead of the ' + '-D set from the database.') # A compilation database lists the -D tokens the compiler was given, not # the macros a settings header derives from them. Without these two the # capture reads an empty translation unit and the recorded configuration @@ -135,6 +156,9 @@ def main(): 'the database names; needs one that supports -M.') args = ap.parse_args() + if args.options_h and (args.settings_h or args.include_dir): + sys.exit("ERROR: --settings-h and --include-dir only feed the -D " + "capture; drop them when passing --options-h.") if not os.path.isfile(args.compdb): sys.exit(f"ERROR: compilation database not found: {args.compdb}") with open(args.compdb) as f: @@ -151,7 +175,9 @@ def main(): # product must not pull an unrelated file (and its path) into the SBOM. root_abs = os.path.realpath(args.root) if args.root else None - srcs, defines, seen, selected = [], set(), set(), [] + srcs, seen, selected = [], set(), [] + defined = {} # macro name -> (token, first file that set it) + conflicts = {} # macro name -> {token: example file} for entry in db: fpath = entry.get('file') if not fpath: @@ -162,18 +188,24 @@ def main(): fpath = os.path.normpath(fpath) if not fpath.lower().endswith(SRC_EXTS): continue + if inc and not any(r.search(fpath) for r in inc): + continue + if exc and any(r.search(fpath) for r in exc): + continue if root_abs: real = os.path.realpath(fpath) if os.path.commonpath([real, root_abs]) != root_abs: sys.stderr.write( f"WARNING: skipping source outside --root: {fpath}\n") continue - if inc and not any(r.search(fpath) for r in inc): - continue - if exc and any(r.search(fpath) for r in exc): - continue for d in extract_defines(entry_tokens(entry)): - defines.add(d) + name = macro_name(d) + if name not in defined: + defined[name] = (d, fpath) + elif define_value(defined[name][0]) != define_value(d): + vals = conflicts.setdefault( + name, {defined[name][0]: defined[name][1]}) + vals.setdefault(d, fpath) if not os.path.isfile(fpath): continue if fpath not in seen: @@ -186,6 +218,17 @@ def main(): if not srcs: sys.exit("ERROR: no matching source files found in compilation database") + if conflicts and not args.options_h: + lines = ["ERROR: the selected compile commands disagree on these " + "macros:"] + for name in sorted(conflicts): + lines.append(f" {name}:") + for tok, src in sorted(conflicts[name].items()): + lines.append(f" -D{tok} (e.g. {src})") + lines.append("Narrow the set with --include/--exclude so one SBOM " + "records one configuration.") + sys.exit("\n".join(lines)) + included = [] if args.included_sources: sys.path.insert(0, SCRIPT_DIR) @@ -229,7 +272,7 @@ def main(): included.append(path) srcs += included - defines = sorted(defines) + defines = sorted(tok for tok, _src in defined.values()) cflags = ' '.join(f'-D{d}' for d in defines) if args.print_only: @@ -251,7 +294,9 @@ def main(): with open(srcs_out, 'w') as f: f.write('\n'.join(srcs) + '\n') - cmd = [args.driver, '--srcs-file', srcs_out, f'--cflags={cflags}', + config = (['--options-h', args.options_h] if args.options_h + else [f'--cflags={cflags}']) + cmd = [args.driver, '--srcs-file', srcs_out, *config, '--name', args.name] for flag, val in (('--version', args.version), ('--version-file', args.version_file), @@ -259,6 +304,13 @@ def main(): ('--root', args.root), ('--license-file', args.license_file), ('--copyright-start', args.copyright_start), + ('--license-override', args.license_override), + ('--license-text', args.license_text), + ('--supplier', args.supplier), + ('--component-type', args.component_type), + ('--dep-wolfssl', args.dep_wolfssl), + ('--dep-wolfcrypt', args.dep_wolfcrypt), + ('--dep-openssl', args.dep_openssl), ('--settings-h', args.settings_h), ('--crypto-only', args.crypto_only), ('--gen-sbom', args.gen_sbom), @@ -268,6 +320,8 @@ def main(): cmd += [flag, val] for inc in args.include_dir: cmd += ['--include-dir', inc] + for dv in args.dep_version: + cmd += ['--dep-version', dv] print(f"compdb SBOM: {len(srcs)} sources ({len(included)} included), " f"{len(defines)} defines") diff --git a/share/sbom-driver.py b/share/sbom-driver.py index 44d12ad..8837615 100755 --- a/share/sbom-driver.py +++ b/share/sbom-driver.py @@ -435,7 +435,9 @@ def main(): with open(args.options_h, encoding="utf-8", errors="replace") as f: defines_text = f.read() elif args.cflags: - if args.settings_h and not os.path.isfile(args.settings_h): + # A relative path is left to the compiler's -include search. + if (os.path.isabs(args.settings_h) + and not os.path.isfile(args.settings_h)): sys.exit(f"ERROR: --settings-h '{args.settings_h}' " f"does not exist.") defines_text = capture_macros( diff --git a/tests/test_compdb_sbom.py b/tests/test_compdb_sbom.py new file mode 100644 index 0000000..702f75a --- /dev/null +++ b/tests/test_compdb_sbom.py @@ -0,0 +1,184 @@ +#!/usr/bin/env python3 +"""Tests for the compilation-database frontend (share/frontends/compdb_sbom.py).""" + +import json +import os +import subprocess +import sys +import tempfile +import unittest + +REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +COMPDB = os.path.join(REPO, "share", "frontends", "compdb_sbom.py") + +# Stands in for sbom-driver and writes its argv, one per line, to STUB_ARGV. +_STUB = """import os, sys +with open(os.environ['STUB_ARGV'], 'w') as f: + f.write('\\n'.join(sys.argv[1:])) +""" + + +class CompdbCase(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.d = self.tmp.name + self.db = os.path.join(self.d, "compile_commands.json") + self.stub = os.path.join(self.d, "stub-driver") + with open(self.stub, "w") as f: + f.write("#!" + sys.executable + "\n" + _STUB) + os.chmod(self.stub, 0o755) + self.argv_path = os.path.join(self.d, "argv.txt") + + def tearDown(self): + self.tmp.cleanup() + + def write_db(self, entries): + db = [] + for name, flags in entries: + src = os.path.join(self.d, name) + open(src, "w").close() + db.append({"directory": self.d, "file": name, + "arguments": ["cc", *flags, "-c", name]}) + with open(self.db, "w") as f: + json.dump(db, f) + + def run_compdb(self, *extra): + return subprocess.run( + [sys.executable, COMPDB, self.db, "--name", "t", + "--driver", self.stub, *extra], + capture_output=True, text=True, + env=dict(os.environ, STUB_ARGV=self.argv_path)) + + def driver_argv(self): + with open(self.argv_path) as f: + return f.read().splitlines() + + +class TestConflictingDefines(CompdbCase): + def test_different_values_fail(self): + self.write_db([("a.c", ["-DFOO=10"]), ("b.c", ["-DFOO=2"])]) + r = self.run_compdb("--print-only") + self.assertNotEqual(r.returncode, 0) + self.assertIn("FOO", r.stderr) + self.assertIn("-DFOO=10", r.stderr) + self.assertIn("-DFOO=2", r.stderr) + self.assertIn("a.c", r.stderr) + self.assertIn("b.c", r.stderr) + self.assertIn("--include/--exclude", r.stderr) + + def test_bare_define_equals_value_one(self): + self.write_db([("a.c", ["-DFOO"]), ("b.c", ["-DFOO=1"])]) + r = self.run_compdb("--print-only") + self.assertEqual(r.returncode, 0, r.stderr) + + def test_empty_value_conflicts_with_bare_define(self): + self.write_db([("a.c", ["-DFOO"]), ("b.c", ["-DFOO="])]) + r = self.run_compdb("--print-only") + self.assertNotEqual(r.returncode, 0) + self.assertIn("FOO", r.stderr) + + def test_options_h_ignores_conflicting_defines(self): + self.write_db([("a.c", ["-DUNIT_ID=1"]), ("b.c", ["-DUNIT_ID=2"])]) + r = self.run_compdb("--options-h", "options.h") + self.assertEqual(r.returncode, 0, r.stderr) + self.assertIn("--options-h", self.driver_argv()) + + def test_exclude_resolves_conflict(self): + self.write_db([("a.c", ["-DFOO=10"]), ("b.c", ["-DFOO=2"])]) + r = self.run_compdb("--exclude", r"b\.c$", "--print-only") + self.assertEqual(r.returncode, 0, r.stderr) + self.assertIn("-DFOO=10", r.stdout) + self.assertNotIn("-DFOO=2", r.stdout) + + def test_undef_in_one_entry_is_not_a_conflict(self): + self.write_db([("a.c", ["-DFOO=1"]), ("b.c", ["-DFOO=1", "-UFOO"])]) + r = self.run_compdb("--print-only") + self.assertEqual(r.returncode, 0, r.stderr) + self.assertIn("-DFOO=1", r.stdout) + + def test_partial_defines_are_merged(self): + self.write_db([("a.c", ["-DFOO=1", "-DONLY_A"]), + ("b.c", ["-DFOO=1", "-DONLY_B=2"])]) + r = self.run_compdb("--print-only") + self.assertEqual(r.returncode, 0, r.stderr) + self.assertIn("# 2 sources (0 included), 3 defines", r.stdout) + for tok in ("-DFOO=1", "-DONLY_A", "-DONLY_B=2"): + self.assertIn(tok, r.stdout) + + +class TestForwarding(CompdbCase): + def test_identity_and_dependency_flags_reach_driver(self): + self.write_db([("a.c", ["-DFOO=1"])]) + r = self.run_compdb( + "--license-override", "GPL-3.0-or-later", + "--license-text", "/lic.txt", + "--supplier", "Example Inc.", + "--component-type", "firmware", + "--dep-wolfssl", "yes", "--dep-wolfcrypt", "yes", + "--dep-openssl", "no", + "--dep-version", "wolfssl=5.9.2", + "--dep-version", "wolfcrypt=5.9.2") + self.assertEqual(r.returncode, 0, r.stderr) + argv = self.driver_argv() + pairs = list(zip(argv, argv[1:])) + for pair in (("--license-override", "GPL-3.0-or-later"), + ("--license-text", "/lic.txt"), + ("--supplier", "Example Inc."), + ("--component-type", "firmware"), + ("--dep-wolfssl", "yes"), + ("--dep-wolfcrypt", "yes"), + ("--dep-openssl", "no"), + ("--dep-version", "wolfssl=5.9.2"), + ("--dep-version", "wolfcrypt=5.9.2")): + self.assertIn(pair, pairs) + self.assertIn("--cflags=-DFOO=1", argv) + + def test_options_h_replaces_cflags(self): + self.write_db([("a.c", ["-DFOO=1"])]) + opts = os.path.join(self.d, "options.h") + open(opts, "w").close() + r = self.run_compdb("--options-h", opts) + self.assertEqual(r.returncode, 0, r.stderr) + argv = self.driver_argv() + self.assertIn(("--options-h", opts), list(zip(argv, argv[1:]))) + self.assertFalse(any(a.startswith("--cflags") for a in argv), argv) + + def test_options_h_with_capture_inputs_fails(self): + self.write_db([("a.c", ["-DFOO=1"])]) + for extra in (["--settings-h", "settings.h"], ["--include-dir", "."]): + r = self.run_compdb("--options-h", "options.h", *extra) + self.assertNotEqual(r.returncode, 0) + self.assertIn("--options-h", r.stderr) + self.assertFalse(os.path.exists(self.argv_path)) + + +class TestRootFilter(CompdbCase): + def write_outside(self): + root = os.path.join(self.d, "mod") + os.mkdir(root) + open(os.path.join(root, "in.c"), "w").close() + open(os.path.join(self.d, "app.c"), "w").close() + with open(self.db, "w") as f: + json.dump([{"directory": self.d, "file": n, + "arguments": ["cc", "-c", n]} + for n in ("mod/in.c", "app.c")], f) + return root + + def test_filtered_out_source_is_not_warned(self): + root = self.write_outside() + r = self.run_compdb("--root", root, "--include", "/mod/", + "--print-only") + self.assertEqual(r.returncode, 0, r.stderr) + self.assertNotIn("outside --root", r.stderr) + self.assertIn("in.c", r.stdout) + + def test_selected_source_outside_root_is_warned(self): + root = self.write_outside() + r = self.run_compdb("--root", root, "--print-only") + self.assertEqual(r.returncode, 0, r.stderr) + self.assertIn("outside --root", r.stderr) + self.assertNotIn("app.c", r.stdout) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/tests/test_sbom.py b/tests/test_sbom.py index b202c09..8ac9285 100755 --- a/tests/test_sbom.py +++ b/tests/test_sbom.py @@ -353,6 +353,31 @@ def integration_tests(gen_sbom): "--cdx-out", "l.cdx.json", "--spdx-out", "l.spdx.json"]) check(r.returncode == 0, "library generate succeeds") + # A relative --settings-h resolves like -include: cwd, then -I in + # CFLAGS, then --include-dir. + for sub, macro in (("a", "PICK_A"), ("b", "PICK_B")): + os.makedirs(os.path.join(d, sub, "cfg")) + with open(os.path.join(d, sub, "cfg", "s.h"), "w") as f: + f.write(f"#ifdef SEL\n#define {macro} 1\n#endif\n") + for extra, want, label in ( + (["--include-dir", os.path.join(d, "b")], "PICK_B", + "relative --settings-h found under --include-dir"), + ([f"--cflags=-DSEL -I{os.path.join(d, 'a')}", + "--include-dir", os.path.join(d, "b")], "PICK_A", + "relative --settings-h prefers the CFLAGS -I")): + if not any(e.startswith("--cflags") for e in extra): + extra = ["--cflags=-DSEL"] + extra + r = run_driver(d, gen_sbom, [ + "--srcs-file", srcs, "--settings-h", "cfg/s.h", *extra, + "--cdx-out", "s.cdx.json", "--spdx-out", "s.spdx.json"]) + text = open(os.path.join(d, "s.cdx.json")).read() \ + if r.returncode == 0 else r.stderr + check(r.returncode == 0 and want in text, label) + r = run_driver(d, gen_sbom, [ + "--srcs-file", srcs, "--cflags=-DSEL", "--settings-h", "cfg/s.h", + "--cdx-out", "s.cdx.json", "--spdx-out", "s.spdx.json"]) + check(r.returncode != 0, "unresolvable relative --settings-h fails") + # Reproducibility: same inputs + SOURCE_DATE_EPOCH => byte-identical. env = dict(os.environ, SOURCE_DATE_EPOCH="1700000000") outs = []