diff --git a/.github/workflows/selftest.yml b/.github/workflows/selftest.yml index e646523..37bf8f6 100644 --- a/.github/workflows/selftest.yml +++ b/.github/workflows/selftest.yml @@ -62,6 +62,7 @@ jobs: tests/test_iar_sbom.py \ tests/test_included_sources.py \ tests/test_compdb_sbom.py \ + tests/test_zephyr_sbom.py \ tests/test_sbom_cmake_install.py \ tests/test_wolfssl_compliance_tooling_sync.py \ tests/test_sbom_autotools.py @@ -93,6 +94,9 @@ jobs: - name: Run compdb frontend tests run: python -m unittest tests/test_compdb_sbom.py + - name: Run Zephyr frontend tests + run: python -m unittest tests/test_zephyr_sbom.py + - name: Run wolfssl-compliance-tooling-sync tests run: python -m unittest tests/test_wolfssl_compliance_tooling_sync.py diff --git a/README.md b/README.md index 145cd41..5d95f13 100644 --- a/README.md +++ b/README.md @@ -103,7 +103,7 @@ system it ships) while all of them converge on the same engine. | `build/sbom.mk` | Plain Make | Product sets `SBOM_NAME`/`SBOM_SRCS`/`SBOM_CFLAGS` (plus `SBOM_SETTINGS_H` when a `user_settings.h` derives the config), includes the fragment. | | `frontends/compdb_sbom.py` | Any `compile_commands.json` | The universal fallback — TI CCS, MPLAB X, Renesas e2studio, or anything wrappable with `bear -- make ...`. | | `frontends/iar_sbom.py` | IAR Embedded Workbench `.ewp` | Parses the project XML directly — there's no build step to hook into. | -| `frontends/zephyr_sbom.py` | Zephyr module | Reads `zephyr_library_sources(...)` since west owns the actual build. | +| `frontends/zephyr_sbom.py` | Zephyr module | Reads the module's sources from a west build's `compile_commands.json`, since west owns the actual build. | ## Products can occupy more than one tier diff --git a/share/README.md b/share/README.md index 9313c1a..b88d114 100644 --- a/share/README.md +++ b/share/README.md @@ -14,7 +14,7 @@ into a product at `tools/sbom/`, together with the pin files (`VERSION` and | `frontends/compdb_sbom.py` | Extractor for any `compile_commands.json`. | | `frontends/iar_sbom.py` | Extractor for an IAR Embedded Workbench `.ewp`. | | `frontends/included_sources.py` | Finds the source files a compile command compiles in through `#include` (`compdb_sbom.py --included-sources`). | -| `frontends/zephyr_sbom.py` | Extractor for a Zephyr module `CMakeLists.txt`. | +| `frontends/zephyr_sbom.py` | Zephyr module, from a west build's `compile_commands.json`. | | `build/sbom.mk` | Shared plain-Make fragment and `wolfssl_compliance_tooling_sbom_rule` macro. | | `build/sbom.cmake` | Shared CMake helper: `wolfssl_compliance_tooling_add_sbom()`. | | `gen-sbom` | The vendored SBOM generator. | @@ -84,6 +84,13 @@ A product does not copy logic. It describes itself: `SBOM_GEN` is the canonical generator override; `GEN_SBOM` remains a legacy alias for compatibility. - Autotools: set the `SBOM_*` variables and `include tools/sbom/sbom.am`. +- Zephyr: call `wolfssl_compliance_tooling_add_sbom(TARGETS + ${ZEPHYR_CURRENT_LIBRARY} ...)` from the module's own CMakeLists, after its + last `zephyr_library_sources()`, so the SBOM records the source set west + actually built. Without that hook, run `zephyr_sbom.py` on the build's + `compile_commands.json` with `--root` set to the module, and pass + `--user-settings` with the module's settings header to record its + configuration. Keep only true product knowledge in the product: the route-through script, the module extractor, and the HAL source selector. diff --git a/share/frontends/zephyr_sbom.py b/share/frontends/zephyr_sbom.py index 2813d1d..c06c566 100755 --- a/share/frontends/zephyr_sbom.py +++ b/share/frontends/zephyr_sbom.py @@ -1,26 +1,36 @@ #!/usr/bin/env python3 -"""Generate a source-inventory SBOM for a Zephyr module. +"""Generate an SBOM for a Zephyr module from a west build. -This is the product-neutral form of wolfBoot's zephyr_sbom.py. A Zephyr module -compiles its sources into a Zephyr application through `zephyr_library_sources(...)`, -built by Zephyr/west rather than by the product's own Makefile or CMake. So -neither the Make nor the CMake SBOM target sees those sources. +A Zephyr module compiles its sources into a Zephyr application through +`zephyr_library_sources(...)`, built by Zephyr/west rather than by the +product's own Makefile or CMake. So neither the Make nor the CMake SBOM target +sees those sources. -This extractor reads the module's source list straight out of its CMakeLists (so -it stays in sync automatically) and hands it to the shared driver. +The most accurate route is to call the CMake helper from the module's own +CMakeLists after its last `zephyr_library_sources()`, +`wolfssl_compliance_tooling_add_sbom(TARGETS ${ZEPHYR_CURRENT_LIBRARY} ...)`. +Without that hook, this frontend reads the compile_commands.json that CMake +writes when west configures the build. The module's sources are the entries +under `--root` whose object Zephyr placed in the module's library directory, +`/modules//`. That set follows Kconfig and any CMake logic +the module uses, and it leaves out application files inside the module tree. -The module configuration is Kconfig-driven (CONFIG_* symbols), not a `-D` macro -set, so by default this produces a source-inventory SBOM. If you have a real -Zephyr build and want the exact compiled config, generate the SBOM from that -build's compilation database with compdb_sbom.py instead. +The configuration is Kconfig-driven, so by default this produces a +source-inventory SBOM. `--user-settings` records the module's -D flags and the +macros its settings header defines, preprocessed by the build's own compiler +with the module's compile command. `--options-h` records a flat `#define` +header instead. Usage: - zephyr_sbom.py --name NAME --cmakelists path/to/zephyr/CMakeLists.txt [options] + zephyr_sbom.py --name NAME --compdb build/compile_commands.json \\ + --root MODULE [--user-settings HEADER] [options] """ import argparse +import json import os import re +import shlex import subprocess import sys import tempfile @@ -29,38 +39,221 @@ DEFAULT_DRIVER = os.path.join(os.path.dirname(SCRIPT_DIR), "sbom-driver") SRC_EXTS = ('.c', '.cc', '.cpp', '.cxx', '.s', '.asm') +_ARG_OPTS = ('-D', '-U', '-I', '-iquote', '-isystem', '-idirafter', + '-imacros', '-include', '-o', '-MF', '-MT', '-MQ', '-x') +_NO_DUMP = ('-o', '-MF', '-MT', '-MQ', '-x', '-c', '-M', '-MM', '-MD', '-MMD', + '-MP') +_HOST_OPTS = ('-D', '-U', '-I', '-iquote') -def parse_library_sources(cmakelists): - """Extract the file list from the zephyr_library_sources(...) blocks.""" - with open(cmakelists) as f: - text = f.read() - module_dir = os.path.dirname(os.path.abspath(cmakelists)) +class ParseError(Exception): + pass - srcs = [] - for m in re.finditer(r'zephyr_library_sources\s*\((.*?)\)', text, re.DOTALL): - for raw in m.group(1).split(): - raw = raw.strip() - if not raw or not raw.lower().endswith(SRC_EXTS): - continue - # Resolve the common CMake module-directory variables. - p = raw.replace('${CMAKE_CURRENT_LIST_DIR}', module_dir) - p = p.replace('${CMAKE_CURRENT_SOURCE_DIR}', module_dir) - p = p.replace('${ZEPHYR_CURRENT_MODULE_DIR}', os.path.dirname(module_dir)) - if '${' in p: - sys.stderr.write(f"WARNING: skipping unresolved source: {raw}\n") + +def module_name(root): + """The Zephyr module name: module.yml's name, else the directory name.""" + for yml in ('module.yml', 'module.yaml'): + path = os.path.join(root, 'zephyr', yml) + if os.path.isfile(path): + try: + with open(path, encoding='utf-8') as f: + lines = f.readlines() + except (OSError, UnicodeDecodeError) as e: + raise ParseError(f"cannot read {path}: {e}") + for line in lines: + m = re.match(r'name:\s*([^\s#]+)', line) + if m: + return m.group(1).strip('"\'') + break + return os.path.basename(os.path.normpath(root)) + + +def compdb_entries(compdb, root, name): + """Return the module library's compile commands as [(source, entry)] + for sources under root, plus the library sources outside it.""" + try: + with open(compdb) as f: + db = json.load(f) + except (OSError, json.JSONDecodeError) as e: + raise ParseError(f"cannot read {compdb}: {e}") + root = os.path.realpath(root) + os.sep + picked, outside, seen = [], [], set() + for e in db: + if 'output' not in e: + raise ParseError(f"{compdb} has no 'output' field; it needs " + "CMake 3.20 or later") + if not e.get('file'): + continue + directory = e.get('directory', '') + lib_dir = os.path.normpath( + os.path.join(directory, 'modules', name)) + os.sep + out = os.path.normpath(os.path.join(directory, e['output'])) + src = os.path.normpath(os.path.join(directory, e['file'])) + if (not out.startswith(lib_dir) + or not src.lower().endswith(SRC_EXTS)): + continue + if os.path.realpath(src).startswith(root): + picked.append((src, e)) + elif src not in seen: + seen.add(src) + outside.append(src) + return picked, outside + + +def split_windows_command(cmd): + """Split a command line the way CommandLineToArgvW does.""" + args, cur, have, quoted, i = [], [], False, False, 0 + while i < len(cmd): + c = cmd[i] + if c == '\\': + n = len(cmd[i:]) - len(cmd[i:].lstrip('\\')) + i += n + if cmd[i:i + 1] == '"': + cur.append('\\' * (n // 2)) + if n % 2: + cur.append('"') + i += 1 + else: + cur.append('\\' * n) + have = True + elif c == '"': + quoted, have, i = not quoted, True, i + 1 + elif c in ' \t' and not quoted: + if have: + args.append(''.join(cur)) + cur, have = [], False + i += 1 + else: + cur.append(c) + have, i = True, i + 1 + if have: + args.append(''.join(cur)) + return args + + +def _entry_tokens(entry): + if entry.get('arguments'): + toks = list(entry['arguments']) + else: + cmd = entry.get('command', '') + try: + toks = (split_windows_command(cmd) if os.name == 'nt' + else shlex.split(cmd)) + except ValueError as e: + raise ParseError(f"cannot split the compile command of " + f"{entry.get('file')}: {e}") + if not toks: + raise ParseError(f"empty compile command for {entry.get('file')}") + return toks + + +def _dump_macros(cc, flags, directory): + try: + r = subprocess.run([cc, '-dM', '-E', *flags, '-x', 'c', os.devnull], + cwd=directory or None, capture_output=True, + text=True) + except OSError as e: + raise ParseError(f"cannot run '{cc}': {e}") + if r.returncode: + raise ParseError(f"'{cc} -dM -E' failed:\n{r.stderr.strip()}") + macros = {} + for line in r.stdout.splitlines(): + m = re.match(r'#define (\S+)', line) + if m: + macros[m.group(1)] = line + return macros + + +def _options(entry): + """Yield (option, argument) pairs of a compile command after the + compiler, with argument None for a flag and option None for a source.""" + toks = _entry_tokens(entry) + i = 1 + while i < len(toks): + t = toks[i] + if t.startswith('@'): + raise ParseError(f"response file {t} in the compile command of " + f"{entry.get('file')} is not supported") + if t in _ARG_OPTS and i + 1 < len(toks): + yield t, toks[i + 1] + i += 2 + continue + joined = next((o for o in _ARG_OPTS + if t.startswith(o) and t != o), None) + if joined: + yield joined, t[len(joined):] + elif t.startswith('-'): + yield t, None + else: + yield None, t + i += 1 + + +def _same_file(path, directory, settings_h): + full = os.path.join(directory or '', path) + return os.path.realpath(full) == os.path.realpath(settings_h) + + +def _defines(entries): + """Merge the net -D/-U state of every entry into one -D/-U list, or + raise ParseError when two entries disagree on a macro.""" + merged = {} + for src, entry in entries: + state = {} + for opt, arg in _options(entry): + if arg is None: continue - if not os.path.isabs(p): - p = os.path.join(module_dir, p) - srcs.append(os.path.normpath(p)) + if opt == '-D': + name = re.split(r'[=(]', arg, maxsplit=1)[0] + state[name] = arg if '=' in arg else arg + '=1' + elif opt == '-U': + state[arg] = None + for name, value in state.items(): + if name in merged and merged[name][0] != value: + raise ParseError( + f"the module's compile commands disagree on {name}: " + f"{merged[name][1]} and {src}") + merged.setdefault(name, (value, src)) + flags = [] + for name in sorted(merged): + value = merged[name][0] + flags += ['-U', name] if value is None else ['-D', value] + return flags - seen, present, missing = set(), [], [] - for s in srcs: - if s in seen: + +def capture_user_settings(entries, settings_h, hostcc=None): + """Return the #define lines the module's -D flags and settings_h add. + + The capture runs the first C entry's compiler and flags, with the -D and + -U flags merged across all C entries. With hostcc it runs that compiler + with only the -D, -U, -I, -iquote and autoconf.h flags.""" + c_entries = [(src, e) for src, e in entries + if not src.lower().endswith(('.s', '.asm'))] or entries + entry = c_entries[0][1] + directory = entry.get('directory') + base = [] + for opt, arg in _options(entry): + if opt is None or opt in _NO_DUMP or opt in ('-D', '-U'): + continue + if opt in ('-include', '-imacros') and _same_file(arg, directory, + settings_h): + continue + if hostcc and opt not in _HOST_OPTS and not ( + opt == '-imacros' and os.path.basename(arg) == 'autoconf.h'): continue - seen.add(s) - (present if os.path.isfile(s) else missing).append(s) - return present, missing + base += [opt] if arg is None else [opt, arg] + cc = hostcc or _entry_tokens(entry)[0] + defs = _defines(c_entries) + before = _dump_macros(cc, base, directory) + with_defs = _dump_macros(cc, base + defs, directory) + after = _dump_macros( + cc, base + defs + ['-include', os.path.abspath(settings_h)], + directory) + if after == with_defs: + raise ParseError(f"{settings_h} adds no macros; the compile command " + f"may already include it or a copy of it") + return sorted(line for name, line in after.items() + if before.get(name) != line) def main(): @@ -68,19 +261,40 @@ def main(): description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument('--name', required=True, help='Package name for the SBOM.') - ap.add_argument('--cmakelists', required=True, - help='Path to the module CMakeLists.txt.') + ap.add_argument('--compdb', required=True, + help="The west build's compile_commands.json.") + ap.add_argument('--root', required=True, help='The module directory.') + ap.add_argument('--module-name', default=None, + help='Zephyr module name. Default: the name in ' + 'zephyr/module.yml, else the --root directory name.') + ap.add_argument('--user-settings', default=None, + help="The module's settings header, captured with the " + "module's compile flags.") + ap.add_argument('--hostcc', default=None, + help='Capture --user-settings with this host compiler ' + 'instead of the build\'s compiler and target flags.') + ap.add_argument('--options-h', default=None, + help='Flat #define header to record as the build ' + 'configuration, instead of --source-only.') ap.add_argument('--driver', default=DEFAULT_DRIVER) ap.add_argument('--gen-sbom', default=None) ap.add_argument('--version', default=None) ap.add_argument('--version-file', default=None) ap.add_argument('--version-macro', default=None) - ap.add_argument('--root', default=None) + ap.add_argument('--supplier', default=None) + ap.add_argument('--component-type', default=None) ap.add_argument('--license-file', default=None) ap.add_argument('--copyright-start', default=None) - # This front end is --source-only: no macros are captured, so the - # generator cannot read WOLFCRYPT_ONLY and will say so unless the - # operator states it here. + ap.add_argument('--license-override', default=None) + ap.add_argument('--license-text', default=None) + ap.add_argument('--dep-wolfssl', default=None) + ap.add_argument('--dep-wolfcrypt', default=None) + ap.add_argument('--dep-openssl', default=None) + ap.add_argument('--dep-version', action='append', default=[], + help='Dependency version, e.g. wolfssl=5.9.2 ' + '(repeatable).') + # In source-only mode the generator cannot read WOLFCRYPT_ONLY and will + # say so unless the operator states it here. ap.add_argument('--crypto-only', default=None, help='auto|yes|no, forwarded to the generator.') ap.add_argument('--cdx-out', default=None) @@ -89,54 +303,106 @@ def main(): ap.add_argument('--print-only', action='store_true') args = ap.parse_args() - if not os.path.isfile(args.cmakelists): - sys.exit(f"ERROR: CMakeLists not found: {args.cmakelists}") + if args.user_settings and args.options_h: + sys.exit("ERROR: pass only one of --user-settings and --options-h.") + if args.hostcc and not args.user_settings: + sys.exit("ERROR: --hostcc only applies to --user-settings.") + for path, what in ((args.compdb, 'compilation database'), + (args.user_settings, '--user-settings')): + if path and not os.path.isfile(path): + sys.exit(f"ERROR: {what} not found: {path}") - srcs, missing = parse_library_sources(args.cmakelists) - if missing: + settings = None + try: + name = args.module_name or module_name(args.root) + entries, outside = compdb_entries(args.compdb, args.root, name) + if not entries: + sys.exit(f"ERROR: {args.compdb} has no compile commands for " + f"module '{name}' from sources under {args.root}; " + f"check --root and --module-name.") + if args.user_settings: + settings = capture_user_settings(entries, args.user_settings, + args.hostcc) + except ParseError as e: + sys.exit(f"ERROR: {e}") + if outside: sys.stderr.write( - f"WARNING: {len(missing)} module source(s) not found on disk " - f"(excluded):\n") - for m in missing: - sys.stderr.write(f" - {m}\n") - if not srcs: - sys.exit("ERROR: no zephyr_library_sources found in " + args.cmakelists) + f"WARNING: {len(outside)} source(s) of module '{name}' lie " + f"outside --root (excluded):\n") + for s in outside: + sys.stderr.write(f" - {s}\n") + srcs = list(dict.fromkeys(src for src, _ in entries)) if args.print_only: print(f"# {args.name}: {len(srcs)} sources") for s in srcs: print(f" {s}") + if settings is not None: + print(f"\n# {len(settings)} macros from {args.user_settings}") + for line in settings: + print(f" {line}") return - srcs_out, tmp = args.srcs_out, None - if not srcs_out: - fd, srcs_out = tempfile.mkstemp(prefix='wolfssl-compliance-tooling-zephyr-srcs-', suffix='.txt') - os.close(fd) - tmp = srcs_out - with open(srcs_out, 'w') as f: - f.write('\n'.join(srcs) + '\n') - - cmd = [args.driver, '--srcs-file', srcs_out, '--source-only', - '--name', args.name] - for flag, val in (('--version', args.version), - ('--version-file', args.version_file), - ('--version-macro', args.version_macro), - ('--root', args.root), - ('--license-file', args.license_file), - ('--copyright-start', args.copyright_start), - ('--crypto-only', args.crypto_only), - ('--gen-sbom', args.gen_sbom), - ('--cdx-out', args.cdx_out), - ('--spdx-out', args.spdx_out)): - if val: - cmd += [flag, val] - - print(f"Zephyr module SBOM: {len(srcs)} sources (source-inventory)") + tmps = [] try: - rc = subprocess.call(cmd) + srcs_out = args.srcs_out + if not srcs_out: + fd, srcs_out = tempfile.mkstemp( + prefix='wolfssl-compliance-tooling-zephyr-srcs-', + suffix='.txt') + os.close(fd) + tmps.append(srcs_out) + try: + with open(srcs_out, 'w') as f: + f.write('\n'.join(srcs) + '\n') + except OSError as e: + sys.exit(f"ERROR: cannot write {srcs_out}: {e}") + + cmd = [args.driver, '--srcs-file', srcs_out, '--name', args.name, + '--root', args.root] + if settings is not None: + fd, opts = tempfile.mkstemp( + prefix='wolfssl-compliance-tooling-zephyr-defs-', suffix='.h') + tmps.append(opts) + with os.fdopen(fd, 'w') as f: + f.write('\n'.join(settings) + '\n') + cmd += ['--options-h', opts] + elif args.options_h: + cmd += ['--options-h', args.options_h] + else: + cmd += ['--source-only'] + for flag, val in (('--version', args.version), + ('--version-file', args.version_file), + ('--version-macro', args.version_macro), + ('--supplier', args.supplier), + ('--component-type', args.component_type), + ('--license-file', args.license_file), + ('--copyright-start', args.copyright_start), + ('--license-override', args.license_override), + ('--license-text', args.license_text), + ('--dep-wolfssl', args.dep_wolfssl), + ('--dep-wolfcrypt', args.dep_wolfcrypt), + ('--dep-openssl', args.dep_openssl), + ('--crypto-only', args.crypto_only), + ('--gen-sbom', args.gen_sbom), + ('--cdx-out', args.cdx_out), + ('--spdx-out', args.spdx_out)): + if val: + cmd += [flag, val] + for dv in args.dep_version: + cmd += ['--dep-version', dv] + + kind = ("user-settings" if settings is not None + else "options-h" if args.options_h else "source-inventory") + print(f"Zephyr module SBOM: {len(srcs)} sources ({kind})") + try: + rc = subprocess.call(cmd) + except OSError as e: + sys.exit(f"ERROR: cannot run {args.driver}: {e}") finally: - if tmp and os.path.exists(tmp): - os.remove(tmp) + for t in tmps: + if os.path.exists(t): + os.remove(t) sys.exit(rc) diff --git a/tests/test_zephyr_sbom.py b/tests/test_zephyr_sbom.py new file mode 100644 index 0000000..85c6824 --- /dev/null +++ b/tests/test_zephyr_sbom.py @@ -0,0 +1,411 @@ +#!/usr/bin/env python3 +"""Tests for the Zephyr frontend (share/frontends/zephyr_sbom.py). + +Focus: the source set must be exactly what the west build compiled into the +module's library, and the settings capture must record the module's +configuration without Kconfig symbols or host built-ins.""" + +import importlib.util +import json +import os +import shlex +import subprocess +import sys +import tempfile +import unittest + +REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +ZEPHYR = os.path.join(REPO, "share", "frontends", "zephyr_sbom.py") + +_spec = importlib.util.spec_from_file_location("zephyr_sbom", ZEPHYR) +zephyr = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(zephyr) + +LIB = "modules/mod/CMakeFiles/..__mod__zephyr.dir/" +APP = "CMakeFiles/app.dir/" + +# Stands in for sbom-driver: writes its argv, and the --options-h header it +# was handed, to files named by STUB_OUT. +_STUB = """import os, sys +out = os.environ['STUB_OUT'] +with open(out + '.argv', 'w') as f: + f.write('\\n'.join(sys.argv[1:])) +if '--options-h' in sys.argv: + src = sys.argv[sys.argv.index('--options-h') + 1] + with open(src) as f, open(out + '.opts', 'w') as g: + g.write(src + '\\n' + f.read()) +""" + + +class _ZephyrTest(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.mod = os.path.join(self.tmp.name, "mod") + self.build = os.path.join(self.tmp.name, "build") + os.makedirs(os.path.join(self.mod, "zephyr")) + os.makedirs(self.build) + self.write("zephyr/module.yml", "name: mod\n") + for rel in ("src/core.c", "src/est.c", "src/scep.c", "zephyr/glue.c", + "tests/app.c"): + self.write(rel, "") + self.outside = os.path.join(self.tmp.name, "other.c") + open(self.outside, "w").close() + + def tearDown(self): + self.tmp.cleanup() + + def write(self, rel, text): + path = os.path.join(self.mod, rel) + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w") as f: + f.write(text) + return path + + def write_compdb(self, entries, flags=()): + db = [{"directory": self.build, + "file": src if os.path.isabs(src) + else os.path.join(self.mod, src), + "arguments": ["cc", *flags, "-c", src], + "output": out + os.path.basename(src) + ".obj"} + for src, out in entries] + path = os.path.join(self.build, "compile_commands.json") + with open(path, "w") as f: + json.dump(db, f) + return path + + def run_frontend(self, *extra): + return subprocess.run( + [sys.executable, ZEPHYR, "--name", "mod", "--root", self.mod, + *extra], capture_output=True, text=True, + env=dict(os.environ, STUB_OUT=os.path.join(self.tmp.name, "stub"))) + + def rels(self, entries): + return sorted(os.path.relpath(src, self.mod) for src, _ in entries) + + +class TestModuleName(_ZephyrTest): + def test_module_yml_name(self): + self.write("zephyr/module.yml", + "build:\n cmake: zephyr\nname: 'mod' # pinned\n") + self.assertEqual(zephyr.module_name(self.mod), "mod") + + def test_directory_name_without_module_yml_name(self): + self.write("zephyr/module.yml", "build:\n cmake: zephyr\n") + self.assertEqual(zephyr.module_name(self.mod), "mod") + os.rename(self.mod, self.mod + "-x") + self.assertEqual(zephyr.module_name(self.mod + "-x"), "mod-x") + + +class TestSourceSelection(_ZephyrTest): + def test_only_module_library_sources_under_root(self): + compdb = self.write_compdb([ + ("src/core.c", LIB), ("src/est.c", LIB), ("zephyr/glue.c", LIB), + ("src/core.c", LIB), + ("tests/app.c", APP), + ("src/scep.c", "modules/mod-extra/CMakeFiles/x.dir/"), + (self.outside, LIB)]) + entries, outside = zephyr.compdb_entries(compdb, self.mod, "mod") + self.assertEqual(self.rels(entries), + ["src/core.c", "src/core.c", "src/est.c", + "zephyr/glue.c"]) + self.assertEqual(outside, [self.outside]) + + def test_unnormalized_build_directory(self): + compdb = self.write_compdb([("src/core.c", LIB)]) + with open(compdb) as f: + db = json.load(f) + db[0]["directory"] = self.build + "/./" + with open(compdb, "w") as f: + json.dump(db, f) + entries, _ = zephyr.compdb_entries(compdb, self.mod, "mod") + self.assertEqual(self.rels(entries), ["src/core.c"]) + + def test_makefile_generator_layout(self): + compdb = self.write_compdb([("src/core.c", LIB)]) + with open(compdb) as f: + db = json.load(f) + db[0]["directory"] = os.path.join(self.build, "modules", "mod") + with open(compdb, "w") as f: + json.dump(db, f) + entries, _ = zephyr.compdb_entries(compdb, self.mod, "mod") + self.assertEqual(self.rels(entries), ["src/core.c"]) + + def test_missing_output_field_fails(self): + compdb = self.write_compdb([("src/core.c", LIB)]) + with open(compdb) as f: + db = json.load(f) + del db[0]["output"] + with open(compdb, "w") as f: + json.dump(db, f) + with self.assertRaises(zephyr.ParseError): + zephyr.compdb_entries(compdb, self.mod, "mod") + + def test_no_module_entries_fails(self): + compdb = self.write_compdb([("src/core.c", APP)]) + r = self.run_frontend("--compdb", compdb, "--print-only") + self.assertNotEqual(r.returncode, 0) + self.assertIn("--module-name", r.stderr) + + def test_source_outside_root_is_warned(self): + compdb = self.write_compdb([("src/core.c", LIB), (self.outside, LIB)]) + r = self.run_frontend("--compdb", compdb, "--print-only") + self.assertEqual(r.returncode, 0, r.stderr) + self.assertIn("outside --root", r.stderr) + self.assertNotIn("other.c", r.stdout) + + +class TestCommandSplitting(unittest.TestCase): + def test_windows_rules(self): + cmd = ('C:\\sdk\\gcc.exe -DSTR=\\"x\\" -DX="a b" ' + '-I"C:\\My Dir" -c a.c') + self.assertEqual(zephyr.split_windows_command(cmd), [ + "C:\\sdk\\gcc.exe", '-DSTR="x"', "-DX=a b", "-IC:\\My Dir", + "-c", "a.c"]) + self.assertEqual(zephyr.split_windows_command('a\\\\"b c" d'), + ["a\\b c", "d"]) + + +class TestDefines(unittest.TestCase): + def entry(self, *flags): + return ("x.c", {"arguments": ["cc", "-c", "x.c", *flags]}) + + def test_bare_define_equals_value_one_and_last_wins(self): + self.assertEqual( + zephyr._defines([self.entry("-DA", "-UB", "-DB=2", "-D"), + self.entry("-DA=1")]), + ["-D", "A=1", "-D", "B=2"]) + + def test_define_and_undefine_disagree(self): + with self.assertRaises(zephyr.ParseError): + zephyr._defines([self.entry("-DA"), self.entry("-UA")]) + + +class TestUserSettings(_ZephyrTest): + def setUp(self): + super().setUp() + self.autoconf = os.path.join(self.build, "autoconf.h") + with open(self.autoconf, "w") as f: + f.write("#define CONFIG_MOD_X 1\n") + self.toolchain = os.path.join(self.build, "toolchain.h") + with open(self.toolchain, "w") as f: + f.write("#define TOOLCHAIN_ONLY 1\n") + self.write("cfg/extra.h", "#define HAVE_Y 2\n") + self.settings = self.write( + "zephyr/user_settings.h", + "#ifdef CONFIG_MOD_X\n#define HAVE_X 1\n#endif\n" + "#ifdef MOD_DEBUG\n#define HAVE_DEBUG_LOG 1\n#endif\n" + "#ifdef TOOLCHAIN_ONLY\n#define SAW_TOOLCHAIN 1\n#endif\n" + "#ifdef __STDC_VERSION__\n#define HAVE_C99 1\n#endif\n" + "#include SETTINGS_FILE\n") + self.flags = ["-std=c89", "-DMOD_DEBUG", '-DSETTINGS_FILE="extra.h"', + "-imacros", self.autoconf, f"-imacros{self.toolchain}", + "-I", os.path.join(self.mod, "cfg"), + "-o", "core.c.obj", "-MD", "-MF", "core.c.d"] + + def capture(self, flags, *extra): + compdb = self.write_compdb([("src/core.c", LIB)], flags) + r = self.run_frontend("--compdb", compdb, "--print-only", + "--user-settings", self.settings, *extra) + self.assertEqual(r.returncode, 0, r.stderr) + return r.stdout.split("macros from", 1)[1] + + def test_build_compiler_and_flags(self): + macros = self.capture(self.flags) + for line in ("#define HAVE_X 1", "#define HAVE_Y 2", + "#define HAVE_DEBUG_LOG 1", "#define MOD_DEBUG 1", + '#define SETTINGS_FILE "extra.h"', + "#define SAW_TOOLCHAIN 1"): + self.assertIn(line, macros) + for name in ("CONFIG_MOD_X", "TOOLCHAIN_ONLY", "HAVE_C99", + "__STDC__", "__VERSION__"): + self.assertNotIn(name, macros) + + def test_hostcc_keeps_only_portable_flags(self): + macros = self.capture(self.flags, "--hostcc", "cc") + for line in ("#define HAVE_X 1", "#define HAVE_Y 2", + "#define MOD_DEBUG 1", "#define HAVE_C99 1"): + self.assertIn(line, macros) + self.assertNotIn("SAW_TOOLCHAIN", macros) + + def test_joined_flag_forms(self): + flags = ["-DMOD_DEBUG", '-DSETTINGS_FILE="extra.h"', + f"-imacros{self.autoconf}", + f"-I{os.path.join(self.mod, 'cfg')}"] + macros = self.capture(flags) + self.assertIn("#define HAVE_X 1", macros) + self.assertIn("#define HAVE_Y 2", macros) + + def test_command_string_form(self): + compdb = self.write_compdb([("src/core.c", LIB)], self.flags) + with open(compdb) as f: + db = json.load(f) + db[0]["command"] = shlex.join(db[0].pop("arguments")) + with open(compdb, "w") as f: + json.dump(db, f) + r = self.run_frontend("--compdb", compdb, "--print-only", + "--user-settings", self.settings) + self.assertEqual(r.returncode, 0, r.stderr) + self.assertIn("#define MOD_DEBUG 1", r.stdout) + self.assertIn('#define SETTINGS_FILE "extra.h"', r.stdout) + + def test_force_included_header_is_still_recorded(self): + macros = self.capture(self.flags + ["-include", self.settings]) + self.assertIn("#define HAVE_X 1", macros) + self.assertIn("#define HAVE_Y 2", macros) + + def test_defines_merge_across_entries(self): + compdb = self.write_compdb([("src/core.c", LIB)], self.flags) + with open(compdb) as f: + db = json.load(f) + other = dict(db[0], file=os.path.join(self.mod, "src/est.c"), + output=LIB + "est.c.obj", + arguments=db[0]["arguments"] + ["-DEST_ONLY", + "-UMOD_DEBUG"]) + with open(compdb, "w") as f: + json.dump(db + [other], f) + r = self.run_frontend("--compdb", compdb, "--print-only", + "--user-settings", self.settings) + self.assertNotEqual(r.returncode, 0) + self.assertIn("disagree on MOD_DEBUG", r.stderr) + other["arguments"] = other["arguments"][:-1] + with open(compdb, "w") as f: + json.dump(db + [other], f) + r = self.run_frontend("--compdb", compdb, "--print-only", + "--user-settings", self.settings) + self.assertEqual(r.returncode, 0, r.stderr) + self.assertIn("#define EST_ONLY 1", r.stdout) + self.assertIn("#define MOD_DEBUG 1", r.stdout) + + def test_same_source_in_two_targets_disagrees(self): + compdb = self.write_compdb([("src/core.c", LIB)], self.flags) + with open(compdb) as f: + db = json.load(f) + other = dict(db[0], output=LIB.replace("mod__zephyr", "mod_b") + + "core.c.obj", + arguments=db[0]["arguments"] + ["-UMOD_DEBUG"]) + with open(compdb, "w") as f: + json.dump(db + [other], f) + r = self.run_frontend("--compdb", compdb, "--print-only", + "--user-settings", self.settings) + self.assertNotEqual(r.returncode, 0) + self.assertIn("disagree on MOD_DEBUG", r.stderr) + r = self.run_frontend("--compdb", compdb, "--print-only") + self.assertEqual(r.returncode, 0, r.stderr) + self.assertIn("# mod: 1 sources", r.stdout) + + def test_header_already_included_as_copy_fails(self): + text = "#ifndef US_H\n#define US_H\n#define HAVE_X 1\n#endif\n" + self.write("zephyr/user_settings.h", text) + copy = os.path.join(self.build, "user_settings.h") + with open(copy, "w") as f: + f.write(text) + compdb = self.write_compdb([("src/core.c", LIB)], + ["-include", copy]) + r = self.run_frontend("--compdb", compdb, "--print-only", + "--user-settings", self.settings) + self.assertNotEqual(r.returncode, 0) + self.assertIn("adds no macros", r.stderr) + + def test_asm_entry_first_is_skipped(self): + self.write("src/start.S", "") + compdb = self.write_compdb([("src/start.S", LIB), + ("src/core.c", LIB)], self.flags) + with open(compdb) as f: + db = json.load(f) + db[0]["arguments"] = ["/nonexistent/as", "-c", "src/start.S"] + with open(compdb, "w") as f: + json.dump(db, f) + r = self.run_frontend("--compdb", compdb, "--print-only", + "--user-settings", self.settings) + self.assertEqual(r.returncode, 0, r.stderr) + self.assertIn("#define HAVE_X 1", r.stdout) + + def test_hostcc_needs_user_settings(self): + compdb = self.write_compdb([("src/core.c", LIB)]) + r = self.run_frontend("--compdb", compdb, "--hostcc", "cc") + self.assertNotEqual(r.returncode, 0) + self.assertIn("--hostcc only applies", r.stderr) + + def test_response_file_fails(self): + compdb = self.write_compdb([("src/core.c", LIB)], ["@flags.rsp"]) + r = self.run_frontend("--compdb", compdb, "--print-only", + "--user-settings", self.settings) + self.assertNotEqual(r.returncode, 0) + self.assertIn("response file", r.stderr) + + def test_missing_compiler_is_an_error(self): + compdb = self.write_compdb([("src/core.c", LIB)], self.flags) + with open(compdb) as f: + db = json.load(f) + db[0]["arguments"][0] = "/nonexistent/cc" + with open(compdb, "w") as f: + json.dump(db, f) + r = self.run_frontend("--compdb", compdb, "--print-only", + "--user-settings", self.settings) + self.assertNotEqual(r.returncode, 0) + self.assertIn("ERROR: cannot run", r.stderr) + self.assertNotIn("Traceback", r.stderr) + + def test_options_h_excludes_user_settings(self): + compdb = self.write_compdb([("src/core.c", LIB)]) + r = self.run_frontend("--compdb", compdb, "--options-h", "o.h", + "--user-settings", self.settings) + self.assertNotEqual(r.returncode, 0) + self.assertIn("only one of", r.stderr) + + +class TestDriverForwarding(_ZephyrTest): + def run_main(self, extra, flags=()): + stub = os.path.join(self.tmp.name, "stub-driver.py") + with open(stub, "w") as f: + f.write(f"#!{sys.executable}\n" + _STUB) + os.chmod(stub, 0o755) + compdb = self.write_compdb([("src/core.c", LIB)], flags) + r = self.run_frontend("--compdb", compdb, "--driver", stub, *extra) + self.assertEqual(r.returncode, 0, r.stderr) + with open(os.path.join(self.tmp.name, "stub.argv")) as f: + return f.read().splitlines() + + def test_identity_license_and_dependency_flags(self): + argv = self.run_main([ + "--license-override", "GPL-3.0-or-later", + "--license-text", "LICENSE.txt", + "--supplier", "Example Inc.", + "--component-type", "firmware", + "--dep-wolfssl", "yes", "--dep-wolfcrypt", "yes", + "--dep-openssl", "no", + "--dep-version", "wolfssl=5.9.2", + "--dep-version", "wolfcrypt=5.9.2"]) + pairs = list(zip(argv, argv[1:])) + for pair in (("--license-override", "GPL-3.0-or-later"), + ("--license-text", "LICENSE.txt"), + ("--supplier", "Example Inc."), + ("--component-type", "firmware"), + ("--dep-wolfssl", "yes"), + ("--dep-wolfcrypt", "yes"), + ("--dep-openssl", "no"), + ("--dep-version", "wolfssl=5.9.2"), + ("--dep-version", "wolfcrypt=5.9.2"), + ("--root", self.mod)): + self.assertIn(pair, pairs) + self.assertIn("--source-only", argv) + + def test_options_h_replaces_source_only(self): + opts = self.write("options.h", "#define HAVE_X 1\n") + argv = self.run_main(["--options-h", opts]) + self.assertIn(("--options-h", opts), list(zip(argv, argv[1:]))) + self.assertNotIn("--source-only", argv) + + def test_user_settings_header_reaches_driver(self): + settings = self.write("zephyr/user_settings.h", "#define HAVE_X 1\n") + argv = self.run_main(["--user-settings", settings], ["-DMOD_DEBUG"]) + self.assertNotIn("--source-only", argv) + with open(os.path.join(self.tmp.name, "stub.opts")) as f: + path, text = f.read().split("\n", 1) + self.assertIn("#define HAVE_X 1", text) + self.assertIn("#define MOD_DEBUG 1", text) + self.assertFalse(os.path.exists(path), "temporary header left behind") + + +if __name__ == "__main__": + unittest.main(verbosity=2)