From 953f2ad4bf74fa7ec4804dcd652641528a405742 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Frauenschl=C3=A4ger?= Date: Mon, 19 Oct 2026 19:21:32 +0200 Subject: [PATCH] fix(zephyr): take module sources and settings from the west build zephyr_sbom.py found sources with a regex over zephyr_library_sources(...). It missed zephyr_library_sources_ifdef(), listed sources inside if(CONFIG_X) blocks whatever Kconfig said, parsed commented-out calls, and silently skipped any ${VAR} other than the three module-dir variables. For a module like wolfCert that meant an SBOM listing the EST and SCEP clients whether or not they were built. wolfssl's module could not be read at all: its source list comes from FIPS-boundary lists, a foreach and IN_LIST checks. The frontend now reads the compile_commands.json CMake writes when west configures the build (--compdb, with --root set to the module). The module's sources are the entries under --root whose object Zephyr placed in the module's library directory, /modules//. The name comes from zephyr/module.yml, else the module directory name, and --module-name overrides it. This follows Kconfig and whatever CMake logic the module uses, and it leaves out application files inside the module tree, such as wolfCert's test app, because those compile into app.dir. Library sources outside --root are reported and excluded. It needs the "output" field CMake 3.20 and later write. Reading the CMakeLists without a build is gone. A .config only exists in a configured build, which already has compile_commands.json, and without one any Kconfig-gated source had to be refused. A static reader also cannot follow list(APPEND), include() or similar, and returned wrong lists for them. No product runs the frontend today; wolfCert and wolfTPM only ship it. --user-settings records the module's configuration: the -D flags of its compile command plus the macros its settings header defines. It runs the compile command's own compiler and target flags with -dM -E twice, once without the -D flags and the header and once with -D flags and -include of the header, and keeps what changed. Compiler built-ins and the Kconfig symbols from autoconf.h cancel out. The build's compiler is used because the host one answers target checks wrongly: wolfssl's Zephyr user_settings.h tests __SIZEOF_INT128__ under CONFIG_WOLFCRYPT_ASM, and an aarch64 host recorded HAVE___UINT128_T for a Cortex-M7 build. --hostcc switches to a host compiler with only the -D, -U, -I, -iquote and autoconf.h flags. Zephyr-wide -D flags such as KERNEL are recorded as well; filtering them out would also drop module flags set with zephyr_compile_definitions(). The -D and -U flags of every C compile command in the module are merged, including a file built by two targets; a macro that one command defines and another undefines or defines differently stops the run. A settings header the command already force-includes is taken out of the first run so its macros still show, and a header that adds no macros at all, for example because a copy of it is force-included, stops the run. Response files are refused, and "command" strings are split with the Windows rules on Windows. Checked on west builds (Zephyr 4.4.1) of wolfCert's unit smoke test on qemu_x86 with EST, with SCEP and with CONFIG_WOLFSSL_DEBUG, and of wolfssl's wolfssl_test sample on qemu_x86 and on nucleo_h743zi with CONFIG_WOLFCRYPT_ASM. The source sets matched the compiled sets every time (wolfssl 101 files, 109 with the Cortex-M assembly; wolfCert 14). The settings capture equalled one taken by hand with the target compiler and recorded DEBUG_WOLFSSL when set. The SBOMs validate, contain no host paths and are byte-reproducible. The frontend also forwards --license-override, --license-text, --supplier, --component-type, --dep-wolfssl, --dep-wolfcrypt, --dep-openssl and --dep-version, and accepts --options-h in place of --source-only. Before, these SBOMs fell back to the inferred licence (GPL-3.0-only for a verbatim GPLv3 file) and recorded no wolfSSL dependency. The docstring and both READMEs describe the CMake helper, called with TARGETS ${ZEPHYR_CURRENT_LIBRARY} after the module's last zephyr_library_sources(), as the accurate route and this frontend as the one without a hook. New tests/test_zephyr_sbom.py covers the module name, the compile_commands.json selection, the settings capture in both compiler modes, the error paths and flag forwarding through a stub driver; selftest.yml runs it. --- .github/workflows/selftest.yml | 4 + README.md | 2 +- share/README.md | 9 +- share/frontends/zephyr_sbom.py | 428 ++++++++++++++++++++++++++------- tests/test_zephyr_sbom.py | 411 +++++++++++++++++++++++++++++++ 5 files changed, 771 insertions(+), 83 deletions(-) create mode 100644 tests/test_zephyr_sbom.py diff --git a/.github/workflows/selftest.yml b/.github/workflows/selftest.yml index e646523..37bf8f6 100644 --- a/.github/workflows/selftest.yml +++ b/.github/workflows/selftest.yml @@ -62,6 +62,7 @@ jobs: tests/test_iar_sbom.py \ tests/test_included_sources.py \ tests/test_compdb_sbom.py \ + tests/test_zephyr_sbom.py \ tests/test_sbom_cmake_install.py \ tests/test_wolfssl_compliance_tooling_sync.py \ tests/test_sbom_autotools.py @@ -93,6 +94,9 @@ jobs: - name: Run compdb frontend tests run: python -m unittest tests/test_compdb_sbom.py + - name: Run Zephyr frontend tests + run: python -m unittest tests/test_zephyr_sbom.py + - name: Run wolfssl-compliance-tooling-sync tests run: python -m unittest tests/test_wolfssl_compliance_tooling_sync.py diff --git a/README.md b/README.md index 145cd41..5d95f13 100644 --- a/README.md +++ b/README.md @@ -103,7 +103,7 @@ system it ships) while all of them converge on the same engine. | `build/sbom.mk` | Plain Make | Product sets `SBOM_NAME`/`SBOM_SRCS`/`SBOM_CFLAGS` (plus `SBOM_SETTINGS_H` when a `user_settings.h` derives the config), includes the fragment. | | `frontends/compdb_sbom.py` | Any `compile_commands.json` | The universal fallback — TI CCS, MPLAB X, Renesas e2studio, or anything wrappable with `bear -- make ...`. | | `frontends/iar_sbom.py` | IAR Embedded Workbench `.ewp` | Parses the project XML directly — there's no build step to hook into. | -| `frontends/zephyr_sbom.py` | Zephyr module | Reads `zephyr_library_sources(...)` since west owns the actual build. | +| `frontends/zephyr_sbom.py` | Zephyr module | Reads the module's sources from a west build's `compile_commands.json`, since west owns the actual build. | ## Products can occupy more than one tier diff --git a/share/README.md b/share/README.md index 9313c1a..b88d114 100644 --- a/share/README.md +++ b/share/README.md @@ -14,7 +14,7 @@ into a product at `tools/sbom/`, together with the pin files (`VERSION` and | `frontends/compdb_sbom.py` | Extractor for any `compile_commands.json`. | | `frontends/iar_sbom.py` | Extractor for an IAR Embedded Workbench `.ewp`. | | `frontends/included_sources.py` | Finds the source files a compile command compiles in through `#include` (`compdb_sbom.py --included-sources`). | -| `frontends/zephyr_sbom.py` | Extractor for a Zephyr module `CMakeLists.txt`. | +| `frontends/zephyr_sbom.py` | Zephyr module, from a west build's `compile_commands.json`. | | `build/sbom.mk` | Shared plain-Make fragment and `wolfssl_compliance_tooling_sbom_rule` macro. | | `build/sbom.cmake` | Shared CMake helper: `wolfssl_compliance_tooling_add_sbom()`. | | `gen-sbom` | The vendored SBOM generator. | @@ -84,6 +84,13 @@ A product does not copy logic. It describes itself: `SBOM_GEN` is the canonical generator override; `GEN_SBOM` remains a legacy alias for compatibility. - Autotools: set the `SBOM_*` variables and `include tools/sbom/sbom.am`. +- Zephyr: call `wolfssl_compliance_tooling_add_sbom(TARGETS + ${ZEPHYR_CURRENT_LIBRARY} ...)` from the module's own CMakeLists, after its + last `zephyr_library_sources()`, so the SBOM records the source set west + actually built. Without that hook, run `zephyr_sbom.py` on the build's + `compile_commands.json` with `--root` set to the module, and pass + `--user-settings` with the module's settings header to record its + configuration. Keep only true product knowledge in the product: the route-through script, the module extractor, and the HAL source selector. diff --git a/share/frontends/zephyr_sbom.py b/share/frontends/zephyr_sbom.py index 2813d1d..c06c566 100755 --- a/share/frontends/zephyr_sbom.py +++ b/share/frontends/zephyr_sbom.py @@ -1,26 +1,36 @@ #!/usr/bin/env python3 -"""Generate a source-inventory SBOM for a Zephyr module. +"""Generate an SBOM for a Zephyr module from a west build. -This is the product-neutral form of wolfBoot's zephyr_sbom.py. A Zephyr module -compiles its sources into a Zephyr application through `zephyr_library_sources(...)`, -built by Zephyr/west rather than by the product's own Makefile or CMake. So -neither the Make nor the CMake SBOM target sees those sources. +A Zephyr module compiles its sources into a Zephyr application through +`zephyr_library_sources(...)`, built by Zephyr/west rather than by the +product's own Makefile or CMake. So neither the Make nor the CMake SBOM target +sees those sources. -This extractor reads the module's source list straight out of its CMakeLists (so -it stays in sync automatically) and hands it to the shared driver. +The most accurate route is to call the CMake helper from the module's own +CMakeLists after its last `zephyr_library_sources()`, +`wolfssl_compliance_tooling_add_sbom(TARGETS ${ZEPHYR_CURRENT_LIBRARY} ...)`. +Without that hook, this frontend reads the compile_commands.json that CMake +writes when west configures the build. The module's sources are the entries +under `--root` whose object Zephyr placed in the module's library directory, +`/modules//`. That set follows Kconfig and any CMake logic +the module uses, and it leaves out application files inside the module tree. -The module configuration is Kconfig-driven (CONFIG_* symbols), not a `-D` macro -set, so by default this produces a source-inventory SBOM. If you have a real -Zephyr build and want the exact compiled config, generate the SBOM from that -build's compilation database with compdb_sbom.py instead. +The configuration is Kconfig-driven, so by default this produces a +source-inventory SBOM. `--user-settings` records the module's -D flags and the +macros its settings header defines, preprocessed by the build's own compiler +with the module's compile command. `--options-h` records a flat `#define` +header instead. Usage: - zephyr_sbom.py --name NAME --cmakelists path/to/zephyr/CMakeLists.txt [options] + zephyr_sbom.py --name NAME --compdb build/compile_commands.json \\ + --root MODULE [--user-settings HEADER] [options] """ import argparse +import json import os import re +import shlex import subprocess import sys import tempfile @@ -29,38 +39,221 @@ DEFAULT_DRIVER = os.path.join(os.path.dirname(SCRIPT_DIR), "sbom-driver") SRC_EXTS = ('.c', '.cc', '.cpp', '.cxx', '.s', '.asm') +_ARG_OPTS = ('-D', '-U', '-I', '-iquote', '-isystem', '-idirafter', + '-imacros', '-include', '-o', '-MF', '-MT', '-MQ', '-x') +_NO_DUMP = ('-o', '-MF', '-MT', '-MQ', '-x', '-c', '-M', '-MM', '-MD', '-MMD', + '-MP') +_HOST_OPTS = ('-D', '-U', '-I', '-iquote') -def parse_library_sources(cmakelists): - """Extract the file list from the zephyr_library_sources(...) blocks.""" - with open(cmakelists) as f: - text = f.read() - module_dir = os.path.dirname(os.path.abspath(cmakelists)) +class ParseError(Exception): + pass - srcs = [] - for m in re.finditer(r'zephyr_library_sources\s*\((.*?)\)', text, re.DOTALL): - for raw in m.group(1).split(): - raw = raw.strip() - if not raw or not raw.lower().endswith(SRC_EXTS): - continue - # Resolve the common CMake module-directory variables. - p = raw.replace('${CMAKE_CURRENT_LIST_DIR}', module_dir) - p = p.replace('${CMAKE_CURRENT_SOURCE_DIR}', module_dir) - p = p.replace('${ZEPHYR_CURRENT_MODULE_DIR}', os.path.dirname(module_dir)) - if '${' in p: - sys.stderr.write(f"WARNING: skipping unresolved source: {raw}\n") + +def module_name(root): + """The Zephyr module name: module.yml's name, else the directory name.""" + for yml in ('module.yml', 'module.yaml'): + path = os.path.join(root, 'zephyr', yml) + if os.path.isfile(path): + try: + with open(path, encoding='utf-8') as f: + lines = f.readlines() + except (OSError, UnicodeDecodeError) as e: + raise ParseError(f"cannot read {path}: {e}") + for line in lines: + m = re.match(r'name:\s*([^\s#]+)', line) + if m: + return m.group(1).strip('"\'') + break + return os.path.basename(os.path.normpath(root)) + + +def compdb_entries(compdb, root, name): + """Return the module library's compile commands as [(source, entry)] + for sources under root, plus the library sources outside it.""" + try: + with open(compdb) as f: + db = json.load(f) + except (OSError, json.JSONDecodeError) as e: + raise ParseError(f"cannot read {compdb}: {e}") + root = os.path.realpath(root) + os.sep + picked, outside, seen = [], [], set() + for e in db: + if 'output' not in e: + raise ParseError(f"{compdb} has no 'output' field; it needs " + "CMake 3.20 or later") + if not e.get('file'): + continue + directory = e.get('directory', '') + lib_dir = os.path.normpath( + os.path.join(directory, 'modules', name)) + os.sep + out = os.path.normpath(os.path.join(directory, e['output'])) + src = os.path.normpath(os.path.join(directory, e['file'])) + if (not out.startswith(lib_dir) + or not src.lower().endswith(SRC_EXTS)): + continue + if os.path.realpath(src).startswith(root): + picked.append((src, e)) + elif src not in seen: + seen.add(src) + outside.append(src) + return picked, outside + + +def split_windows_command(cmd): + """Split a command line the way CommandLineToArgvW does.""" + args, cur, have, quoted, i = [], [], False, False, 0 + while i < len(cmd): + c = cmd[i] + if c == '\\': + n = len(cmd[i:]) - len(cmd[i:].lstrip('\\')) + i += n + if cmd[i:i + 1] == '"': + cur.append('\\' * (n // 2)) + if n % 2: + cur.append('"') + i += 1 + else: + cur.append('\\' * n) + have = True + elif c == '"': + quoted, have, i = not quoted, True, i + 1 + elif c in ' \t' and not quoted: + if have: + args.append(''.join(cur)) + cur, have = [], False + i += 1 + else: + cur.append(c) + have, i = True, i + 1 + if have: + args.append(''.join(cur)) + return args + + +def _entry_tokens(entry): + if entry.get('arguments'): + toks = list(entry['arguments']) + else: + cmd = entry.get('command', '') + try: + toks = (split_windows_command(cmd) if os.name == 'nt' + else shlex.split(cmd)) + except ValueError as e: + raise ParseError(f"cannot split the compile command of " + f"{entry.get('file')}: {e}") + if not toks: + raise ParseError(f"empty compile command for {entry.get('file')}") + return toks + + +def _dump_macros(cc, flags, directory): + try: + r = subprocess.run([cc, '-dM', '-E', *flags, '-x', 'c', os.devnull], + cwd=directory or None, capture_output=True, + text=True) + except OSError as e: + raise ParseError(f"cannot run '{cc}': {e}") + if r.returncode: + raise ParseError(f"'{cc} -dM -E' failed:\n{r.stderr.strip()}") + macros = {} + for line in r.stdout.splitlines(): + m = re.match(r'#define (\S+)', line) + if m: + macros[m.group(1)] = line + return macros + + +def _options(entry): + """Yield (option, argument) pairs of a compile command after the + compiler, with argument None for a flag and option None for a source.""" + toks = _entry_tokens(entry) + i = 1 + while i < len(toks): + t = toks[i] + if t.startswith('@'): + raise ParseError(f"response file {t} in the compile command of " + f"{entry.get('file')} is not supported") + if t in _ARG_OPTS and i + 1 < len(toks): + yield t, toks[i + 1] + i += 2 + continue + joined = next((o for o in _ARG_OPTS + if t.startswith(o) and t != o), None) + if joined: + yield joined, t[len(joined):] + elif t.startswith('-'): + yield t, None + else: + yield None, t + i += 1 + + +def _same_file(path, directory, settings_h): + full = os.path.join(directory or '', path) + return os.path.realpath(full) == os.path.realpath(settings_h) + + +def _defines(entries): + """Merge the net -D/-U state of every entry into one -D/-U list, or + raise ParseError when two entries disagree on a macro.""" + merged = {} + for src, entry in entries: + state = {} + for opt, arg in _options(entry): + if arg is None: continue - if not os.path.isabs(p): - p = os.path.join(module_dir, p) - srcs.append(os.path.normpath(p)) + if opt == '-D': + name = re.split(r'[=(]', arg, maxsplit=1)[0] + state[name] = arg if '=' in arg else arg + '=1' + elif opt == '-U': + state[arg] = None + for name, value in state.items(): + if name in merged and merged[name][0] != value: + raise ParseError( + f"the module's compile commands disagree on {name}: " + f"{merged[name][1]} and {src}") + merged.setdefault(name, (value, src)) + flags = [] + for name in sorted(merged): + value = merged[name][0] + flags += ['-U', name] if value is None else ['-D', value] + return flags - seen, present, missing = set(), [], [] - for s in srcs: - if s in seen: + +def capture_user_settings(entries, settings_h, hostcc=None): + """Return the #define lines the module's -D flags and settings_h add. + + The capture runs the first C entry's compiler and flags, with the -D and + -U flags merged across all C entries. With hostcc it runs that compiler + with only the -D, -U, -I, -iquote and autoconf.h flags.""" + c_entries = [(src, e) for src, e in entries + if not src.lower().endswith(('.s', '.asm'))] or entries + entry = c_entries[0][1] + directory = entry.get('directory') + base = [] + for opt, arg in _options(entry): + if opt is None or opt in _NO_DUMP or opt in ('-D', '-U'): + continue + if opt in ('-include', '-imacros') and _same_file(arg, directory, + settings_h): + continue + if hostcc and opt not in _HOST_OPTS and not ( + opt == '-imacros' and os.path.basename(arg) == 'autoconf.h'): continue - seen.add(s) - (present if os.path.isfile(s) else missing).append(s) - return present, missing + base += [opt] if arg is None else [opt, arg] + cc = hostcc or _entry_tokens(entry)[0] + defs = _defines(c_entries) + before = _dump_macros(cc, base, directory) + with_defs = _dump_macros(cc, base + defs, directory) + after = _dump_macros( + cc, base + defs + ['-include', os.path.abspath(settings_h)], + directory) + if after == with_defs: + raise ParseError(f"{settings_h} adds no macros; the compile command " + f"may already include it or a copy of it") + return sorted(line for name, line in after.items() + if before.get(name) != line) def main(): @@ -68,19 +261,40 @@ def main(): description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument('--name', required=True, help='Package name for the SBOM.') - ap.add_argument('--cmakelists', required=True, - help='Path to the module CMakeLists.txt.') + ap.add_argument('--compdb', required=True, + help="The west build's compile_commands.json.") + ap.add_argument('--root', required=True, help='The module directory.') + ap.add_argument('--module-name', default=None, + help='Zephyr module name. Default: the name in ' + 'zephyr/module.yml, else the --root directory name.') + ap.add_argument('--user-settings', default=None, + help="The module's settings header, captured with the " + "module's compile flags.") + ap.add_argument('--hostcc', default=None, + help='Capture --user-settings with this host compiler ' + 'instead of the build\'s compiler and target flags.') + ap.add_argument('--options-h', default=None, + help='Flat #define header to record as the build ' + 'configuration, instead of --source-only.') ap.add_argument('--driver', default=DEFAULT_DRIVER) ap.add_argument('--gen-sbom', default=None) ap.add_argument('--version', default=None) ap.add_argument('--version-file', default=None) ap.add_argument('--version-macro', default=None) - ap.add_argument('--root', default=None) + ap.add_argument('--supplier', default=None) + ap.add_argument('--component-type', default=None) ap.add_argument('--license-file', default=None) ap.add_argument('--copyright-start', default=None) - # This front end is --source-only: no macros are captured, so the - # generator cannot read WOLFCRYPT_ONLY and will say so unless the - # operator states it here. + ap.add_argument('--license-override', default=None) + ap.add_argument('--license-text', default=None) + ap.add_argument('--dep-wolfssl', default=None) + ap.add_argument('--dep-wolfcrypt', default=None) + ap.add_argument('--dep-openssl', default=None) + ap.add_argument('--dep-version', action='append', default=[], + help='Dependency version, e.g. wolfssl=5.9.2 ' + '(repeatable).') + # In source-only mode the generator cannot read WOLFCRYPT_ONLY and will + # say so unless the operator states it here. ap.add_argument('--crypto-only', default=None, help='auto|yes|no, forwarded to the generator.') ap.add_argument('--cdx-out', default=None) @@ -89,54 +303,106 @@ def main(): ap.add_argument('--print-only', action='store_true') args = ap.parse_args() - if not os.path.isfile(args.cmakelists): - sys.exit(f"ERROR: CMakeLists not found: {args.cmakelists}") + if args.user_settings and args.options_h: + sys.exit("ERROR: pass only one of --user-settings and --options-h.") + if args.hostcc and not args.user_settings: + sys.exit("ERROR: --hostcc only applies to --user-settings.") + for path, what in ((args.compdb, 'compilation database'), + (args.user_settings, '--user-settings')): + if path and not os.path.isfile(path): + sys.exit(f"ERROR: {what} not found: {path}") - srcs, missing = parse_library_sources(args.cmakelists) - if missing: + settings = None + try: + name = args.module_name or module_name(args.root) + entries, outside = compdb_entries(args.compdb, args.root, name) + if not entries: + sys.exit(f"ERROR: {args.compdb} has no compile commands for " + f"module '{name}' from sources under {args.root}; " + f"check --root and --module-name.") + if args.user_settings: + settings = capture_user_settings(entries, args.user_settings, + args.hostcc) + except ParseError as e: + sys.exit(f"ERROR: {e}") + if outside: sys.stderr.write( - f"WARNING: {len(missing)} module source(s) not found on disk " - f"(excluded):\n") - for m in missing: - sys.stderr.write(f" - {m}\n") - if not srcs: - sys.exit("ERROR: no zephyr_library_sources found in " + args.cmakelists) + f"WARNING: {len(outside)} source(s) of module '{name}' lie " + f"outside --root (excluded):\n") + for s in outside: + sys.stderr.write(f" - {s}\n") + srcs = list(dict.fromkeys(src for src, _ in entries)) if args.print_only: print(f"# {args.name}: {len(srcs)} sources") for s in srcs: print(f" {s}") + if settings is not None: + print(f"\n# {len(settings)} macros from {args.user_settings}") + for line in settings: + print(f" {line}") return - srcs_out, tmp = args.srcs_out, None - if not srcs_out: - fd, srcs_out = tempfile.mkstemp(prefix='wolfssl-compliance-tooling-zephyr-srcs-', suffix='.txt') - os.close(fd) - tmp = srcs_out - with open(srcs_out, 'w') as f: - f.write('\n'.join(srcs) + '\n') - - cmd = [args.driver, '--srcs-file', srcs_out, '--source-only', - '--name', args.name] - for flag, val in (('--version', args.version), - ('--version-file', args.version_file), - ('--version-macro', args.version_macro), - ('--root', args.root), - ('--license-file', args.license_file), - ('--copyright-start', args.copyright_start), - ('--crypto-only', args.crypto_only), - ('--gen-sbom', args.gen_sbom), - ('--cdx-out', args.cdx_out), - ('--spdx-out', args.spdx_out)): - if val: - cmd += [flag, val] - - print(f"Zephyr module SBOM: {len(srcs)} sources (source-inventory)") + tmps = [] try: - rc = subprocess.call(cmd) + srcs_out = args.srcs_out + if not srcs_out: + fd, srcs_out = tempfile.mkstemp( + prefix='wolfssl-compliance-tooling-zephyr-srcs-', + suffix='.txt') + os.close(fd) + tmps.append(srcs_out) + try: + with open(srcs_out, 'w') as f: + f.write('\n'.join(srcs) + '\n') + except OSError as e: + sys.exit(f"ERROR: cannot write {srcs_out}: {e}") + + cmd = [args.driver, '--srcs-file', srcs_out, '--name', args.name, + '--root', args.root] + if settings is not None: + fd, opts = tempfile.mkstemp( + prefix='wolfssl-compliance-tooling-zephyr-defs-', suffix='.h') + tmps.append(opts) + with os.fdopen(fd, 'w') as f: + f.write('\n'.join(settings) + '\n') + cmd += ['--options-h', opts] + elif args.options_h: + cmd += ['--options-h', args.options_h] + else: + cmd += ['--source-only'] + for flag, val in (('--version', args.version), + ('--version-file', args.version_file), + ('--version-macro', args.version_macro), + ('--supplier', args.supplier), + ('--component-type', args.component_type), + ('--license-file', args.license_file), + ('--copyright-start', args.copyright_start), + ('--license-override', args.license_override), + ('--license-text', args.license_text), + ('--dep-wolfssl', args.dep_wolfssl), + ('--dep-wolfcrypt', args.dep_wolfcrypt), + ('--dep-openssl', args.dep_openssl), + ('--crypto-only', args.crypto_only), + ('--gen-sbom', args.gen_sbom), + ('--cdx-out', args.cdx_out), + ('--spdx-out', args.spdx_out)): + if val: + cmd += [flag, val] + for dv in args.dep_version: + cmd += ['--dep-version', dv] + + kind = ("user-settings" if settings is not None + else "options-h" if args.options_h else "source-inventory") + print(f"Zephyr module SBOM: {len(srcs)} sources ({kind})") + try: + rc = subprocess.call(cmd) + except OSError as e: + sys.exit(f"ERROR: cannot run {args.driver}: {e}") finally: - if tmp and os.path.exists(tmp): - os.remove(tmp) + for t in tmps: + if os.path.exists(t): + os.remove(t) sys.exit(rc) diff --git a/tests/test_zephyr_sbom.py b/tests/test_zephyr_sbom.py new file mode 100644 index 0000000..85c6824 --- /dev/null +++ b/tests/test_zephyr_sbom.py @@ -0,0 +1,411 @@ +#!/usr/bin/env python3 +"""Tests for the Zephyr frontend (share/frontends/zephyr_sbom.py). + +Focus: the source set must be exactly what the west build compiled into the +module's library, and the settings capture must record the module's +configuration without Kconfig symbols or host built-ins.""" + +import importlib.util +import json +import os +import shlex +import subprocess +import sys +import tempfile +import unittest + +REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +ZEPHYR = os.path.join(REPO, "share", "frontends", "zephyr_sbom.py") + +_spec = importlib.util.spec_from_file_location("zephyr_sbom", ZEPHYR) +zephyr = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(zephyr) + +LIB = "modules/mod/CMakeFiles/..__mod__zephyr.dir/" +APP = "CMakeFiles/app.dir/" + +# Stands in for sbom-driver: writes its argv, and the --options-h header it +# was handed, to files named by STUB_OUT. +_STUB = """import os, sys +out = os.environ['STUB_OUT'] +with open(out + '.argv', 'w') as f: + f.write('\\n'.join(sys.argv[1:])) +if '--options-h' in sys.argv: + src = sys.argv[sys.argv.index('--options-h') + 1] + with open(src) as f, open(out + '.opts', 'w') as g: + g.write(src + '\\n' + f.read()) +""" + + +class _ZephyrTest(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.mod = os.path.join(self.tmp.name, "mod") + self.build = os.path.join(self.tmp.name, "build") + os.makedirs(os.path.join(self.mod, "zephyr")) + os.makedirs(self.build) + self.write("zephyr/module.yml", "name: mod\n") + for rel in ("src/core.c", "src/est.c", "src/scep.c", "zephyr/glue.c", + "tests/app.c"): + self.write(rel, "") + self.outside = os.path.join(self.tmp.name, "other.c") + open(self.outside, "w").close() + + def tearDown(self): + self.tmp.cleanup() + + def write(self, rel, text): + path = os.path.join(self.mod, rel) + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w") as f: + f.write(text) + return path + + def write_compdb(self, entries, flags=()): + db = [{"directory": self.build, + "file": src if os.path.isabs(src) + else os.path.join(self.mod, src), + "arguments": ["cc", *flags, "-c", src], + "output": out + os.path.basename(src) + ".obj"} + for src, out in entries] + path = os.path.join(self.build, "compile_commands.json") + with open(path, "w") as f: + json.dump(db, f) + return path + + def run_frontend(self, *extra): + return subprocess.run( + [sys.executable, ZEPHYR, "--name", "mod", "--root", self.mod, + *extra], capture_output=True, text=True, + env=dict(os.environ, STUB_OUT=os.path.join(self.tmp.name, "stub"))) + + def rels(self, entries): + return sorted(os.path.relpath(src, self.mod) for src, _ in entries) + + +class TestModuleName(_ZephyrTest): + def test_module_yml_name(self): + self.write("zephyr/module.yml", + "build:\n cmake: zephyr\nname: 'mod' # pinned\n") + self.assertEqual(zephyr.module_name(self.mod), "mod") + + def test_directory_name_without_module_yml_name(self): + self.write("zephyr/module.yml", "build:\n cmake: zephyr\n") + self.assertEqual(zephyr.module_name(self.mod), "mod") + os.rename(self.mod, self.mod + "-x") + self.assertEqual(zephyr.module_name(self.mod + "-x"), "mod-x") + + +class TestSourceSelection(_ZephyrTest): + def test_only_module_library_sources_under_root(self): + compdb = self.write_compdb([ + ("src/core.c", LIB), ("src/est.c", LIB), ("zephyr/glue.c", LIB), + ("src/core.c", LIB), + ("tests/app.c", APP), + ("src/scep.c", "modules/mod-extra/CMakeFiles/x.dir/"), + (self.outside, LIB)]) + entries, outside = zephyr.compdb_entries(compdb, self.mod, "mod") + self.assertEqual(self.rels(entries), + ["src/core.c", "src/core.c", "src/est.c", + "zephyr/glue.c"]) + self.assertEqual(outside, [self.outside]) + + def test_unnormalized_build_directory(self): + compdb = self.write_compdb([("src/core.c", LIB)]) + with open(compdb) as f: + db = json.load(f) + db[0]["directory"] = self.build + "/./" + with open(compdb, "w") as f: + json.dump(db, f) + entries, _ = zephyr.compdb_entries(compdb, self.mod, "mod") + self.assertEqual(self.rels(entries), ["src/core.c"]) + + def test_makefile_generator_layout(self): + compdb = self.write_compdb([("src/core.c", LIB)]) + with open(compdb) as f: + db = json.load(f) + db[0]["directory"] = os.path.join(self.build, "modules", "mod") + with open(compdb, "w") as f: + json.dump(db, f) + entries, _ = zephyr.compdb_entries(compdb, self.mod, "mod") + self.assertEqual(self.rels(entries), ["src/core.c"]) + + def test_missing_output_field_fails(self): + compdb = self.write_compdb([("src/core.c", LIB)]) + with open(compdb) as f: + db = json.load(f) + del db[0]["output"] + with open(compdb, "w") as f: + json.dump(db, f) + with self.assertRaises(zephyr.ParseError): + zephyr.compdb_entries(compdb, self.mod, "mod") + + def test_no_module_entries_fails(self): + compdb = self.write_compdb([("src/core.c", APP)]) + r = self.run_frontend("--compdb", compdb, "--print-only") + self.assertNotEqual(r.returncode, 0) + self.assertIn("--module-name", r.stderr) + + def test_source_outside_root_is_warned(self): + compdb = self.write_compdb([("src/core.c", LIB), (self.outside, LIB)]) + r = self.run_frontend("--compdb", compdb, "--print-only") + self.assertEqual(r.returncode, 0, r.stderr) + self.assertIn("outside --root", r.stderr) + self.assertNotIn("other.c", r.stdout) + + +class TestCommandSplitting(unittest.TestCase): + def test_windows_rules(self): + cmd = ('C:\\sdk\\gcc.exe -DSTR=\\"x\\" -DX="a b" ' + '-I"C:\\My Dir" -c a.c') + self.assertEqual(zephyr.split_windows_command(cmd), [ + "C:\\sdk\\gcc.exe", '-DSTR="x"', "-DX=a b", "-IC:\\My Dir", + "-c", "a.c"]) + self.assertEqual(zephyr.split_windows_command('a\\\\"b c" d'), + ["a\\b c", "d"]) + + +class TestDefines(unittest.TestCase): + def entry(self, *flags): + return ("x.c", {"arguments": ["cc", "-c", "x.c", *flags]}) + + def test_bare_define_equals_value_one_and_last_wins(self): + self.assertEqual( + zephyr._defines([self.entry("-DA", "-UB", "-DB=2", "-D"), + self.entry("-DA=1")]), + ["-D", "A=1", "-D", "B=2"]) + + def test_define_and_undefine_disagree(self): + with self.assertRaises(zephyr.ParseError): + zephyr._defines([self.entry("-DA"), self.entry("-UA")]) + + +class TestUserSettings(_ZephyrTest): + def setUp(self): + super().setUp() + self.autoconf = os.path.join(self.build, "autoconf.h") + with open(self.autoconf, "w") as f: + f.write("#define CONFIG_MOD_X 1\n") + self.toolchain = os.path.join(self.build, "toolchain.h") + with open(self.toolchain, "w") as f: + f.write("#define TOOLCHAIN_ONLY 1\n") + self.write("cfg/extra.h", "#define HAVE_Y 2\n") + self.settings = self.write( + "zephyr/user_settings.h", + "#ifdef CONFIG_MOD_X\n#define HAVE_X 1\n#endif\n" + "#ifdef MOD_DEBUG\n#define HAVE_DEBUG_LOG 1\n#endif\n" + "#ifdef TOOLCHAIN_ONLY\n#define SAW_TOOLCHAIN 1\n#endif\n" + "#ifdef __STDC_VERSION__\n#define HAVE_C99 1\n#endif\n" + "#include SETTINGS_FILE\n") + self.flags = ["-std=c89", "-DMOD_DEBUG", '-DSETTINGS_FILE="extra.h"', + "-imacros", self.autoconf, f"-imacros{self.toolchain}", + "-I", os.path.join(self.mod, "cfg"), + "-o", "core.c.obj", "-MD", "-MF", "core.c.d"] + + def capture(self, flags, *extra): + compdb = self.write_compdb([("src/core.c", LIB)], flags) + r = self.run_frontend("--compdb", compdb, "--print-only", + "--user-settings", self.settings, *extra) + self.assertEqual(r.returncode, 0, r.stderr) + return r.stdout.split("macros from", 1)[1] + + def test_build_compiler_and_flags(self): + macros = self.capture(self.flags) + for line in ("#define HAVE_X 1", "#define HAVE_Y 2", + "#define HAVE_DEBUG_LOG 1", "#define MOD_DEBUG 1", + '#define SETTINGS_FILE "extra.h"', + "#define SAW_TOOLCHAIN 1"): + self.assertIn(line, macros) + for name in ("CONFIG_MOD_X", "TOOLCHAIN_ONLY", "HAVE_C99", + "__STDC__", "__VERSION__"): + self.assertNotIn(name, macros) + + def test_hostcc_keeps_only_portable_flags(self): + macros = self.capture(self.flags, "--hostcc", "cc") + for line in ("#define HAVE_X 1", "#define HAVE_Y 2", + "#define MOD_DEBUG 1", "#define HAVE_C99 1"): + self.assertIn(line, macros) + self.assertNotIn("SAW_TOOLCHAIN", macros) + + def test_joined_flag_forms(self): + flags = ["-DMOD_DEBUG", '-DSETTINGS_FILE="extra.h"', + f"-imacros{self.autoconf}", + f"-I{os.path.join(self.mod, 'cfg')}"] + macros = self.capture(flags) + self.assertIn("#define HAVE_X 1", macros) + self.assertIn("#define HAVE_Y 2", macros) + + def test_command_string_form(self): + compdb = self.write_compdb([("src/core.c", LIB)], self.flags) + with open(compdb) as f: + db = json.load(f) + db[0]["command"] = shlex.join(db[0].pop("arguments")) + with open(compdb, "w") as f: + json.dump(db, f) + r = self.run_frontend("--compdb", compdb, "--print-only", + "--user-settings", self.settings) + self.assertEqual(r.returncode, 0, r.stderr) + self.assertIn("#define MOD_DEBUG 1", r.stdout) + self.assertIn('#define SETTINGS_FILE "extra.h"', r.stdout) + + def test_force_included_header_is_still_recorded(self): + macros = self.capture(self.flags + ["-include", self.settings]) + self.assertIn("#define HAVE_X 1", macros) + self.assertIn("#define HAVE_Y 2", macros) + + def test_defines_merge_across_entries(self): + compdb = self.write_compdb([("src/core.c", LIB)], self.flags) + with open(compdb) as f: + db = json.load(f) + other = dict(db[0], file=os.path.join(self.mod, "src/est.c"), + output=LIB + "est.c.obj", + arguments=db[0]["arguments"] + ["-DEST_ONLY", + "-UMOD_DEBUG"]) + with open(compdb, "w") as f: + json.dump(db + [other], f) + r = self.run_frontend("--compdb", compdb, "--print-only", + "--user-settings", self.settings) + self.assertNotEqual(r.returncode, 0) + self.assertIn("disagree on MOD_DEBUG", r.stderr) + other["arguments"] = other["arguments"][:-1] + with open(compdb, "w") as f: + json.dump(db + [other], f) + r = self.run_frontend("--compdb", compdb, "--print-only", + "--user-settings", self.settings) + self.assertEqual(r.returncode, 0, r.stderr) + self.assertIn("#define EST_ONLY 1", r.stdout) + self.assertIn("#define MOD_DEBUG 1", r.stdout) + + def test_same_source_in_two_targets_disagrees(self): + compdb = self.write_compdb([("src/core.c", LIB)], self.flags) + with open(compdb) as f: + db = json.load(f) + other = dict(db[0], output=LIB.replace("mod__zephyr", "mod_b") + + "core.c.obj", + arguments=db[0]["arguments"] + ["-UMOD_DEBUG"]) + with open(compdb, "w") as f: + json.dump(db + [other], f) + r = self.run_frontend("--compdb", compdb, "--print-only", + "--user-settings", self.settings) + self.assertNotEqual(r.returncode, 0) + self.assertIn("disagree on MOD_DEBUG", r.stderr) + r = self.run_frontend("--compdb", compdb, "--print-only") + self.assertEqual(r.returncode, 0, r.stderr) + self.assertIn("# mod: 1 sources", r.stdout) + + def test_header_already_included_as_copy_fails(self): + text = "#ifndef US_H\n#define US_H\n#define HAVE_X 1\n#endif\n" + self.write("zephyr/user_settings.h", text) + copy = os.path.join(self.build, "user_settings.h") + with open(copy, "w") as f: + f.write(text) + compdb = self.write_compdb([("src/core.c", LIB)], + ["-include", copy]) + r = self.run_frontend("--compdb", compdb, "--print-only", + "--user-settings", self.settings) + self.assertNotEqual(r.returncode, 0) + self.assertIn("adds no macros", r.stderr) + + def test_asm_entry_first_is_skipped(self): + self.write("src/start.S", "") + compdb = self.write_compdb([("src/start.S", LIB), + ("src/core.c", LIB)], self.flags) + with open(compdb) as f: + db = json.load(f) + db[0]["arguments"] = ["/nonexistent/as", "-c", "src/start.S"] + with open(compdb, "w") as f: + json.dump(db, f) + r = self.run_frontend("--compdb", compdb, "--print-only", + "--user-settings", self.settings) + self.assertEqual(r.returncode, 0, r.stderr) + self.assertIn("#define HAVE_X 1", r.stdout) + + def test_hostcc_needs_user_settings(self): + compdb = self.write_compdb([("src/core.c", LIB)]) + r = self.run_frontend("--compdb", compdb, "--hostcc", "cc") + self.assertNotEqual(r.returncode, 0) + self.assertIn("--hostcc only applies", r.stderr) + + def test_response_file_fails(self): + compdb = self.write_compdb([("src/core.c", LIB)], ["@flags.rsp"]) + r = self.run_frontend("--compdb", compdb, "--print-only", + "--user-settings", self.settings) + self.assertNotEqual(r.returncode, 0) + self.assertIn("response file", r.stderr) + + def test_missing_compiler_is_an_error(self): + compdb = self.write_compdb([("src/core.c", LIB)], self.flags) + with open(compdb) as f: + db = json.load(f) + db[0]["arguments"][0] = "/nonexistent/cc" + with open(compdb, "w") as f: + json.dump(db, f) + r = self.run_frontend("--compdb", compdb, "--print-only", + "--user-settings", self.settings) + self.assertNotEqual(r.returncode, 0) + self.assertIn("ERROR: cannot run", r.stderr) + self.assertNotIn("Traceback", r.stderr) + + def test_options_h_excludes_user_settings(self): + compdb = self.write_compdb([("src/core.c", LIB)]) + r = self.run_frontend("--compdb", compdb, "--options-h", "o.h", + "--user-settings", self.settings) + self.assertNotEqual(r.returncode, 0) + self.assertIn("only one of", r.stderr) + + +class TestDriverForwarding(_ZephyrTest): + def run_main(self, extra, flags=()): + stub = os.path.join(self.tmp.name, "stub-driver.py") + with open(stub, "w") as f: + f.write(f"#!{sys.executable}\n" + _STUB) + os.chmod(stub, 0o755) + compdb = self.write_compdb([("src/core.c", LIB)], flags) + r = self.run_frontend("--compdb", compdb, "--driver", stub, *extra) + self.assertEqual(r.returncode, 0, r.stderr) + with open(os.path.join(self.tmp.name, "stub.argv")) as f: + return f.read().splitlines() + + def test_identity_license_and_dependency_flags(self): + argv = self.run_main([ + "--license-override", "GPL-3.0-or-later", + "--license-text", "LICENSE.txt", + "--supplier", "Example Inc.", + "--component-type", "firmware", + "--dep-wolfssl", "yes", "--dep-wolfcrypt", "yes", + "--dep-openssl", "no", + "--dep-version", "wolfssl=5.9.2", + "--dep-version", "wolfcrypt=5.9.2"]) + pairs = list(zip(argv, argv[1:])) + for pair in (("--license-override", "GPL-3.0-or-later"), + ("--license-text", "LICENSE.txt"), + ("--supplier", "Example Inc."), + ("--component-type", "firmware"), + ("--dep-wolfssl", "yes"), + ("--dep-wolfcrypt", "yes"), + ("--dep-openssl", "no"), + ("--dep-version", "wolfssl=5.9.2"), + ("--dep-version", "wolfcrypt=5.9.2"), + ("--root", self.mod)): + self.assertIn(pair, pairs) + self.assertIn("--source-only", argv) + + def test_options_h_replaces_source_only(self): + opts = self.write("options.h", "#define HAVE_X 1\n") + argv = self.run_main(["--options-h", opts]) + self.assertIn(("--options-h", opts), list(zip(argv, argv[1:]))) + self.assertNotIn("--source-only", argv) + + def test_user_settings_header_reaches_driver(self): + settings = self.write("zephyr/user_settings.h", "#define HAVE_X 1\n") + argv = self.run_main(["--user-settings", settings], ["-DMOD_DEBUG"]) + self.assertNotIn("--source-only", argv) + with open(os.path.join(self.tmp.name, "stub.opts")) as f: + path, text = f.read().split("\n", 1) + self.assertIn("#define HAVE_X 1", text) + self.assertIn("#define MOD_DEBUG 1", text) + self.assertFalse(os.path.exists(path), "temporary header left behind") + + +if __name__ == "__main__": + unittest.main(verbosity=2)