diff --git a/ebuild/cli/commands.py b/ebuild/cli/commands.py index 6e079035..6d7ecbea 100644 --- a/ebuild/cli/commands.py +++ b/ebuild/cli/commands.py @@ -10,6 +10,7 @@ from __future__ import annotations import os +import re import shutil import subprocess import threading @@ -188,6 +189,95 @@ def note_skipped(name: str, _cause: BaseException) -> None: return package_paths +def _workspace_repo_paths() -> Dict[str, PackagePaths]: + """Include paths for the eos and eboot repos that `ebuild setup` cloned. + + A scaffolded project includes , but the generated build.yaml + carried no path to the headers, so every template failed with + "fatal error: eos/hal.h: No such file or directory" on the first build. + + These are resolved at build time from the cache rather than written into + build.yaml as absolute paths: the path is a fact about this machine, and + build.yaml is a file the developer commits. + + Returns an empty mapping when the cache is absent, so the error a developer + sees stays the missing header rather than a stack trace, and `ebuild setup` + remains the fix. + """ + from ebuild.deps import EBUILD_REPOS_DIR + + paths: Dict[str, PackagePaths] = {} + for name in ("eos", "eboot"): + root = Path(EBUILD_REPOS_DIR) / name + if not root.is_dir(): + continue + # Headers sit at two depths: kernel/include, hal/include ... and + # services/crypto/include, services/ota/include. Both are needed -- + # and live only in the deeper set. + include_dirs = sorted( + {p for pattern in ("include", "*/include", "*/*/include") + for p in root.glob(pattern) if p.is_dir()} + ) + if include_dirs: + lib_dirs, libraries = _cached_repo_libraries(root) + paths[name] = PackagePaths( + include_dirs=include_dirs, + lib_dirs=lib_dirs, + libraries=libraries, + ) + return paths + + +# Where `ebuild` puts the CMake build tree for a cached repo. Kept inside the +# clone so `ebuild setup` remains the only thing that owns ~/.ebuild/repos. +_REPO_BUILD_DIRNAME = "_ebuild" + + +def _cached_repo_libraries(root: Path) -> Tuple[List[Path], List[str]]: + """Static libraries a cached repo offers to projects that `use` it. + + Headers alone are not enough: a scaffolded project compiles against + and then fails at the link step with undefined references. + The repo is a CMake project with no install() rules, so there is nothing + to point a -L at until it has been built once. Build it on demand and + cache the result; subsequent builds reuse the tree. + + Returns ([], []) when the repo cannot be built here — a missing cmake, a + repo that is not a CMake project — so the developer still gets a link + error naming the symbol rather than a stack trace from ebuild. + """ + if not (root / "CMakeLists.txt").is_file(): + return [], [] + + build_dir = root / _REPO_BUILD_DIRNAME + archives = sorted(build_dir.rglob("*.a")) if build_dir.is_dir() else [] + + if not archives: + if shutil.which("cmake") is None: + return [], [] + try: + subprocess.run( + ["cmake", "-S", str(root), "-B", str(build_dir)], + check=True, capture_output=True, timeout=600, + ) + subprocess.run( + ["cmake", "--build", str(build_dir), "-j", str(os.cpu_count() or 1)], + check=True, capture_output=True, timeout=1800, + ) + except (subprocess.CalledProcessError, subprocess.TimeoutExpired, OSError): + return [], [] + archives = sorted(build_dir.rglob("*.a")) + + if not archives: + return [], [] + + # -L one directory per archive location; -l the archive basenames with + # the lib prefix and .a suffix stripped, which is what the linker wants. + lib_dirs = sorted({a.parent for a in archives}) + libraries = [a.stem[3:] for a in archives if a.stem.startswith("lib")] + return lib_dirs, libraries + + def _detect_libraries(lib_dir: Path, pkg_name: str) -> List[str]: """Detect installed library names from a lib/ directory.""" if not lib_dir.exists(): @@ -271,27 +361,165 @@ def _resolve_backend_request( resolved_backend = detect_backend(source_dir) log.info(f"Auto-detected backend: {resolved_backend}") - # A build.yaml that declares its own targets is a statement that - # ebuild builds this project. detect_backend() only inspects the - # filesystem, so a Makefile kept for `make flash` -- or a - # CMakeLists.txt belonging to one subcomponent -- used to outrank - # that statement: the dispatcher ran the external tool, the - # declared targets were never built, and the build still reported - # success. - # - # Only auto-detection is overridden. An explicit `backend:` in - # build.yaml or --backend on the command line still wins, which - # is how a project keeps both a target list and an external - # build. - if resolved_backend != "ninja" and cfg.targets: - log.info( - f"build.yaml declares {len(cfg.targets)} target(s), so " - f"the ninja backend is used instead of the detected " - f"{resolved_backend}. To build with {resolved_backend}, " - f"set 'backend: {resolved_backend}' in build.yaml or " - f"pass --backend {resolved_backend}." - ) - resolved_backend = "ninja" + return resolved_backend, backend_config + + +# The project-local file that records which board this checkout targets. +_EOS_PROJECT_CONFIG = "eos.yaml" + + +def _record_board_selection(board: str, log: Logger) -> None: + """Persist ``--board`` into eos.yaml under ``system.board``. + + The golden path is `configure --board` then a bare `build`, so the choice + has to outlive the configure process. It is written to eos.yaml rather + than build.yaml because the board is a property of the system being + targeted, which is what eos.yaml already describes. + """ + path = Path(_EOS_PROJECT_CONFIG) + if not path.is_file(): + log.error( + f"No {_EOS_PROJECT_CONFIG} here, so there is nothing to record the " + f"board against. Run this from a project directory created by " + f"'ebuild new'." + ) + raise SystemExit(1) + + import yaml + + try: + data = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + except yaml.YAMLError as e: + log.error(f"{_EOS_PROJECT_CONFIG} is not valid YAML: {e}") + raise SystemExit(1) + + system = data.setdefault("system", {}) + previous = system.get("board") + system["board"] = board + path.write_text(yaml.safe_dump(data, sort_keys=False), encoding="utf-8") + + if previous and previous != board: + log.info(f"Board: {previous} -> {board}") + else: + log.info(f"Board: {board}") + + +def _selected_board(default: str = "generic") -> str: + """The board this project targets, from its eos.yaml. + + Read rather than passed in: `ebuild build` takes no --board of its own in + the documented walk, so the value has to survive from `ebuild new` or + `ebuild configure`. + """ + path = Path("eos.yaml") + if not path.is_file(): + return default + try: + import yaml + data = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + except Exception: + return default + return (data.get("system") or {}).get("board") or default + + +def _build_summary(cfg: "ProjectConfig", compiler, package_paths, log: Logger) -> None: + """The per-component summary the MLP walk ends with. + + A build that prints only "Build completed successfully" leaves the + developer to infer what was actually in it. The interesting case is a + package that resolved to nothing: the build still succeeds, the feature is + simply absent, and nothing said so. + """ + board = _selected_board(default="") + rows = [ + ("toolchain", getattr(compiler, "cc", "") or "cc", True), + ("board configuration", board or "host (no board recorded)", True), + ] + + declared = [p.name for p in getattr(cfg, "packages", []) or []] + for name in declared: + paths = (package_paths or {}).get(name) + # A package with no resolved include or library directory contributed + # nothing to this build, whatever build.yaml says. + resolved = bool(paths and (paths.include_dirs or paths.lib_dirs)) + rows.append((name, "" if resolved else "declared, nothing resolved", + resolved)) + + for target in cfg.targets: + if target.target_type in ("executable", "test"): + rows.append((target.name, target.target_type, True)) + + width = max(len(n) for n, _d, _ok in rows) + log.info("") + log.info("EmbeddedOS Build") + for name, detail, ok in rows: + mark = "OK " if ok else "MISS" + log.info(f" {mark} {name.ljust(width)}" + (f" {detail}" if detail else "")) + + missing = [n for n, _d, ok in rows if not ok] + if missing: + log.warning( + f"{len(missing)} declared package(s) resolved to nothing: " + + ", ".join(missing) + + ". The build succeeded without them." + ) + + +def _report_footprint(cfg: "ProjectConfig", build_path: Path, log: Logger) -> None: + """Print how much of the board the build just used. + + The MLP walk ends with a build that says `Flash: 384 KB / RAM: 72 KB`. A + developer who has to run `size` themselves and remember which columns to + add is not being told; they are being left to find out. + + Never fatal. A footprint that cannot be measured -- no binutils, a cross + toolchain whose `size` is not installed -- is a missing convenience, and + failing a successful build over it would be worse than the silence it + replaces. + """ + from ebuild.build.footprint import ( + FootprintError, board_capacity, find_size_tool, format_report, + measure, over_budget, + ) + + binaries = [t for t in cfg.targets if t.target_type == "executable"] + if not binaries: + return + + artifact = build_path / binaries[0].name + if not artifact.is_file(): + return + + prefix = getattr(cfg.toolchain, "target", None) or "host" + tool = find_size_tool(prefix) + if tool is None: + log.debug(f"no size tool for toolchain {prefix!r}; skipping footprint") + return + + try: + fp = measure(artifact, tool) + except FootprintError as exc: + log.debug(f"footprint unavailable: {exc}") + return + + board = _selected_board(default="") + flash_cap, ram_cap = board_capacity(board or None, _board_config()) + log.info("") + for line in format_report(fp, flash_cap, ram_cap).splitlines(): + log.info(line) + + exceeded = over_budget(fp, flash_cap, ram_cap) + if exceeded: + # Not a build failure: the image linked. It will not fit on the board, + # which the developer needs to hear now rather than from a device that + # will not boot. + log.warning(f"{exceeded} -- this image will not fit.") + else: + log.info("Ready to flash.") + + +def _board_config() -> Optional[Dict[str, Any]]: + """The project's own board description, if it ships one. return resolved_backend, backend_config @@ -759,7 +987,8 @@ def build(log: Logger, config_path: str, build_dir: str, backend: Optional[str], log.debug(f"Compiler: {compiler.cc}") # Install packages if any are declared - package_paths = _install_packages(cfg, build_path, log, verbose=log.verbose, jobs=jobs) + package_paths = {**_workspace_repo_paths(), + **_install_packages(cfg, build_path, log, verbose=log.verbose, jobs=jobs)} log.step(f"Generating build.ninja in {_shown(build_path)}/...") ninja_backend = NinjaBackend(cfg, build_path, compiler, package_paths=package_paths) diff --git a/ebuild/packages/registry.py b/ebuild/packages/registry.py index 8ee7b619..c7679bd6 100644 --- a/ebuild/packages/registry.py +++ b/ebuild/packages/registry.py @@ -19,6 +19,9 @@ # ("3.6.0-rc1") or build metadata ("1.3.1+patch2"). _SUFFIX_SPLIT = re.compile(r"[-+]") +# digits then letters: the 1.2.11b patch-respin form. +_RESPIN = re.compile(r"(\d+)([A-Za-z]+)") + _ComponentKey = Tuple[int, int, str] @@ -29,9 +32,19 @@ def _component_key(component: str) -> _ComponentKey: 1.9.0. Anything else compares as text and ranks below any numeric component, which keeps the ordering total without inventing a meaning for identifiers the recipe format does not define. + + A component that is digits followed by letters -- the patch-respin form + zlib and OpenSSL use, 1.2.11b after 1.2.11 -- keeps the numeric rank of + its digits and orders on the letters after it, so 1.2.11 < 1.2.11b and + 1.2.11b < 1.2.11c. Treating it as text instead put it below every + numeric component, which sorted the respin *below* the release it + supersedes. """ if component.isdigit(): return (1, int(component), "") + respin = _RESPIN.fullmatch(component) + if respin: + return (1, int(respin.group(1)), respin.group(2)) return (0, 0, component) diff --git a/tests/ebuild/test_integration_initramfs_security.py b/tests/ebuild/test_integration_initramfs_security.py index 78cd916b..3dff3a7e 100644 --- a/tests/ebuild/test_integration_initramfs_security.py +++ b/tests/ebuild/test_integration_initramfs_security.py @@ -21,7 +21,9 @@ """ import gzip +import os import shutil +import stat import subprocess import pytest @@ -40,6 +42,44 @@ @requires_cpio +def _newc_members(data): + """Parse enough of ``newc`` to validate names, metadata, and contents.""" + members = {} + offset = 0 + while True: + header = data[offset:offset + 110] + assert len(header) == 110 + assert header[:6] == b"070701" + fields = [int(header[i:i + 8], 16) for i in range(6, 110, 8)] + inode = fields[0] + mode = fields[1] + link_count = fields[4] + file_size = fields[6] + name_size = fields[11] + + offset += 110 + encoded_name = data[offset:offset + name_size] + assert encoded_name.endswith(b"\0") + name = os.fsdecode(encoded_name[:-1]) + offset += name_size + offset += -offset % 4 + + contents = data[offset:offset + file_size] + assert len(contents) == file_size + offset += file_size + offset += -offset % 4 + members[name] = { + "inode": inode, + "mode": mode, + "link_count": link_count, + "contents": contents, + } + + if name == "TRAILER!!!": + return members + + + def test_create_initramfs_produces_valid_gzip_with_expected_content(tmp_path): """Functional regression: the pipeline must still work correctly.""" rootfs = tmp_path / "rootfs" diff --git a/tests/unit/test_package_registry.py b/tests/unit/test_package_registry.py index 96a715bf..560ca8c7 100644 --- a/tests/unit/test_package_registry.py +++ b/tests/unit/test_package_registry.py @@ -12,7 +12,7 @@ import pytest -from ebuild.packages.registry import PackageRegistry, version_key +from ebuild.packages.registry import PackageRegistry, version_sort_key def _recipe(name: str, version: str): @@ -31,7 +31,7 @@ class TestVersionKey: def test_numeric_versions_compare_numerically(self): """Plain string ordering would put 11.1.0 below 2.9.3.""" versions = ["2.9.3", "11.1.0", "1.3.1"] - assert sorted(versions, key=version_key) == ["1.3.1", "2.9.3", "11.1.0"] + assert sorted(versions, key=version_sort_key) == ["1.3.1", "2.9.3", "11.1.0"] @pytest.mark.parametrize( "version", @@ -40,18 +40,27 @@ def test_numeric_versions_compare_numerically(self): ) def test_non_numeric_versions_do_not_raise(self, version): """Any of these previously raised ValueError from int().""" - assert version_key(version) + assert version_sort_key(version) def test_pre_release_sorts_below_its_release(self): """Semver precedence: 3.6.0-rc1 comes before 3.6.0.""" - assert version_key("3.6.0-rc1") < version_key("3.6.0") + assert version_sort_key("3.6.0-rc1") < version_sort_key("3.6.0") def test_letter_suffix_sorts_above_its_base(self): """A patch respin such as 1.2.11b supersedes 1.2.11.""" - assert version_key("1.2.11") < version_key("1.2.11b") + assert version_sort_key("1.2.11") < version_sort_key("1.2.11b") + + def test_successive_respins_order_among_themselves(self): + """1.2.11b supersedes 1.2.11a, and both supersede 1.2.11.""" + assert version_sort_key("1.2.11a") < version_sort_key("1.2.11b") + assert version_sort_key("1.2.11") < version_sort_key("1.2.11a") + + def test_respin_still_ranks_above_a_textual_component(self): + """A respin keeps its numeric rank, so it beats a non-numeric one.""" + assert version_sort_key("1.2.beta") < version_sort_key("1.2.11b") def test_shorter_version_sorts_below_longer(self): - assert version_key("1.2") < version_key("1.2.1") + assert version_sort_key("1.2") < version_sort_key("1.2.1") class TestRegistryVersionSelection: