diff --git a/.bazelrc b/.bazelrc index 60f96f6d..81e373a3 100644 --- a/.bazelrc +++ b/.bazelrc @@ -70,10 +70,10 @@ build:time-arm64-qnx --extra_toolchains=@score_qcc_aarch64_toolchain//:aarch64-q # ------------------------------------------------------------------------------- try-import %workspace%/user.bazelrc -# Coverage configuration for C++ -coverage --features=coverage -coverage --combined_report=lcov -coverage --cache_test_results=no +# Coverage configuration — LLVM (Linux) + gcov (QNX). +# All coverage settings live in quality/coverage/coverage.bazelrc. +# Usage: bazel coverage //score/... --build_tests_only +import %workspace%/quality/coverage/coverage.bazelrc # ------------------------------------------------------------------------------- # Sanitizer configurations — powered by score_cpp_policies diff --git a/.github/workflows/code-coverage.yml b/.github/workflows/code-coverage.yml index 333ed94f..da1180a1 100644 --- a/.github/workflows/code-coverage.yml +++ b/.github/workflows/code-coverage.yml @@ -10,7 +10,7 @@ # # SPDX-License-Identifier: Apache-2.0 # ******************************************************************************* -name: Test / Code Coverage (>85%) +name: Test / Code Coverage Linux permissions: contents: read @@ -29,12 +29,45 @@ on: types: [checks_requested] jobs: - code-coverage: - uses: eclipse-score/cicd-workflows/.github/workflows/cpp-coverage.yml@93aac16ada7d247bbb6ae926509ddea74cf5213a # v0.0.2 - with: - bazel-target: "//score/..." - bazel-config: "time-x86_64-linux" - extra-bazel-flags: "--test_output=errors --nocache_test_results" - artifact-name-suffix: "_cpp" - retention-days: 10 - min-coverage: 85 + coverage-linux: + name: LLVM Coverage (Linux x86_64) + runs-on: ubuntu-24.04 + permissions: + contents: read + actions: write + + steps: + - uses: eclipse-score/more-disk-space@6a3b48901846bf7f8cc985925157d71a8973e61f # v1 + with: + level: 4 + + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Setup Bazel with shared caching + uses: bazel-contrib/setup-bazel@0.18.0 + with: + disk-cache: ${{ github.workflow }} + repository-cache: true + + - name: Run coverage + run: bazel coverage --build_tests_only --test_output=errors //score/... + + - name: Check coverage thresholds + if: always() + run: | + output_path="$(bazel info output_path)" + unzip -o "${output_path}/_coverage/_coverage_report.dat" -d coverage_output/ + run: python3 quality/coverage/check_coverage.py \ + --coverage-dir coverage_output/ \ + --min-line 85 \ + --min-branch 70 + + - name: Upload coverage artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: inc_time_coverage_report_${{ github.sha }} + path: coverage_output/ + if-no-files-found: ignore + retention-days: 10 diff --git a/MODULE.bazel b/MODULE.bazel index 19aa03e7..6bf1430e 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -70,6 +70,8 @@ bazel_dep(name = "score_logging", version = "0.2.1") ### Modules that are used internally within the repository but not exposed as part of the public API bazel_dep(name = "score_docs_as_code", version = "4.5.0") +# Required by //quality/coverage/llvm_cov:enable_llvm_coverage_for_death_tests (cc_feature). +bazel_dep(name = "rules_cc", version = "0.2.17", dev_dependency = True) bazel_dep(name = "score_cpp_policies", version = "0.0.1", dev_dependency = True) @@ -120,8 +122,11 @@ llvm = use_extension( ) llvm.toolchain( llvm_version = "19.1.7", + extra_known_features = [ + "//quality/coverage/llvm_cov:enable_llvm_coverage_for_death_tests", + ], ) -use_repo(llvm, "llvm_toolchain") +use_repo(llvm, "llvm_toolchain", "llvm_toolchain_llvm") # grpc-java@1.66.0 has a BCR bug (extension no longer generates # com_envoyproxy_protoc_gen_validate). Pulled transitively via diff --git a/README.md b/README.md index 1e8af750..5832f6a8 100644 --- a/README.md +++ b/README.md @@ -67,9 +67,37 @@ artifacts of the module. ### 3️⃣ Run Tests ```sh -bazel test //tests/... +bazel test --config=time-x86_64-linux //score/... ``` +### 4️⃣ Run Coverage (Linux — LLVM source-based) + +All coverage configuration lives in [`quality/coverage/`](quality/coverage/README.md). +See that README for the full pipeline architecture, scope configuration, and +how to extend coverage to new components. + +```sh +bazel coverage --build_tests_only //score/... +``` + +The reporter generates an HTML report, LCOV data, and a text summary, packaged +in a zip at `$(bazel info output_path)/_coverage/_coverage_report.dat`. +Extract and check thresholds with the bundled script: + +```sh +output_path="$(bazel info output_path)" +unzip -o "${output_path}/_coverage/_coverage_report.dat" -d coverage_output/ +python3 quality/coverage/check_coverage.py \ + --coverage-dir coverage_output/ \ + --min-line 85 \ + --min-branch 70 +# HTML report: coverage_output/html_report/index.html +# LCOV data: coverage_output/lcov_report/lcov.dat +``` + +**QNX coverage** uses the gcov pipeline. Add `--config=time-x86_64-qnx` to +activate it; the LLVM settings are reset automatically. + --- ## 🛠 Tools & Linters diff --git a/quality/coverage/BUILD b/quality/coverage/BUILD new file mode 100644 index 00000000..11326e94 --- /dev/null +++ b/quality/coverage/BUILD @@ -0,0 +1,39 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +load("//quality/coverage:coverage_scope.bzl", "coverage_scope") + +# Defines the production source file scope for coverage reporting. +# The aspect traverses each dep's cc_library deps transitively — +# add new top-level production libraries here when they are not reachable +# via the deps of an existing entry. +coverage_scope( + name = "time_coverage_scope", + testonly = True, + visibility = ["//quality/coverage:__subpackages__"], + deps = [ + # score/time: package-level facades; transitively cover clock_core, + # ptp_types, and all clock interfaces + implementations + "//score/time/vehicle_time:vehicle_time", + "//score/time/steady_time:steady_time", + "//score/time/system_time:system_time", + "//score/time/high_res_steady_time:high_res_steady_time", + # score/time_daemon: binary alias transitively covers svt_handler, + # job_runner, control_flow_divider, msg_broker, ptp_machine/shm, + # verification_machine and their common dep chains, plus the binary's + # own application sources + "//score/time_daemon:time_daemon", + # score/time_slave: binary alias covers application sources + gptp_engine + "//score/time_slave:time_slave", + ], +) diff --git a/quality/coverage/README.md b/quality/coverage/README.md new file mode 100644 index 00000000..ca875bc1 --- /dev/null +++ b/quality/coverage/README.md @@ -0,0 +1,184 @@ +# Coverage Pipeline + +This directory contains the LLVM source-based coverage pipeline for `score/time`, +`score/time_daemon`, and `score/time_slave`. + +--- + +## Why LLVM Source-Based Coverage + +GCC/gcov instruments at assembly level and generates **phantom branches** for: + +- Exception-unwind edges on every non-`noexcept` call site +- GMock internal bookkeeping branches (all mock files) +- Assertion-abort edges only coverable via death-test subprocesses + +These phantom branches permanently force mock files and PTP headers to ~50% +branch coverage regardless of test quality, making the 90% branch goal +unreachable with gcov. + +LLVM source-based coverage (`llvm-cov`) tracks **source regions**, not assembly +branches. The result: no phantom branches, accurate metrics, and a 95%+ line +coverage baseline with gaps only from genuinely untested code paths. + +--- + +## File Layout + +``` +quality/coverage/ +├── BUILD # coverage_scope target (production file allowlist) +├── coverage.bazelrc # All Bazel coverage flags — imported by .bazelrc +├── coverage_scope.bzl # Starlark rule + aspect for allowlist generation +├── check_coverage.py # Per-component threshold gating script +└── llvm_cov/ + ├── BUILD # merger, reporter, reporter_wrapper targets + ├── merger.py # Per-test profraw → profdata zip generator + ├── reporter.py # Final HTML + LCOV report generator + └── reporter_wrapper.bzl # Bakes allowlist + baseline-objects into the launcher +``` + +--- + +## Architecture: Coverage Scope + +The **`coverage_scope` rule** uses a Bazel aspect to traverse the `deps` graph of +declared production library targets and emit two generated files: + +| File | Contents | +|---|---| +| `time_coverage_scope_allowlist.txt` | One workspace-relative source path per line | +| `time_coverage_scope_objects.txt` | Paths to `.a` archives for baseline coverage | + +The **`reporter_wrapper` rule** generates a shell launcher that calls `reporter.py` +with `--coverage_allowlist` and `--baseline_objects` pre-baked. No manual +`--ignore_filename_regex` flags are needed. + +### Production scope roots (`quality/coverage/BUILD`) + +The scope is rooted at the package-level production library targets. The aspect +traverses their `deps` **transitively**, so adding a new sub-library as a dep of +an existing root automatically includes it. + +When a new **top-level** production library is added (not reachable via any +existing root's dep chain), add it to `coverage_scope(deps=[...])` in +[`quality/coverage/BUILD`](BUILD). + +### Baseline coverage + +Files in the allowlist that compiled but were never exercised by any test appear +at **0% coverage** in the report rather than being silently omitted. This makes +coverage gaps visible without any test having to explicitly import the file. + +--- + +## Pipeline Data Flow + +``` +bazel coverage --build_tests_only //score/... + │ + ├─ [per test] merger.py + │ profraw ──────────────► profdata zip + │ + └─ [once, final] reporter_wrapper.sh + │ calls reporter.py with: + │ --coverage_allowlist=time_coverage_scope_allowlist.txt + │ --baseline_objects=time_coverage_scope_objects.txt + │ + │ + └─ reporter.py + llvm-profdata merge all profdata ──► merged.profdata + llvm-cov export (LCOV) ──► lcov.dat + llvm-cov show (HTML) ──► html_report/index.html + [filtered to allowlist; baseline objects fill 0% gaps] + zip output ──► _coverage_report.dat +``` + +--- + +## Running Coverage Locally + +```sh +# Run all tests and generate the coverage report +bazel coverage --build_tests_only //score/... + +# Unpack the report +output_path="$(bazel info output_path)" +unzip -o "${output_path}/_coverage/_coverage_report.dat" -d coverage_output/ + +# Open HTML report +xdg-open coverage_output/html_report/index.html + +# Check per-component thresholds +python3 quality/coverage/check_coverage.py \ + --coverage-dir coverage_output/ \ + --min-line 85 \ + --min-branch 70 +``` + +**QNX coverage** resets all LLVM flags and uses the gcov pipeline. Add +`--config=time-x86_64-qnx` to any coverage command to activate it. + +--- + +## `check_coverage.py` — Threshold Gating + +Parses `coverage_output/lcov_report/lcov.dat`, groups files by component +(`score//`), and prints a summary table: + +``` +Component Lines Branches +score/time 95.4% (245/257) 88.2% (90/102) +score/time_daemon 82.1% (...) ... +``` + +Exits with code 1 if any component falls below `--min-line` or `--min-branch`. +Used in CI (`code-coverage.yml`). + +### Arguments + +| Flag | Default | Description | +|---|---|---| +| `--coverage-dir` | required | Directory containing `lcov_report/lcov.dat` | +| `--min-line` | `85.0` | Minimum line coverage % per component | +| `--min-branch` | `0.0` | Minimum branch coverage % per component | + +--- + +## Extending the Scope + +### Adding a new clock domain + +The new domain's production library (e.g. `//score/time/my_clock:my_clock`) is +automatically included if it is a transitive dep of any existing entry in +`coverage_scope(deps=[...])`. Otherwise add it explicitly: + +```python +# quality/coverage/BUILD +coverage_scope( + name = "time_coverage_scope", + ... + deps = [ + ... + "//score/time/my_clock:my_clock", + ], +) +``` + +### Adding a new time_daemon sub-component + +Same rule: if not reachable from `svt_handler` or `ipc/svt/receiver:factory`, +add the production library target to `coverage_scope(deps=[...])`. + +--- + +## Bazel Feature: Death-Test Coverage + +`enable_llvm_coverage_for_death_tests` (defined in [`llvm_cov/BUILD`](llvm_cov/BUILD)) +adds `-mllvm -runtime-counter-relocation` compiler flags. This enables +`LLVM_PROFILE_CONTINUOUS_MODE=1` so that `ASSERT_DEATH` subprocesses write +`.profraw` files before the process exits, making death-test branches coverable. + +The feature is wired via `llvm.toolchain(extra_known_features=[...])` in +`MODULE.bazel` and activated automatically for all coverage builds via +`coverage --features=enable_llvm_coverage_for_death_tests` in `coverage.bazelrc`. diff --git a/quality/coverage/check_coverage.py b/quality/coverage/check_coverage.py new file mode 100644 index 00000000..b2048d5e --- /dev/null +++ b/quality/coverage/check_coverage.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +"""Check per-component line and branch coverage from an LLVM coverage report. + +Reads lcov_report/lcov.dat from the unpacked reporter output directory, +groups every tracked source file by its score// prefix, computes +line and branch coverage per component, and exits non-zero if any component +falls below either threshold. + +Components are discovered automatically from the report — no hardcoded list. +A component is the path segment directly below score/ (e.g. score/time/). + +Usage: + python3 check_coverage.py --coverage-dir DIR [--min-line PCT] [--min-branch PCT] + +Arguments: + --coverage-dir Directory produced by unzipping the reporter output zip. + --min-line Minimum line coverage %% per component (default: 85). + --min-branch Minimum branch coverage %% per component (default: 0, + i.e. not enforced unless explicitly set). + +Exit codes: + 0 All components meet both thresholds. + 1 One or more components are below a threshold. + 2 Coverage report not found. +""" + +import argparse +import sys +from pathlib import Path + + +def parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + p.add_argument("--coverage-dir", type=Path, required=True, metavar="DIR", + help="directory produced by unzipping the reporter output zip") + p.add_argument("--min-line", type=float, default=85.0, metavar="PCT", + help="minimum effective line coverage %% per component (default: 85)") + p.add_argument("--min-branch", type=float, default=0.0, metavar="PCT", + help="minimum effective branch coverage %% per component (default: 0 = not enforced)") + return p.parse_args() + + +def component_of(path: str) -> str | None: + """Return the score// prefix for a workspace source path, or None. + + Skips files from external repositories (path contains /external/). + """ + if "/external/" in path or "bazel-out/" in path: + return None + marker = "/score/" + idx = path.find(marker) + if idx == -1: + return None + rest = path[idx + len(marker):] + parts = rest.split("/") + return f"score/{parts[0]}/" if parts[0] else None + + +def parse_lcov(lcov_path: Path) -> dict[str, tuple[int, int, int, int]]: + """Return {component: (lines_hit, lines_found, branches_hit, branches_found)}.""" + totals: dict[str, list[int]] = {} + current: str | None = None + for line in lcov_path.read_text(errors="replace").splitlines(): + if line.startswith("SF:"): + current = component_of(line[3:]) + elif current: + if line.startswith("LH:"): + totals.setdefault(current, [0, 0, 0, 0])[0] += int(line[3:]) + elif line.startswith("LF:"): + totals.setdefault(current, [0, 0, 0, 0])[1] += int(line[3:]) + elif line.startswith("BRH:"): + totals.setdefault(current, [0, 0, 0, 0])[2] += int(line[4:]) + elif line.startswith("BRF:"): + totals.setdefault(current, [0, 0, 0, 0])[3] += int(line[4:]) + return {k: tuple(v) for k, v in sorted(totals.items())} # type: ignore[return-value] + + +def main() -> None: + args = parse_args() + lcov_path = args.coverage_dir / "lcov_report" / "lcov.dat" + if not lcov_path.exists(): + print(f"::warning::Coverage report not found: {lcov_path}") + sys.exit(2) + + totals = parse_lcov(lcov_path) + line_threshold = args.min_line + branch_threshold = args.min_branch + check_branches = branch_threshold > 0 + + header = f"{'Component':<32} {'Line%':>8} {'Branch%':>8} (line≥{line_threshold:.0f}%" + header += f", branch≥{branch_threshold:.0f}%)" if check_branches else ")" + print(f"\n{header}") + print("-" * (len(header) + 4)) + + failed: list[str] = [] + for comp, (lh, lf, brh, brf) in totals.items(): + line_pct = lh / lf * 100 if lf else 0.0 + branch_pct = brh / brf * 100 if brf else 0.0 + + line_ok = line_pct >= line_threshold + branch_ok = (not check_branches) or branch_pct >= branch_threshold + status = "✓" if (line_ok and branch_ok) else "✗ FAIL" + + branch_str = f" {branch_pct:>7.1f}%" if check_branches else "" + suffix = f" (line)" if not line_ok else (" (branch)" if not branch_ok else "") + print(f"{comp:<32} {line_pct:>7.1f}%{branch_str} {status}{suffix}") + + if not line_ok: + failed.append(f"{comp}: line {line_pct:.1f}% < {line_threshold:.0f}%") + if not branch_ok: + failed.append(f"{comp}: branch {branch_pct:.1f}% < {branch_threshold:.0f}%") + + print() + if failed: + for msg in failed: + print(f"::error::Coverage gate failed — {msg}") + sys.exit(1) + print("All components meet the thresholds.") + + +if __name__ == "__main__": + main() diff --git a/quality/coverage/coverage.bazelrc b/quality/coverage/coverage.bazelrc new file mode 100644 index 00000000..b1451ef1 --- /dev/null +++ b/quality/coverage/coverage.bazelrc @@ -0,0 +1,74 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +# ============================================================================ +# Common settings (Linux and QNX) +# ============================================================================ +coverage --nocache_test_results +coverage --cxxopt=-O0 +coverage --combined_report=lcov +# Static linking eliminates the coverage race where headers instrumented in one +# .so shadow differently-instrumented copies in another. +coverage --dynamic_mode=off + +# Sandbox and host platform settings (mirrors time_shared) so coverage commands +# work without an additional --config flag. +coverage --incompatible_strict_action_env +coverage --sandbox_writable_path=/var/tmp +coverage --host_platform=@score_bazel_platforms//:x86_64-linux + +# ============================================================================ +# Linux: LLVM source-based coverage (default) +# ============================================================================ +# Source-based coverage tracks AST regions, not assembly branches, eliminating +# phantom branches from exception-unwind edges and GMock bookkeeping. +coverage --experimental_fetch_all_coverage_outputs +coverage --experimental_generate_llvm_lcov +coverage --experimental_use_llvm_covmap +# Generic Linux platform — the GCC-specific platform from time-x86_64-linux +# would block the LLVM toolchain resolution. +coverage --platforms=@score_bazel_platforms//:x86_64-linux +coverage --extra_toolchains=@llvm_toolchain//:cc-toolchain-x86_64-linux + +# --experimental_use_llvm_covmap instruments ALL targets; --instrumentation_filter +# still controls which object files appear in the per-test coverage manifest +# passed to the merger. Without it Bazel defaults to the test's own package only. +coverage --instrumentation_filter="^//score[/:]" + +# Custom per-test merger (profraw→profdata zip) and final reporter (HTML+LCOV). +coverage --coverage_output_generator=//quality/coverage/llvm_cov:merger +coverage --coverage_report_generator=//quality/coverage/llvm_cov:reporter_wrapper + +# Suppress gcov post-processing; the merger handles profdata directly. +coverage --test_env=COVERAGE_GCOV_PATH=/usr/bin/true +coverage --test_env=GENERATE_LLVM_LCOV=0 + +# Continuous-mode profiling — writes .profraw even from ASSERT_DEATH subprocesses. +# Requires counter relocation via the enable_llvm_coverage_for_death_tests feature +# defined in //quality/coverage:enable_llvm_coverage_for_death_tests. +coverage --test_env=LLVM_PROFILE_CONTINUOUS_MODE=1 +coverage --features=enable_llvm_coverage_for_death_tests + +# ============================================================================ +# QNX: GCC/gcov pipeline — activated by --config=time-x86_64-qnx +# ============================================================================ +coverage:time-x86_64-qnx --noexperimental_use_llvm_covmap +coverage:time-x86_64-qnx --noexperimental_generate_llvm_lcov +coverage:time-x86_64-qnx --noexperimental_fetch_all_coverage_outputs +coverage:time-x86_64-qnx --coverage_output_generator=@bazel_tools//tools/test:lcov_merger +coverage:time-x86_64-qnx --coverage_report_generator=@bazel_tools//tools/test:coverage_report_generator +coverage:time-x86_64-qnx --test_env=GENERATE_LLVM_LCOV +coverage:time-x86_64-qnx --test_env=COVERAGE_GCOV_PATH +coverage:time-x86_64-qnx --test_env=LLVM_PROFILE_CONTINUOUS_MODE +coverage:time-x86_64-qnx --cxxopt=-fprofile-update=atomic +coverage:time-x86_64-qnx --test_env=COVERAGE_GCOV_OPTIONS=-bcu diff --git a/quality/coverage/coverage_scope.bzl b/quality/coverage/coverage_scope.bzl new file mode 100644 index 00000000..aaa25ebe --- /dev/null +++ b/quality/coverage/coverage_scope.bzl @@ -0,0 +1,127 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +"""Coverage scope rule: derives file-level allowlists from cc_library dep graphs. + +The aspect traverses the build graph starting from the declared `deps`, walking +any `deps`, `implementation_deps`, and `exported_deps` edges. At each cc_library +it collects the actual source files (srcs + hdrs). External files and generated +files are excluded. + +The resulting allowlist contains one workspace-relative source file path per +line. The coverage reporter uses this to restrict the HTML/LCOV report to +exactly the files that are part of the declared production scope. +""" + +visibility(["//..."]) + +_CoverageScopeInfo = provider( + doc = "Carries source file paths and object files collected by the coverage scope aspect.", + fields = { + "source_files": "Depset of source file path strings (workspace-relative).", + "object_files": "Depset of compiled .a File objects for baseline coverage.", + }, +) + +def _coverage_scope_aspect_impl(target, ctx): + """Collects source file paths and archive files from the build graph.""" + direct_files = [] + direct_archives = [] + transitive = [] + transitive_archives = [] + + if CcInfo in target: + for attr_name in ["srcs", "hdrs"]: + if hasattr(ctx.rule.attr, attr_name): + for src in getattr(ctx.rule.attr, attr_name): + for f in src.files.to_list(): + # f.path is the exec-root path: always "external//..." + # for external deps, in both WORKSPACE and bzlmod layouts. + # f.short_path ("../repo/...") is stored — not used for filtering. + if not f.path.startswith("external/") and f.is_source: + direct_files.append(f.short_path) + + # Collect only workspace-internal archives; @@// labels are the workspace root in bzlmod. + if not str(target.label).startswith("@@") or str(target.label).startswith("@@//"): + for linker_input in target[CcInfo].linking_context.linker_inputs.to_list(): + for lib in linker_input.libraries: + for archive in [lib.static_library, lib.pic_static_library]: + if archive and "/external/" not in archive.path and not archive.path.startswith("external/"): + direct_archives.append(archive) + break + + for attr_name in ["deps", "implementation_deps", "exported_deps"]: + if hasattr(ctx.rule.attr, attr_name): + for dep in getattr(ctx.rule.attr, attr_name): + if _CoverageScopeInfo in dep: + transitive.append(dep[_CoverageScopeInfo].source_files) + transitive_archives.append(dep[_CoverageScopeInfo].object_files) + + return [_CoverageScopeInfo( + source_files = depset(direct_files, transitive = transitive), + object_files = depset(direct_archives, transitive = transitive_archives), + )] + +_coverage_scope_aspect = aspect( + implementation = _coverage_scope_aspect_impl, + attr_aspects = ["deps", "implementation_deps", "exported_deps"], + doc = "Traverses cc_library dep graphs to collect implementation source files.", +) + +def _coverage_scope_impl(ctx): + """Aggregates aspect results into an allowlist file and an objects manifest.""" + all_files = {} + all_objects = [] + + for dep in ctx.attr.deps: + if _CoverageScopeInfo in dep: + for path in dep[_CoverageScopeInfo].source_files.to_list(): + if path: + all_files[path] = True + all_objects.append(dep[_CoverageScopeInfo].object_files) + + sorted_files = sorted(all_files.keys()) + object_depset = depset(transitive = all_objects) + + output = ctx.actions.declare_file(ctx.attr.name + "_allowlist.txt") + ctx.actions.write( + output = output, + content = "\n".join(sorted_files) + "\n" if sorted_files else "", + ) + + archive_paths = sorted({f.short_path: None for f in object_depset.to_list()}.keys()) + objects_output = ctx.actions.declare_file(ctx.attr.name + "_objects.txt") + ctx.actions.write( + output = objects_output, + content = "\n".join(archive_paths) + "\n" if archive_paths else "", + ) + + return [ + DefaultInfo(files = depset([output, objects_output], transitive = [object_depset])), + OutputGroupInfo( + allowlist = depset([output]), + objects = depset([objects_output]), + object_files = object_depset, + ), + ] + +coverage_scope = rule( + implementation = _coverage_scope_impl, + attrs = { + "deps": attr.label_list( + aspects = [_coverage_scope_aspect], + doc = "Production cc_library targets whose transitive source files form the coverage scope.", + ), + }, + doc = "Derives a source-file allowlist and baseline-objects manifest from cc_library dep graphs.", +) diff --git a/quality/coverage/llvm_cov/BUILD b/quality/coverage/llvm_cov/BUILD new file mode 100644 index 00000000..8427564b --- /dev/null +++ b/quality/coverage/llvm_cov/BUILD @@ -0,0 +1,62 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +load("@rules_cc//cc/toolchains:args.bzl", "cc_args") +load("@rules_cc//cc/toolchains:feature.bzl", "cc_feature") +load("@rules_python//python:defs.bzl", "py_binary") +load("//quality/coverage/llvm_cov:reporter_wrapper.bzl", "reporter_wrapper") + +# Enables LLVM runtime counter relocation — required for LLVM_PROFILE_CONTINUOUS_MODE. +# Without this, continuous-mode profiling from ASSERT_DEATH subprocesses fails. +cc_args( + name = "runtime_relocation_args", + actions = [ + "@rules_cc//cc/toolchains/actions:compile_actions", + "@rules_cc//cc/toolchains/actions:link_actions", + ], + args = [ + "-mllvm", + "-runtime-counter-relocation", + ], +) + +cc_feature( + name = "enable_llvm_coverage_for_death_tests", + args = [":runtime_relocation_args"], + feature_name = "enable_llvm_coverage_for_death_tests", + visibility = ["//visibility:public"], +) + +py_binary( + name = "merger", + srcs = ["merger.py"], +) + +py_binary( + name = "reporter", + testonly = True, + srcs = ["reporter.py"], + data = [ + "@llvm_toolchain//:llvm-cov", + "@llvm_toolchain//:llvm-profdata", + "@llvm_toolchain_llvm//:bin/llvm-cxxfilt", + ], + deps = ["@rules_python//python/runfiles"], +) + +reporter_wrapper( + name = "reporter_wrapper", + testonly = True, + coverage_scope = "//quality/coverage:time_coverage_scope", + reporter = ":reporter", +) diff --git a/quality/coverage/llvm_cov/merger.py b/quality/coverage/llvm_cov/merger.py new file mode 100644 index 00000000..33624506 --- /dev/null +++ b/quality/coverage/llvm_cov/merger.py @@ -0,0 +1,236 @@ +#!/usr/bin/env python3 +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +"""Per-test coverage output generator using llvm-cov. + +This script is invoked by Bazel as the --coverage_output_generator for each test. +It receives profraw files from test execution, merges them into profdata, generates +an HTML coverage report using llvm-cov show, and packages everything into a zip file +that the reporter can later aggregate. + +Expected Bazel interface (from collect_coverage.sh): + --coverage_dir= Directory containing *.profraw files + --output_file= Where to write the output (zip) + --source_file_manifest= File listing instrumented sources and object files + --filter_sources= Source path regexes to exclude (repeatable) + [--sources_to_replace_file=] Optional source mapping file +""" + +import argparse +import json +import os +import subprocess +import sys +import zipfile +from pathlib import Path +from typing import List, Set + + +def main() -> None: + args = parse_args() + + # Get object files from the manifest. + object_files = get_object_files_from_manifest(args.source_file_manifest) + if not object_files: + print("INFO: No instrumented object files found, skipping coverage.", file=sys.stderr) + cleanup_dangling_symlinks(args.coverage_dir) + sys.exit(0) + + # Find profraw files. + profraw_files = sorted(args.coverage_dir.glob("*.profraw")) + if not profraw_files: + print("INFO: No *.profraw files found, skipping coverage.", file=sys.stderr) + cleanup_dangling_symlinks(args.coverage_dir) + sys.exit(0) + + llvm_profdata = find_llvm_profdata() + + # Merge profraw → profdata. + profdata_dir = args.coverage_dir / "profdata" + profdata_dir.mkdir(exist_ok=True) + profdata_file = profdata_dir / "target.profdata" + + run_command([ + llvm_profdata, "merge", + "--sparse", + "--output", str(profdata_file), + ] + [str(f) for f in profraw_files]) + + # Create meta.json with object files for the reporter. + meta_dir = args.coverage_dir / "meta" + meta_dir.mkdir(exist_ok=True) + meta = { + "object_files": [os.path.realpath(f) for f in sorted(object_files)], + } + with open(meta_dir / "meta.json", "w", encoding="utf-8") as f: + json.dump(meta, f) + + # Package into zip at output_file. + create_zip( + root=args.coverage_dir, + directories=[profdata_dir, meta_dir], + output_file=args.output_file, + ) + + # Clean up dangling symlinks in coverage_dir that would cause Bazel tree + # artifact validation to fail (e.g. the 'gcov' symlink created by + # collect_cc_coverage.sh's init_gcov() pointing into the destroyed sandbox). + cleanup_dangling_symlinks(args.coverage_dir) + + target = os.environ.get("TEST_TARGET", "unknown") + print(f"INFO: Coverage merger completed for '{target}'", file=sys.stderr) + + +def find_llvm_profdata() -> str: + """Locate the llvm-profdata binary, terminating with an error when absent. + + C++ tests: Bazel exports LLVM_PROFDATA from the cc toolchain. + Rust tests: rules_rust exports RUST_LLVM_PROFDATA (an execroot-relative + path to the rust_toolchain's llvm_profdata) instead; resolve it against + the current directory, ROOT and the runfiles dir. + """ + direct = os.environ.get("LLVM_PROFDATA") + if direct and Path(direct).exists(): + return direct + + rust = os.environ.get("RUST_LLVM_PROFDATA") + if rust: + exec_root = Path(os.environ.get("ROOT", ".")) + runfiles_dir = Path(os.environ.get("RUNFILES_DIR", "")) / os.environ.get("TEST_WORKSPACE", "_main") + for candidate in [Path(rust), exec_root / rust, runfiles_dir / rust]: + if candidate.exists(): + return str(candidate) + + print( + "ERROR: llvm-profdata not found. Checked LLVM_PROFDATA " + f"({direct or 'unset'}) and RUST_LLVM_PROFDATA ({rust or 'unset'}). " + "For C++ tests the cc toolchain must export LLVM_PROFDATA; for Rust " + "tests the rust_toolchain must declare llvm_profdata " + "(score_toolchains_rust >= 0.9.2 with coverage-tools >= 1.3.0).", + file=sys.stderr, + ) + sys.exit(1) + + +def cleanup_dangling_symlinks(directory: Path) -> None: + """Remove symlinks in the coverage directory that would become dangling. + + Bazel's tree artifact validation rejects directories containing dangling + symlinks. The 'gcov' symlink created by collect_cc_coverage.sh's init_gcov() + points into the sandbox which is torn down before validation runs. Since we + use llvm-cov directly, this symlink is not needed. + """ + gcov_link = directory / "gcov" + if gcov_link.is_symlink(): + gcov_link.unlink() + + # Also remove any other symlinks pointing into sandbox paths. + for entry in directory.iterdir(): + if entry.is_symlink(): + target = os.readlink(entry) + if "sandbox" in target: + entry.unlink() + + +def get_object_files_from_manifest(source_file_manifest: Path) -> Set[str]: + """Parse the coverage manifest to find instrumented object files.""" + runfiles_dir = Path(os.environ.get("RUNFILES_DIR", "")) / os.environ.get("TEST_WORKSPACE", "_main") + exec_root = Path(os.environ.get("ROOT")) + + object_files = set() + with open(source_file_manifest, encoding="utf-8") as f: + manifests = [line.strip() for line in f.readlines()] + + for manifest in manifests: + if "objects_list.txt" in manifest: + with open(manifest, encoding="utf-8") as f: + for line in f: + obj_path = line.strip() + if not obj_path: + continue + # Try runfiles first, then exec_root. + candidate = runfiles_dir / obj_path + if candidate.exists(): + object_files.add(str(candidate)) + else: + object_files.add(str(exec_root / obj_path)) + else: + # Rust tests: rules_rust lists the instrumented test executable + # itself in the manifest (via coverage metadata_files) instead of + # an objects_list.txt. Pick up manifest entries that are ELF + # binaries directly. Skip external/ entries: the rust_toolchain + # also lists its llvm-cov/llvm-profdata binaries as metadata + # files, and those are not instrumented objects. + if manifest.startswith("external/") or "/external/" in manifest: + continue + for candidate in [runfiles_dir / manifest, exec_root / manifest, Path(manifest)]: + if candidate.is_file() and is_elf(candidate): + object_files.add(str(candidate)) + break + + return object_files + + +def is_elf(path: Path) -> bool: + """Return True if the file at path is an ELF binary.""" + try: + with open(path, "rb") as f: + return f.read(4) == b"\x7fELF" + except OSError: + return False + + +def run_command(cmd: List[str]) -> subprocess.CompletedProcess: + """Run a command and exit on failure.""" + try: + return subprocess.run( + cmd, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) + except subprocess.CalledProcessError as e: + print(f"ERROR: Command failed with code {e.returncode}:", file=sys.stderr) + print(f" {' '.join(cmd)}", file=sys.stderr) + if e.stdout: + print(e.stdout, file=sys.stderr) + sys.exit(1) + + +def create_zip(root: Path, directories: List[Path], output_file: Path) -> None: + """Create a zip file from the given directories relative to root.""" + with zipfile.ZipFile(output_file, "w", zipfile.ZIP_DEFLATED) as zf: + for directory in directories: + if not directory.exists(): + continue + for dirpath, _, files in os.walk(directory): + for filename in files: + file_path = Path(dirpath) / filename + arcname = file_path.relative_to(root) + zf.write(file_path, arcname) + + +def parse_args() -> argparse.Namespace: + """Parse command-line arguments matching the Bazel LCOV_MERGER interface.""" + parser = argparse.ArgumentParser(description="LLVM coverage merger for Bazel") + parser.add_argument("--coverage_dir", type=Path, required=True) + parser.add_argument("--output_file", type=Path, required=True) + parser.add_argument("--source_file_manifest", type=Path, required=True) + parser.add_argument("--filter_sources", action="append", default=[]) + parser.add_argument("--sources_to_replace_file", type=str, default=None) + return parser.parse_args() + + +if __name__ == "__main__": + main() diff --git a/quality/coverage/llvm_cov/reporter.py b/quality/coverage/llvm_cov/reporter.py new file mode 100644 index 00000000..1e1906df --- /dev/null +++ b/quality/coverage/llvm_cov/reporter.py @@ -0,0 +1,656 @@ +#!/usr/bin/env python3 +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +"""Final coverage report generator using llvm-cov. + +This script is invoked by Bazel as the --coverage_report_generator after all tests +complete. It reads the per-test zip files produced by the merger, merges all profdata +into one, and generates the final combined HTML report. + +Expected Bazel interface: + --reports_file= Text file listing paths to all per-test coverage outputs + --output_file= Where to write the final report (zip) +""" + +import argparse +import json +import os +import re +import subprocess +import sys +import zipfile +from pathlib import Path +from typing import List, Optional, Set, Tuple +from python.runfiles import Runfiles + + +def main() -> None: + """Main entry point.""" + args = parse_args() + r = Runfiles.Create() + + workspace_root = args.workspace_root + if not workspace_root: + # In the coverage sandbox cwd is the execroot; resolve() follows symlinks + # to the real workspace directory on the host filesystem. + workspace_root = str(Path.cwd().resolve()) + "/" + + # Read the list of per-test report files. + reports = read_reports_file(args.reports_file) + if not reports: + print("ERROR: No coverage reports found.", file=sys.stderr) + sys.exit(-1) + + # Extract profdata and object files from each per-test zip. + valid_profdata_files, valid_object_files = extract_reports(reports) + + if not valid_profdata_files or not valid_object_files: + print("INFO: No valid profdata or object files found.", file=sys.stderr) + sys.exit(-1) + + sorted_objects = sorted(valid_object_files) + + # Get llvm tools via runfiles. + llvm_bin_path = Path(r.Rlocation("llvm_toolchain/llvm-cov")) + + llvm_profdata = r.Rlocation("llvm_toolchain/llvm-profdata") + + # Merge all per-test profdata files. + merged_profdata = Path.cwd() / "merged_coverage.profdata" + merge_inputs = sorted(set(valid_profdata_files)) + run_command([ + llvm_profdata, "merge", + "--output", str(merged_profdata), + ] + merge_inputs) + + # --empty-profile was removed in LLVM 19; attempt to generate an equivalent + # empty profile for baseline coverage (compiled-but-untested files at 0%). + # If llvm-profdata rejects the call, baseline coverage is skipped gracefully. + _empty_path = Path.cwd() / "empty.profdata" + _empty_result = subprocess.run( + [llvm_profdata, "merge", "--sparse", "--output", str(_empty_path)], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, + ) + empty_profdata: Optional[str] = str(_empty_path) if _empty_result.returncode == 0 else None + if empty_profdata is None: + print("WARNING: Could not create empty profile — baseline coverage (0% for " + "untested compiled files) will be skipped.", file=sys.stderr) + + # Load baseline objects (production library archives) for zero-coverage baseline. + baseline_objects = load_baseline_objects( + r, args.baseline_objects, workspace_root) + + # Rust rlib archives (exposed as .a symlinks by rules_rust) start with a + # lib.rmeta member, which makes llvm-cov reject the whole archive with + # "no coverage data found" even though the .o members carry the covmap. + # Expand such archives into their object members. + baseline_objects = expand_rlib_archives( + baseline_objects, Path.cwd() / "rlib_baseline_objects") + + # Determine filter regexes: prefer allowlist-based filtering, fall back to manual regexes. + allowlist_files = [] + filter_regexes = [] + baseline_only_archives = [] + baseline_only_files = set() + + if args.coverage_allowlist: + allowlist_files = load_coverage_allowlist(r, args.coverage_allowlist) + if allowlist_files: + print(f"INFO: Using coverage allowlist with {len(allowlist_files)} source files.", + file=sys.stderr) + allowlist_set = set(allowlist_files) + + # Get files covered by test binaries. + test_covered_files = get_covered_files( + llvm_bin_path, sorted_objects, str(merged_profdata), workspace_root) + print(f"INFO: Test binaries cover {len(test_covered_files)} files.", + file=sys.stderr) + + # Get files from baseline archives via a SEPARATE llvm-cov run. + # Combining archives with test binaries in a single llvm-cov invocation + # causes some files to vanish (suspected llvm-cov deduplication issue). + # Some archives may have oversized coverage mappings ("malformed coverage + # data"), so we iteratively remove bad ones. + baseline_files = set() + if baseline_objects and empty_profdata is not None: + baseline_files = get_covered_files( + llvm_bin_path, baseline_objects, empty_profdata, workspace_root) + print(f"INFO: Baseline archives contain {len(baseline_files)} files.", + file=sys.stderr) + + # Files only in baseline archives (not in any test binary). + baseline_only_files = (baseline_files & allowlist_set) - test_covered_files + baseline_only_archives = [] + if baseline_only_files: + print(f"INFO: {len(baseline_only_files)} allowlisted files only in baseline " + f"(e.g., {sorted(baseline_only_files)[:5]})", file=sys.stderr) + # Use all valid baseline archives for LCOV generation. + # The _filter_lcov function will filter to only baseline-only files. + baseline_only_archives = list(baseline_objects) + + # Union of test + baseline for exclude-set calculation. + all_covered_files = test_covered_files | baseline_files + files_to_exclude = all_covered_files - allowlist_set + filter_regexes = [re.escape(f) + "$" for f in sorted(files_to_exclude)] + print(f"INFO: Excluding {len(filter_regexes)} files not in allowlist.", + file=sys.stderr) + else: + print("ERROR: Coverage allowlist is empty, falling back to filter_regexes.txt.", + file=sys.stderr) + sys.exit(-1) + elif args.ignore_filename_regex: + filter_regexes = args.ignore_filename_regex + print(f"INFO: Applying {len(filter_regexes)} --ignore-filename-regex exclusion pattern(s).", + file=sys.stderr) + common_args = { + "llvm_bin_path": llvm_bin_path, + "objects": sorted_objects, + "instr_profile": str(merged_profdata), + "filter_regexes": sorted(filter_regexes), + "workspace_root": workspace_root, + } + + # Generate HTML report including baseline-only files when valid archives are available. + html_report_dir = Path.cwd() / "html_report" + if baseline_only_archives: + all_html_objects = sorted_objects + baseline_only_archives + html_args = { + **common_args, + "objects": all_html_objects, + } + try: + run_llvm_cov_show( + **html_args, + output_format="html", + html_report_dir=html_report_dir, + ) + except SystemExit: + # Some baseline archives caused llvm-cov show to fail; retry with test binaries only. + print("WARNING: HTML generation with baseline archives failed; " + "falling back to test-only HTML.", file=sys.stderr) + run_llvm_cov_show( + **common_args, + output_format="html", + html_report_dir=html_report_dir, + ) + else: + run_llvm_cov_show( + **common_args, + output_format="html", + html_report_dir=html_report_dir, + ) + + # Generate LCOV report from test binaries. + lcov_report_dir = Path.cwd() / "lcov_report" + lcov_report_dir.mkdir(exist_ok=True) + lcov_result = run_llvm_cov_export(**common_args) + lcov_content = lcov_result.stdout + + # If there are baseline-only files, generate a separate baseline LCOV and merge. + if baseline_only_archives and empty_profdata is not None: + baseline_lcov_args = { + "llvm_bin_path": llvm_bin_path, + "objects": baseline_only_archives, + "instr_profile": empty_profdata, + "filter_regexes": [], # No filtering — we only have the needed archives. + "workspace_root": workspace_root, + } + baseline_lcov = run_llvm_cov_export(**baseline_lcov_args) + if baseline_lcov.stdout: + # Filter baseline LCOV to only include baseline-only files. + filtered_baseline = _filter_lcov(baseline_lcov.stdout, baseline_only_files) + if filtered_baseline: + lcov_content += filtered_baseline + print(f"INFO: Merged baseline LCOV for {len(baseline_only_files)} files.", + file=sys.stderr) + + with open(lcov_report_dir / "lcov.dat", "w", encoding="utf-8") as f: + f.write(lcov_content) + + # Generate text summary. + text_report_dir = Path.cwd() / "text_report" + text_report_dir.mkdir(exist_ok=True) + summary = run_llvm_cov_report(**common_args) + with open(text_report_dir / "summary.txt", "w", encoding="utf-8") as f: + f.write(summary.stdout) + print(summary.stdout, file=sys.stderr) + + # Package everything into the output zip. + directories = [html_report_dir, lcov_report_dir, text_report_dir] + create_zip( + root=Path.cwd(), + directories=directories, + output_file=args.output_file, + ) + + print(f"INFO: Coverage reporter completed. Output: {args.output_file}", file=sys.stderr) + + +def _filter_lcov(lcov_content: str, target_files: set) -> str: + """Filter LCOV content to only include records for target files. + + LCOV format: SF: starts a record, end_of_record ends it. + """ + result = [] + current_record = [] + include = False + + for line in lcov_content.splitlines(keepends=True): + if line.startswith("SF:"): + current_record = [line] + filepath = line[3:].strip() + # Check if the file path (or its suffix) matches any target file. + include = any(filepath.endswith(f) for f in target_files) + elif line.strip() == "end_of_record": + current_record.append(line) + if include: + result.extend(current_record) + current_record = [] + include = False + else: + current_record.append(line) + + return "".join(result) + + +def get_covered_files( + llvm_bin_path: Path, + objects: List[str], + instr_profile: Optional[str], + workspace_root: str, +) -> set: + """Run a quick llvm-cov report to discover all files with coverage data. + + Returns a set of workspace-relative file paths. + """ + cmd = [ + str(llvm_bin_path), + "report", + f"--path-equivalence=/proc/self/cwd/,{workspace_root}", + ] + cmd.extend(["--instr-profile", instr_profile]) + cmd.append(objects[0]) + for obj in objects[1:]: + cmd.extend(["--object", obj]) + + result = run_command(cmd) + if result.returncode != 0: + return set() + + files = set() + in_files = False + for line in result.stdout.splitlines(): + if line.startswith("---"): + in_files = True + continue + if line.startswith("TOTAL"): + break + if not in_files: + continue + # Extract filename (everything before first multi-space + digit sequence) + match = re.match(r"^(.+?)\s{2,}\d+", line) + if match: + filename = match.group(1).strip() + # Normalize to workspace-relative form. llvm-cov report prints the + # raw covmap path: for C++ that is the recorded compilation dir + # /proc/self/cwd/, --path-equivalence does not rewrite the + # DISPLAYED path. Rust covmap paths are already exec-root relative. + for prefix in (workspace_root, "/proc/self/cwd/"): + if filename.startswith(prefix): + filename = filename[len(prefix):] + break + files.add(filename) + + return files + + + + + +def run_llvm_cov_show( + llvm_bin_path: Path, + objects: List[str], + instr_profile: Optional[str], + filter_regexes: List[str], + workspace_root: str, + output_format: str, + html_report_dir: Path = None, +) -> subprocess.CompletedProcess: + """Run llvm-cov show.""" + cmd = [ + str(llvm_bin_path), + "show", + f"--format={output_format}", + f"--path-equivalence=/proc/self/cwd/,{workspace_root}", + f"--compilation-dir={workspace_root}", + "--show-branches=count", + "--show-region-summary=0", + ] + + cxxfilt = find_cxxfilt(llvm_bin_path) + if cxxfilt: + cmd.append(f"--Xdemangler={cxxfilt}") + + for regex in filter_regexes: + cmd.append(f"--ignore-filename-regex={regex}") + + if html_report_dir: + cmd.append(f"--output-dir={html_report_dir}") + cmd.append("--coverage-watermark=100,50") + cmd.append("--show-expansions") + + cmd.extend(["--instr-profile", instr_profile]) + cmd.append(objects[0]) + for obj in objects[1:]: + cmd.extend(["--object", obj]) + + return run_command(cmd) + + +def run_llvm_cov_export( + llvm_bin_path: Path, + objects: List[str], + instr_profile: Optional[str], + filter_regexes: List[str], + workspace_root: str, +) -> subprocess.CompletedProcess: + """Run llvm-cov export to produce LCOV format.""" + cmd = [ + str(llvm_bin_path), + "export", + "--format=lcov", + f"--path-equivalence=/proc/self/cwd/,{workspace_root}", + f"--compilation-dir={workspace_root}", + ] + + for regex in filter_regexes: + cmd.append(f"--ignore-filename-regex={regex}") + + cmd.extend(["--instr-profile", instr_profile]) + cmd.append(objects[0]) + for obj in objects[1:]: + cmd.extend(["--object", obj]) + + return run_command(cmd) + + +def run_llvm_cov_report( + llvm_bin_path: Path, + objects: List[str], + instr_profile: Optional[str], + filter_regexes: List[str], + workspace_root: str, +) -> subprocess.CompletedProcess: + """Run llvm-cov report for a text summary.""" + cmd = [ + str(llvm_bin_path), + "report", + f"--path-equivalence=/proc/self/cwd/,{workspace_root}", + "--show-region-summary=0", + "--show-branch-summary=1", + ] + + for regex in filter_regexes: + cmd.append(f"--ignore-filename-regex={regex}") + + cmd.extend(["--instr-profile", instr_profile]) + cmd.append(objects[0]) + for obj in objects[1:]: + cmd.extend(["--object", obj]) + + return run_command(cmd) + + +def extract_reports(reports: List[str]) -> Tuple[Set[str], Set[str]]: + """Extract profdata and object files from per-test zip files.""" + valid_profdata_files = set() + valid_object_files = set() + + for i, report_path in enumerate(reports): + # Skip baseline_coverage files (LCOV format, not our zip). + if "baseline_coverage" in report_path: + continue + + report = Path(report_path) + if not report.exists() or report.stat().st_size == 0: + continue + + # Check if it's a valid zip. + if not zipfile.is_zipfile(report): + continue + + profdata_name = f"coverage_report_{i:08d}.profdata" + + try: + with zipfile.ZipFile(report, "r") as archive: + # Extract meta. + meta_json = archive.read("meta/meta.json") + target_meta = json.loads(meta_json) + + # Extract profdata. + profdata_content = archive.read("profdata/target.profdata") + profdata_path = Path.cwd() / profdata_name + with open(profdata_path, "wb") as f: + f.write(profdata_content) + + valid_profdata_files.add(str(profdata_path)) + + # Collect object files. + for obj in target_meta.get("object_files", []): + if obj and Path(obj).exists(): + valid_object_files.add(os.path.realpath(obj)) + + except (zipfile.BadZipFile, KeyError, json.JSONDecodeError) as e: + print(f"WARNING: Skipping invalid report {report_path}: {e}", file=sys.stderr) + continue + + return valid_profdata_files, valid_object_files + +def read_reports_file(reports_file: Path) -> List[str]: + """Read the reports file listing all per-test coverage outputs.""" + with open(reports_file, encoding="utf-8") as f: + return [line.strip() for line in f if line.strip()] + + +def _read_ar_members(path: str) -> List[tuple]: + """Parse a Unix ar archive, returning (name, data_offset, size) tuples. + + Handles the GNU long-name table ("//" member with "/" references). + Returns an empty list when the file is not an ar archive. + """ + members = [] + longnames = b"" + with open(path, "rb") as f: + if f.read(8) != b"!\n": + return [] + while True: + header = f.read(60) + if len(header) < 60: + break + name = header[0:16].decode(errors="replace").rstrip() + try: + size = int(header[48:58].decode().strip() or "0") + except ValueError: + break + data_offset = f.tell() + if name == "//": + longnames = f.read(size) + else: + if name.startswith("/") and name[1:].isdigit(): + start = int(name[1:]) + end = longnames.find(b"\n", start) + name = longnames[start:end].decode(errors="replace").rstrip("/") + elif name.endswith("/"): + name = name[:-1] + members.append((name, data_offset, size)) + f.seek(size, 1) + if size % 2 == 1: + f.seek(1, 1) + return members + + +def expand_rlib_archives(objects: List[str], workdir: Path) -> List[str]: + """Replace Rust rlib archives with their extracted object members. + + llvm-cov rejects rlib archives ("no coverage data found") because of the + leading lib.rmeta member, even though the .o members carry the coverage + mapping. Non-rlib entries (C++ .a archives, executables) pass through + unchanged. + """ + result = [] + extracted = 0 + for obj in objects: + members = _read_ar_members(obj) if obj.endswith((".a", ".rlib")) else [] + if not any(name == "lib.rmeta" for name, _, _ in members): + result.append(obj) + continue + workdir.mkdir(parents=True, exist_ok=True) + with open(obj, "rb") as f: + for index, (name, offset, size) in enumerate(members): + if not name.endswith(".o"): + continue + f.seek(offset) + out_path = workdir / f"{Path(obj).stem}.{index}.o" + out_path.write_bytes(f.read(size)) + result.append(str(out_path)) + extracted += 1 + if extracted: + print(f"INFO: Expanded {extracted} object(s) from Rust rlib baseline archives.", + file=sys.stderr) + return result + + +def find_cxxfilt(llvm_bin_path: Path) -> str: + """Locate llvm-cxxfilt for demangling (C++ Itanium and Rust v0/legacy symbols). + + Tries the directory of llvm-cov first, then the @llvm_toolchain_llvm + distribution via runfiles (toolchains_llvm declares no alias for + llvm-cxxfilt, so it is wired as a direct data dependency). + Terminates with an error when unavailable: the binary is a declared data + dependency of the reporter, so its absence indicates a broken setup. + """ + sibling = llvm_bin_path.parent / "llvm-cxxfilt" + if sibling.exists(): + return str(sibling) + r = Runfiles.Create() + if r: + location = r.Rlocation("llvm_toolchain_llvm/bin/llvm-cxxfilt") + if location and Path(location).exists(): + return location + print( + "ERROR: llvm-cxxfilt not found (checked next to llvm-cov and in the " + "@llvm_toolchain_llvm runfiles). It is a declared data dependency of " + "the reporter; check //quality/coverage/llvm_cov:reporter.", + file=sys.stderr, + ) + sys.exit(1) + + +def load_coverage_allowlist(runfiles: Runfiles, rlocation_path: str) -> List[str]: + """Load coverage allowlist (package paths) from a file via Bazel runfiles.""" + path = runfiles.Rlocation(rlocation_path) + if not path or not Path(path).exists(): + return [] + + lines = Path(path).read_text(encoding="utf-8").splitlines() + return [line.strip() for line in lines if line.strip() and not line.strip().startswith("#")] + + +def load_baseline_objects( + runfiles: Runfiles, rlocation_path: str, workspace_root: str, +) -> List[str]: + """Load baseline object archive paths and resolve them to absolute paths. + + The objects manifest lists relative paths to .a files. When the reporter runs + in the exec config, the manifest paths use the exec config dir + (e.g., k8-opt-exec-*). + """ + if not rlocation_path: + return [] + + path = runfiles.Rlocation(rlocation_path) + if not path or not Path(path).exists(): + print(f"WARNING: Baseline objects manifest not found: {rlocation_path}", file=sys.stderr) + return [] + + lines = Path(path).read_text(encoding="utf-8").splitlines() + resolved = [] + for line in lines: + line = line.strip() + if not line or line.startswith("#"): + continue + # Dynamically gets the canonical repository name + repo_root = runfiles.CurrentRepository() + if not repo_root: + repo_root = "_main" # Safe Bzlmod root fallback + + # Cleanly stitch the path together + path = runfiles.Rlocation(os.path.join(repo_root, line)) + if os.path.exists(path): + resolved.append(path) + else: + print(f"ERROR: Baseline object not found: {line}", file=sys.stderr) + sys.exit(-1) + return sorted(resolved) + + +def run_command(cmd: List[str]) -> subprocess.CompletedProcess: + """Run a command and exit on failure.""" + try: + return subprocess.run( + cmd, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) + except subprocess.CalledProcessError as e: + print(f"ERROR: Command failed with code {e.returncode}:", file=sys.stderr) + print(f" {' '.join(cmd[:10])}{'...' if len(cmd) > 10 else ''}", file=sys.stderr) + if e.stdout: + print(e.stdout, file=sys.stderr) + sys.exit(1) + + +def create_zip(root: Path, directories: List[Path], output_file: Path) -> None: + """Create a zip file from the given directories relative to root.""" + with zipfile.ZipFile(output_file, "w", zipfile.ZIP_DEFLATED) as zf: + for directory in directories: + if not directory.exists(): + continue + for dirpath, _, files in os.walk(directory): + for filename in files: + file_path = Path(dirpath) / filename + arcname = file_path.relative_to(root) + zf.write(file_path, arcname) + + +def parse_args() -> argparse.Namespace: + """Parse command-line arguments matching the Bazel coverage_report_generator interface.""" + parser = argparse.ArgumentParser(description="LLVM coverage reporter for Bazel") + parser.add_argument("--output_file", type=Path, required=True) + parser.add_argument("--reports_file", type=Path, required=True) + parser.add_argument("--coverage_allowlist", type=str, default=None, + help="Rlocation path to the coverage allowlist file (preferred over filter_regexes)") + parser.add_argument("--ignore_filename_regex", action="append", default=[], + help="Path regex to exclude from the report (repeatable). Used when no allowlist is set.") + parser.add_argument("--baseline_objects", type=str, default=None, + help="Rlocation path to the baseline objects manifest (archive .a files)") + parser.add_argument("--workspace_root", type=str, default=None, + help="Real workspace root. Auto-detected from MODULE.bazel if not set.") + return parser.parse_args() + + + +if __name__ == "__main__": + main() diff --git a/quality/coverage/llvm_cov/reporter_wrapper.bzl b/quality/coverage/llvm_cov/reporter_wrapper.bzl new file mode 100644 index 00000000..f6737df0 --- /dev/null +++ b/quality/coverage/llvm_cov/reporter_wrapper.bzl @@ -0,0 +1,87 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +"""Executable wrapper rule for coverage reporter. + +Bakes the coverage_scope allowlist and baseline-objects manifest into the +reporter launcher so no manual --ignore_filename_regex flags are needed. +The workspace root is auto-detected by reporter.py via Path.cwd().resolve(). +""" + +visibility(["//..."]) + +def _reporter_wrapper_impl(ctx): + launcher = ctx.actions.declare_file(ctx.label.name + ".sh") + + reporter = ctx.executable.reporter + coverage_scope = ctx.attr.coverage_scope + allowlist_group = coverage_scope[OutputGroupInfo].allowlist.to_list() + objects_group = coverage_scope[OutputGroupInfo].objects.to_list() + object_files = coverage_scope[OutputGroupInfo].object_files + + if len(allowlist_group) != 1: + fail("coverage_scope must provide exactly one allowlist file") + if len(objects_group) != 1: + fail("coverage_scope must provide exactly one objects manifest file") + + allowlist = allowlist_group[0] + baseline_objects = objects_group[0] + + script = """#!/usr/bin/env bash +set -euo pipefail +if [[ -z "${{RUNFILES_DIR:-}}" ]]; then + if [[ -d "$0.runfiles" ]]; then + export RUNFILES_DIR="$0.runfiles" + fi +fi +exec "${{RUNFILES_DIR}}/{reporter}" \\ + --coverage_allowlist="{allowlist}" \\ + --baseline_objects="{baseline_objects}" \\ + "$@" +""".format( + reporter = "_main/" + reporter.short_path, + allowlist = "_main/" + allowlist.short_path, + baseline_objects = "_main/" + baseline_objects.short_path, + ) + + ctx.actions.write( + output = launcher, + content = script, + is_executable = True, + ) + + runfiles = ctx.runfiles( + files = [reporter, allowlist, baseline_objects], + transitive_files = object_files, + ).merge(ctx.attr.reporter[DefaultInfo].default_runfiles) + + return [DefaultInfo( + executable = launcher, + runfiles = runfiles, + )] + +reporter_wrapper = rule( + implementation = _reporter_wrapper_impl, + executable = True, + attrs = { + "reporter": attr.label( + executable = True, + cfg = "exec", + doc = "The underlying reporter py_binary.", + ), + "coverage_scope": attr.label( + cfg = "target", + doc = "The coverage_scope target that provides the allowlist and objects manifest.", + ), + }, +)