Add downstream coverage workflow tests

.github/workflows/downstream_tests.yml

@@ -0,0 +1,73 @@
+# *******************************************************************************
+# Copyright (c) 2026 Contributors to the Eclipse Foundation
+#
+# See the NOTICE file(s) distributed with this work for additional
+# information regarding copyright ownership.
+#
+# This program and the accompanying materials are made available under the
+# terms of the Apache License Version 2.0 which is available at
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# SPDX-License-Identifier: Apache-2.0
+# *******************************************************************************
+name: Downstream tests
+
+on:
+  pull_request:
+    types: [opened, reopened, synchronize]
+  push:
+    branches: [main]
+  merge_group:
+    types: [checks_requested]
+
+concurrency:
+  group: downstream-tests-${{ github.event.pull_request.number || github.run_id }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+permissions:
+  contents: read
+
+jobs:
+  consumer:
+    name: ${{ matrix.consumer }}
+    runs-on: ubuntu-24.04
+    strategy:
+      fail-fast: false
+      matrix:
+        consumer: [baselibs, lifecycle]
+    steps:
+      - name: Checkout coverage_tool
+        uses: actions/checkout@v7.0.1
+
+      - name: Free disk space
+        uses: eclipse-score/more-disk-space@v1
+        with:
+          level: 4
+
+      - name: Setup Bazel cache
+        uses: eclipse-score/cicd-actions/setup-bazel-cache@setup-bazel-cache/v0.1.0
+        with:
+          disk-cache-key: downstream-${{ matrix.consumer }}
+
+      - name: Run the consumer's coverage workflow
+        run: bazel test --lockfile_mode=error //tools/downstream_tests:${{ matrix.consumer }}
+
+      - name: Publish coverage summary
+        if: always()
+        env:
+          REPORT_ARCHIVE: bazel-testlogs/tools/downstream_tests/${{ matrix.consumer }}/test.outputs/coverage-report.zip
+        run: |
+          if [ -f "$REPORT_ARCHIVE" ] && unzip -Z1 "$REPORT_ARCHIVE" | grep -x coverage_summary.md > /dev/null; then
+            unzip -p "$REPORT_ARCHIVE" coverage_summary.md >> "$GITHUB_STEP_SUMMARY"
+          fi
+
+      - name: Upload pytest results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: downstream-${{ matrix.consumer }}-results
+          path: |
+            bazel-testlogs/tools/downstream_tests/${{ matrix.consumer }}/test.xml
+            bazel-testlogs/tools/downstream_tests/${{ matrix.consumer }}/test.outputs/coverage-report.zip
+          if-no-files-found: ignore
+          retention-days: 3

.github/workflows/tests.yml

@@ -45,7 +45,7 @@ jobs:
       - name: Build everything
         run: bazel build --lockfile_mode=error //...
       - name: Run unit and Starlark analysis tests
-        run: bazel test --lockfile_mode=error //score_coverage/... //tools/...
+        run: bazel test --lockfile_mode=error --test_tag_filters=-integration //score_coverage/... //tools/...
       - name: Static analysis of the Python (ruff, pylint, ty; findings fail the build)
         run: bazel build --lockfile_mode=error --config=lint //score_coverage/... //tools/...
       - name: Measure structural coverage of the tool itself (coverage.py)

MODULE.bazel

@@ -59,6 +59,7 @@ use_repo(pip, "pip_score_coverage")
 # Development-only dependencies (repository hygiene: copyright, formatting)
 ###############################################################################
 bazel_dep(name = "score_tooling", version = "2.2.0", dev_dependency = True)
+bazel_dep(name = "score_tools", version = "0.0.3", dev_dependency = True)
 
 # use_format_targets() (from score_tooling) loads these from the ROOT module's
 # repo mapping, so the root has to declare them itself.

tools/downstream_tests/BUILD

@@ -0,0 +1,46 @@
+# *******************************************************************************
+# Copyright (c) 2026 Contributors to the Eclipse Foundation
+#
+# See the NOTICE file(s) distributed with this work for additional
+# information regarding copyright ownership.
+#
+# This program and the accompanying materials are made available under the
+# terms of the Apache License Version 2.0 which is available at
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# SPDX-License-Identifier: Apache-2.0
+# *******************************************************************************
+
+load("@score_tools//score_pytest:pytest.bzl", "score_pytest")
+
+# Each test runs the consumer's full coverage pipeline, which can take many
+# minutes on a cold CI runner. The tests need network access to clone current
+# consumer main branches and run Bazel in those nested workspaces.
+#
+# Keep each consumer scenario in its own target so CI and JUnit reports show
+# exactly which real downstream workflow failed.
+score_pytest(
+    name = "baselibs",
+    size = "large",
+    timeout = "eternal",
+    srcs = ["test_baselibs.py"],
+    data = ["_consumer_workspace.py"],
+    imports = ["."],
+    tags = [
+        "integration",
+        "local",
+    ],
+)
+
+score_pytest(
+    name = "lifecycle",
+    size = "large",
+    timeout = "eternal",
+    srcs = ["test_lifecycle.py"],
+    data = ["_consumer_workspace.py"],
+    imports = ["."],
+    tags = [
+        "integration",
+        "local",
+    ],
+)

tools/downstream_tests/_consumer_workspace.py

@@ -0,0 +1,162 @@
+# *******************************************************************************
+# Copyright (c) 2026 Contributors to the Eclipse Foundation
+#
+# See the NOTICE file(s) distributed with this work for additional
+# information regarding copyright ownership.
+#
+# This program and the accompanying materials are made available under the
+# terms of the Apache License Version 2.0 which is available at
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# SPDX-License-Identifier: Apache-2.0
+# *******************************************************************************
+"""Prepare real consumer workspaces for the downstream coverage scenarios."""
+
+from __future__ import annotations
+
+import os
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+
+_COVERAGE_TOOL_ROOT = Path(__file__).resolve().parents[2]
+
+
+def clone_consumer(name: str, parent_directory: Path) -> Path:
+    """Clone a consumer's default branch and point it at this checkout."""
+    workspace = parent_directory / name
+    repository_url = f"https://github.com/eclipse-score/{name}.git"
+    subprocess.run(
+        ["git", "clone", "--depth", "1", repository_url, str(workspace)],
+        check=True,
+    )
+
+    tool_link = parent_directory / "coverage_tool"
+    tool_link.symlink_to(_COVERAGE_TOOL_ROOT, target_is_directory=True)
+
+    module_file = workspace / "MODULE.bazel"
+    module_contents = module_file.read_text(encoding="utf-8")
+    module_contents = (
+        module_contents.rstrip()
+        + '\n\nlocal_path_override(\n    module_name = "score_coverage",\n    path = "../coverage_tool",\n)\n'
+    )
+    module_file.write_text(module_contents, encoding="utf-8")
+
+    # Updating the consumer lockfile for the local override lets its production
+    # commands keep their normal --lockfile_mode=error setting.
+    subprocess.run(
+        ["bazel", "mod", "deps", "--lockfile_mode=update"],
+        cwd=workspace,
+        check=True,
+    )
+    return workspace
+
+
+def run_bazel(
+    workspace: Path,
+    *arguments: str,
+    extra_environment: dict[str, str] | None = None,
+) -> None:
+    """Run one consumer-facing Bazel command, keeping failure output concise."""
+    environment = os.environ.copy()
+    if extra_environment:
+        environment.update(extra_environment)
+
+    command = ["bazel", *arguments]
+    result = subprocess.run(
+        command,
+        cwd=workspace,
+        env=environment,
+        capture_output=True,
+        check=False,
+        text=True,
+    )
+    if result.returncode:
+        output = (result.stdout + result.stderr).splitlines()
+        output_tail = "\n".join(output[-80:])
+        raise AssertionError(
+            f"`{' '.join(command)}` exited with code {result.returncode}.\nLast Bazel output lines:\n{output_tail}"
+        )
+
+
+def retain_coverage_results(workspace: Path, archive_directory: str) -> None:
+    """Keep whatever report files exist, including reports from a failed gate."""
+    archive = workspace / archive_directory
+    html_report = archive / "coverage_linux"
+    lcov_report = archive / "coverage_report.dat"
+    outputs_directory = os.environ.get("TEST_UNDECLARED_OUTPUTS_DIR")
+    if not archive.is_dir() or not outputs_directory:
+        return
+
+    retained_report = Path(outputs_directory) / "coverage-report"
+    summary_markdown = workspace / "coverage_summary.md"
+    try:
+        retained_report.mkdir(parents=True, exist_ok=True)
+        if html_report.is_dir():
+            shutil.copytree(html_report, retained_report / "coverage_linux")
+        if lcov_report.is_file():
+            shutil.copy2(lcov_report, retained_report / lcov_report.name)
+        if summary_markdown.is_file():
+            shutil.copy2(summary_markdown, retained_report / summary_markdown.name)
+
+        for name in ("justification_report", "unmapped_files.txt"):
+            result = archive / name
+            if result.is_dir():
+                shutil.copytree(result, retained_report / name)
+            elif result.is_file():
+                shutil.copy2(result, retained_report / name)
+    except Exception as error:
+        print(f"Could not copy all coverage report files: {error}", file=sys.stderr)
+
+    try:
+        report_archive = Path(shutil.make_archive(str(retained_report), "zip", retained_report))
+    except Exception as error:
+        print(f"Could not archive the coverage report: {error}", file=sys.stderr)
+        return
+
+    try:
+        shutil.rmtree(retained_report)
+    except OSError as error:
+        print(f"Could not remove the unpacked coverage report: {error}", file=sys.stderr)
+
+    print(f"Coverage report retained at {report_archive}.")
+
+
+def verify_coverage_results(workspace: Path, archive_directory: str) -> None:
+    """Require measurable coverage after retaining all available report files."""
+    archive = workspace / archive_directory
+    html_report = archive / "coverage_linux"
+    lcov_report = archive / "coverage_report.dat"
+    summary_markdown = workspace / "coverage_summary.md"
+
+    assert (html_report / "index.html").is_file(), "Coverage HTML index was not generated"
+    assert lcov_report.is_file(), "LCOV coverage report was not generated"
+    assert summary_markdown.is_file(), "Markdown coverage summary was not generated"
+
+    source_files = 0
+    lines_found = 0
+    lines_hit = 0
+    for line in lcov_report.read_text(encoding="utf-8", errors="replace").splitlines():
+        if line.startswith("SF:"):
+            source_files += 1
+        elif line.startswith("LF:"):
+            lines_found += int(line[3:])
+        elif line.startswith("LH:"):
+            lines_hit += int(line[3:])
+
+    assert source_files > 0, "LCOV report contains no source files"
+    assert lines_found > 0, "LCOV report contains no measurable lines"
+    assert lines_hit > 0, "LCOV report contains no covered lines"
+
+    outputs_directory = os.environ.get("TEST_UNDECLARED_OUTPUTS_DIR")
+    assert outputs_directory, "Bazel did not provide TEST_UNDECLARED_OUTPUTS_DIR"
+    report_archive = Path(outputs_directory) / "coverage-report.zip"
+    assert report_archive.is_file(), "Coverage report was not retained for Bazel"
+
+    coverage_percent = 100 * lines_hit / lines_found
+    print(
+        f"Coverage report contains {source_files} source files and "
+        f"{lines_hit}/{lines_found} covered lines ({coverage_percent:.2f}%). "
+        f"Report retained at {report_archive}."
+    )

tools/downstream_tests/test_baselibs.py

@@ -0,0 +1,60 @@
+# *******************************************************************************
+# Copyright (c) 2026 Contributors to the Eclipse Foundation
+#
+# See the NOTICE file(s) distributed with this work for additional
+# information regarding copyright ownership.
+#
+# This program and the accompanying materials are made available under the
+# terms of the Apache License Version 2.0 which is available at
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# SPDX-License-Identifier: Apache-2.0
+# *******************************************************************************
+"""Black-box test for baselibs' production LLVM coverage workflow."""
+
+from pathlib import Path
+
+from _consumer_workspace import clone_consumer, retain_coverage_results, run_bazel, verify_coverage_results
+
+
+def test_baselibs_coverage_workflow_reports_measured_coverage(tmp_path: Path) -> None:
+    """The checked-in baselibs workflow produces a non-empty coverage report."""
+    workspace = clone_consumer("baselibs", tmp_path)
+
+    run_bazel(
+        workspace,
+        "coverage",
+        "--lockfile_mode=error",
+        "--config=llvm_cov",
+        "--build_tests_only",
+        "--",
+        "//score/...",
+        # This upstream-main test currently fails under the coverage config
+        # because its compact-JSON expectations conflict with default pretty printing.
+        "-//score/json/internal/writer/vajson:vajson_serialize_test",
+        "-//score/language/safecpp/aborts_upon_exception/...",
+        "-//score/language/safecpp/safe_math/details:floating_point_environment_test",
+        "-//score/os/linux/utils/test:network_interface_test",
+    )
+    # The generator assembles artifacts before returning a gate failure.
+    try:
+        run_bazel(
+            workspace,
+            "run",
+            "--lockfile_mode=error",
+            "@score_coverage//:generate_coverage_html",
+            "--",
+            "--yaml",
+            "tools/coverage/coverage_justifications.yaml",
+            "--summary-md",
+            "coverage_summary.md",
+            "--testlogs-subdir",
+            "score",
+            "--archive-dir",
+            "coverage_artifact",
+            extra_environment={"COVERAGE_THRESHOLD": "0"},
+        )
+    finally:
+        retain_coverage_results(workspace, "coverage_artifact")
+
+    verify_coverage_results(workspace, "coverage_artifact")