From cd28556c61edc4b9161136abcde71f04c6ba47b2 Mon Sep 17 00:00:00 2001 From: Alexander Lanin Date: Fri, 2 Oct 2026 23:19:54 +0200 Subject: [PATCH 1/7] test: add downstream coverage checks --- .github/workflows/downstream_tests.yml | 60 ++++++++++++++ .github/workflows/tests.yml | 2 +- MODULE.bazel | 1 + MODULE.bazel.lock | 2 + tools/downstream_tests/BUILD | 46 +++++++++++ tools/downstream_tests/_consumer_workspace.py | 78 +++++++++++++++++++ tools/downstream_tests/test_baselibs.py | 55 +++++++++++++ tools/downstream_tests/test_lifecycle.py | 45 +++++++++++ 8 files changed, 288 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/downstream_tests.yml create mode 100644 tools/downstream_tests/BUILD create mode 100644 tools/downstream_tests/_consumer_workspace.py create mode 100644 tools/downstream_tests/test_baselibs.py create mode 100644 tools/downstream_tests/test_lifecycle.py diff --git a/.github/workflows/downstream_tests.yml b/.github/workflows/downstream_tests.yml new file mode 100644 index 0000000..e404f44 --- /dev/null +++ b/.github/workflows/downstream_tests.yml @@ -0,0 +1,60 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +name: Downstream tests + +on: + pull_request: + types: [opened, reopened, synchronize] + merge_group: + types: [checks_requested] + +concurrency: + group: downstream-tests-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +permissions: + contents: read + +jobs: + consumer: + name: ${{ matrix.consumer }} + runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: + consumer: [baselibs, lifecycle] + steps: + - name: Checkout coverage_tool + uses: actions/checkout@v7.0.1 + + - name: Free disk space + uses: eclipse-score/more-disk-space@v1 + with: + level: 4 + + - name: Setup Bazel cache + uses: eclipse-score/cicd-actions/setup-bazel-cache@setup-bazel-cache/v0.1.0 + with: + disk-cache-key: downstream-${{ matrix.consumer }} + + - name: Run the consumer's coverage workflow + run: bazel test --lockfile_mode=error //tools/downstream_tests:${{ matrix.consumer }} + + - name: Upload pytest results + if: always() + uses: actions/upload-artifact@v4 + with: + name: downstream-${{ matrix.consumer }}-pytest-results + path: bazel-testlogs/tools/downstream_tests/${{ matrix.consumer }}/test.xml + if-no-files-found: ignore + retention-days: 3 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 2741efe..d5b3473 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -45,7 +45,7 @@ jobs: - name: Build everything run: bazel build --lockfile_mode=error //... - name: Run unit and Starlark analysis tests - run: bazel test --lockfile_mode=error //score_coverage/... //tools/... + run: bazel test --lockfile_mode=error --test_tag_filters=-integration //score_coverage/... //tools/... - name: Static analysis of the Python (ruff, pylint, ty; findings fail the build) run: bazel build --lockfile_mode=error --config=lint //score_coverage/... //tools/... - name: Measure structural coverage of the tool itself (coverage.py) diff --git a/MODULE.bazel b/MODULE.bazel index e1cfd46..00a2f6f 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -59,6 +59,7 @@ use_repo(pip, "pip_score_coverage") # Development-only dependencies (repository hygiene: copyright, formatting) ############################################################################### bazel_dep(name = "score_tooling", version = "2.2.0", dev_dependency = True) +bazel_dep(name = "score_tools", version = "0.0.3", dev_dependency = True) # use_format_targets() (from score_tooling) loads these from the ROOT module's # repo mapping, so the root has to declare them itself. diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 939ccf6..650c2d2 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -821,6 +821,8 @@ "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/score_toolchains_rust/0.10.0/source.json": "8bda773be264da16d2a82a03ebb737421dd4a35855f1e9a5d03d9722d84c1df5", "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/score_tooling/2.2.0/MODULE.bazel": "178ba4862246b6ba2bbcd96b7e9e728299b19fb94bcf23d315dc2299aabf7178", "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/score_tooling/2.2.0/source.json": "a76f2d093cff26d5b256ce531bb2f69b6c667c968f99a156327fd194d4f36e61", + "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/score_tools/0.0.3/MODULE.bazel": "69f441bf28d938de2b6aef61ec91e65633f8ea908d2527e6f5a424895c9ac388", + "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/score_tools/0.0.3/source.json": "84fda1c7e1e73811b67c239ce4d10800132c87dbc0af549ab425e66e679dc37f", "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/sphinxdocs/2.2.0/MODULE.bazel": "not found", "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/stardoc/0.5.0/MODULE.bazel": "not found", "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/stardoc/0.5.1/MODULE.bazel": "not found", diff --git a/tools/downstream_tests/BUILD b/tools/downstream_tests/BUILD new file mode 100644 index 0000000..e17d07e --- /dev/null +++ b/tools/downstream_tests/BUILD @@ -0,0 +1,46 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +load("@score_tools//score_pytest:pytest.bzl", "score_pytest") + +# Each test runs the consumer's full coverage pipeline, which can take many +# minutes on a cold CI runner. The tests need network access to clone current +# consumer main branches and run Bazel in those nested workspaces. +# +# Keep each consumer scenario in its own target so CI and JUnit reports show +# exactly which real downstream workflow failed. +score_pytest( + name = "baselibs", + size = "large", + timeout = "eternal", + srcs = ["test_baselibs.py"], + data = ["_consumer_workspace.py"], + imports = ["."], + tags = [ + "integration", + "local", + ], +) + +score_pytest( + name = "lifecycle", + size = "large", + timeout = "eternal", + srcs = ["test_lifecycle.py"], + data = ["_consumer_workspace.py"], + imports = ["."], + tags = [ + "integration", + "local", + ], +) diff --git a/tools/downstream_tests/_consumer_workspace.py b/tools/downstream_tests/_consumer_workspace.py new file mode 100644 index 0000000..656c595 --- /dev/null +++ b/tools/downstream_tests/_consumer_workspace.py @@ -0,0 +1,78 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +"""Prepare real consumer workspaces for the downstream coverage scenarios.""" + +from __future__ import annotations + +import os +import subprocess +from pathlib import Path + +_COVERAGE_TOOL_ROOT = Path(__file__).resolve().parents[2] + + +def clone_consumer(name: str, parent_directory: Path) -> Path: + """Clone a consumer's default branch and point it at this checkout.""" + workspace = parent_directory / name + repository_url = f"https://github.com/eclipse-score/{name}.git" + subprocess.run( + ["git", "clone", "--depth", "1", repository_url, str(workspace)], + check=True, + ) + + tool_link = parent_directory / "coverage_tool" + tool_link.symlink_to(_COVERAGE_TOOL_ROOT, target_is_directory=True) + + module_file = workspace / "MODULE.bazel" + module_contents = module_file.read_text(encoding="utf-8") + module_contents = ( + module_contents.rstrip() + + '\n\nlocal_path_override(\n module_name = "score_coverage",\n path = "../coverage_tool",\n)\n' + ) + module_file.write_text(module_contents, encoding="utf-8") + + # Updating the consumer lockfile for the local override lets its production + # commands keep their normal --lockfile_mode=error setting. + subprocess.run( + ["bazel", "mod", "deps", "--lockfile_mode=update"], + cwd=workspace, + check=True, + ) + return workspace + + +def run_bazel( + workspace: Path, + *arguments: str, + extra_environment: dict[str, str] | None = None, +) -> None: + """Run one consumer-facing Bazel command, keeping failure output concise.""" + environment = os.environ.copy() + if extra_environment: + environment.update(extra_environment) + + command = ["bazel", *arguments] + result = subprocess.run( + command, + cwd=workspace, + env=environment, + capture_output=True, + check=False, + text=True, + ) + if result.returncode: + output = (result.stdout + result.stderr).splitlines() + output_tail = "\n".join(output[-80:]) + raise AssertionError( + f"`{' '.join(command)}` exited with code {result.returncode}.\nLast Bazel output lines:\n{output_tail}" + ) diff --git a/tools/downstream_tests/test_baselibs.py b/tools/downstream_tests/test_baselibs.py new file mode 100644 index 0000000..68908cf --- /dev/null +++ b/tools/downstream_tests/test_baselibs.py @@ -0,0 +1,55 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +"""Black-box test for baselibs' production LLVM coverage workflow.""" + +from pathlib import Path + +from _consumer_workspace import clone_consumer, run_bazel + + +def test_baselibs_coverage_workflow_creates_report(tmp_path: Path) -> None: + """The checked-in baselibs coverage commands work with this tool checkout.""" + workspace = clone_consumer("baselibs", tmp_path) + + run_bazel( + workspace, + "coverage", + "--lockfile_mode=error", + "--config=llvm_cov", + "--build_tests_only", + "--", + "//score/...", + # This upstream-main test currently fails under the coverage config + # because its compact-JSON expectations conflict with default pretty printing. + "-//score/json/internal/writer/vajson:vajson_serialize_test", + "-//score/language/safecpp/aborts_upon_exception/...", + "-//score/language/safecpp/safe_math/details:floating_point_environment_test", + "-//score/os/linux/utils/test:network_interface_test", + ) + run_bazel( + workspace, + "run", + "--lockfile_mode=error", + "@score_coverage//:generate_coverage_html", + "--", + "--yaml", + "tools/coverage/coverage_justifications.yaml", + "--testlogs-subdir", + "score", + "--archive-dir", + "coverage_artifact", + extra_environment={"COVERAGE_THRESHOLD": "0"}, + ) + + assert (workspace / "coverage_artifact/coverage_linux/index.html").is_file() + assert (workspace / "coverage_artifact/coverage_report.dat").is_file() diff --git a/tools/downstream_tests/test_lifecycle.py b/tools/downstream_tests/test_lifecycle.py new file mode 100644 index 0000000..afe02ac --- /dev/null +++ b/tools/downstream_tests/test_lifecycle.py @@ -0,0 +1,45 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +"""Black-box test for lifecycle's production Linux coverage workflow.""" + +from pathlib import Path + +from _consumer_workspace import clone_consumer, run_bazel + + +def test_lifecycle_coverage_workflow_creates_report(tmp_path: Path) -> None: + """The checked-in lifecycle coverage commands work with this tool checkout.""" + workspace = clone_consumer("lifecycle", tmp_path) + + run_bazel( + workspace, + "coverage", + "--config=llvm_cov", + "//score/...", + "--lockfile_mode=error", + "--build_tests_only", + ) + run_bazel( + workspace, + "run", + "@score_coverage//:generate_coverage_html", + "--", + "--yaml", + "quality/coverage/coverage_justifications.yaml", + "--archive-dir", + "coverage_artifacts", + extra_environment={"COVERAGE_THRESHOLD": "66"}, + ) + + assert (workspace / "coverage_artifacts/coverage_linux/index.html").is_file() + assert (workspace / "coverage_artifacts/coverage_report.dat").is_file() From a72df627e6cc37948ded1f61d087e06fb95e3758 Mon Sep 17 00:00:00 2001 From: Alexander Lanin Date: Fri, 2 Oct 2026 23:52:31 +0200 Subject: [PATCH 2/7] ci: seed downstream cache on main --- .github/workflows/downstream_tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/downstream_tests.yml b/.github/workflows/downstream_tests.yml index e404f44..32fe81c 100644 --- a/.github/workflows/downstream_tests.yml +++ b/.github/workflows/downstream_tests.yml @@ -15,6 +15,8 @@ name: Downstream tests on: pull_request: types: [opened, reopened, synchronize] + push: + branches: [main] merge_group: types: [checks_requested] From b6bb6cfa4ee2e850cd06328929ad5300cba74da0 Mon Sep 17 00:00:00 2001 From: Alexander Lanin Date: Sat, 3 Oct 2026 00:05:24 +0200 Subject: [PATCH 3/7] test: publish downstream coverage results --- .github/workflows/downstream_tests.yml | 8 ++-- tools/downstream_tests/_consumer_workspace.py | 45 +++++++++++++++++++ tools/downstream_tests/test_baselibs.py | 9 ++-- tools/downstream_tests/test_lifecycle.py | 9 ++-- 4 files changed, 58 insertions(+), 13 deletions(-) diff --git a/.github/workflows/downstream_tests.yml b/.github/workflows/downstream_tests.yml index 32fe81c..68c5b77 100644 --- a/.github/workflows/downstream_tests.yml +++ b/.github/workflows/downstream_tests.yml @@ -50,13 +50,15 @@ jobs: disk-cache-key: downstream-${{ matrix.consumer }} - name: Run the consumer's coverage workflow - run: bazel test --lockfile_mode=error //tools/downstream_tests:${{ matrix.consumer }} + run: bazel test --lockfile_mode=error --test_env=GITHUB_STEP_SUMMARY //tools/downstream_tests:${{ matrix.consumer }} - name: Upload pytest results if: always() uses: actions/upload-artifact@v4 with: - name: downstream-${{ matrix.consumer }}-pytest-results - path: bazel-testlogs/tools/downstream_tests/${{ matrix.consumer }}/test.xml + name: downstream-${{ matrix.consumer }}-results + path: | + bazel-testlogs/tools/downstream_tests/${{ matrix.consumer }}/test.xml + bazel-testlogs/tools/downstream_tests/${{ matrix.consumer }}/test.outputs/outputs.zip if-no-files-found: ignore retention-days: 3 diff --git a/tools/downstream_tests/_consumer_workspace.py b/tools/downstream_tests/_consumer_workspace.py index 656c595..5df272d 100644 --- a/tools/downstream_tests/_consumer_workspace.py +++ b/tools/downstream_tests/_consumer_workspace.py @@ -15,6 +15,7 @@ from __future__ import annotations import os +import shutil import subprocess from pathlib import Path @@ -76,3 +77,47 @@ def run_bazel( raise AssertionError( f"`{' '.join(command)}` exited with code {result.returncode}.\nLast Bazel output lines:\n{output_tail}" ) + + +def publish_coverage_results(workspace: Path, archive_directory: str) -> None: + """Check for measured coverage and retain the report as a Bazel test output.""" + archive = workspace / archive_directory + html_report = archive / "coverage_linux" + lcov_report = archive / "coverage_report.dat" + + assert (html_report / "index.html").is_file(), "Coverage HTML index was not generated" + assert lcov_report.is_file(), "LCOV coverage report was not generated" + + source_files = 0 + lines_found = 0 + lines_hit = 0 + for line in lcov_report.read_text(encoding="utf-8", errors="replace").splitlines(): + if line.startswith("SF:"): + source_files += 1 + elif line.startswith("LF:"): + lines_found += int(line[3:]) + elif line.startswith("LH:"): + lines_hit += int(line[3:]) + + assert source_files > 0, "LCOV report contains no source files" + assert lines_found > 0, "LCOV report contains no measurable lines" + assert lines_hit > 0, "LCOV report contains no covered lines" + + outputs_directory = os.environ.get("TEST_UNDECLARED_OUTPUTS_DIR") + assert outputs_directory, "Bazel did not provide TEST_UNDECLARED_OUTPUTS_DIR" + retained_report = Path(outputs_directory) / "coverage-report" + shutil.copytree(html_report, retained_report / "coverage_linux") + shutil.copy2(lcov_report, retained_report / lcov_report.name) + + for name in ("justification_report", "unmapped_files.txt"): + result = archive / name + if result.is_dir(): + shutil.copytree(result, retained_report / name) + elif result.is_file(): + shutil.copy2(result, retained_report / name) + + coverage_percent = 100 * lines_hit / lines_found + print( + f"Coverage report contains {source_files} source files and " + f"{lines_hit}/{lines_found} covered lines ({coverage_percent:.2f}%)." + ) diff --git a/tools/downstream_tests/test_baselibs.py b/tools/downstream_tests/test_baselibs.py index 68908cf..e087fb0 100644 --- a/tools/downstream_tests/test_baselibs.py +++ b/tools/downstream_tests/test_baselibs.py @@ -14,11 +14,11 @@ from pathlib import Path -from _consumer_workspace import clone_consumer, run_bazel +from _consumer_workspace import clone_consumer, publish_coverage_results, run_bazel -def test_baselibs_coverage_workflow_creates_report(tmp_path: Path) -> None: - """The checked-in baselibs coverage commands work with this tool checkout.""" +def test_baselibs_coverage_workflow_reports_measured_coverage(tmp_path: Path) -> None: + """The checked-in baselibs workflow produces a non-empty coverage report.""" workspace = clone_consumer("baselibs", tmp_path) run_bazel( @@ -51,5 +51,4 @@ def test_baselibs_coverage_workflow_creates_report(tmp_path: Path) -> None: extra_environment={"COVERAGE_THRESHOLD": "0"}, ) - assert (workspace / "coverage_artifact/coverage_linux/index.html").is_file() - assert (workspace / "coverage_artifact/coverage_report.dat").is_file() + publish_coverage_results(workspace, "coverage_artifact") diff --git a/tools/downstream_tests/test_lifecycle.py b/tools/downstream_tests/test_lifecycle.py index afe02ac..d29afc5 100644 --- a/tools/downstream_tests/test_lifecycle.py +++ b/tools/downstream_tests/test_lifecycle.py @@ -14,11 +14,11 @@ from pathlib import Path -from _consumer_workspace import clone_consumer, run_bazel +from _consumer_workspace import clone_consumer, publish_coverage_results, run_bazel -def test_lifecycle_coverage_workflow_creates_report(tmp_path: Path) -> None: - """The checked-in lifecycle coverage commands work with this tool checkout.""" +def test_lifecycle_coverage_workflow_reports_measured_coverage(tmp_path: Path) -> None: + """The checked-in workflow meets its gate and produces a coverage report.""" workspace = clone_consumer("lifecycle", tmp_path) run_bazel( @@ -41,5 +41,4 @@ def test_lifecycle_coverage_workflow_creates_report(tmp_path: Path) -> None: extra_environment={"COVERAGE_THRESHOLD": "66"}, ) - assert (workspace / "coverage_artifacts/coverage_linux/index.html").is_file() - assert (workspace / "coverage_artifacts/coverage_report.dat").is_file() + publish_coverage_results(workspace, "coverage_artifacts") From 81359199005f6385b03156b52ef48d136bc84b68 Mon Sep 17 00:00:00 2001 From: Alexander Lanin Date: Sat, 3 Oct 2026 02:25:32 +0200 Subject: [PATCH 4/7] fix: publish downstream coverage summaries --- .github/workflows/downstream_tests.yml | 2 +- tools/downstream_tests/_consumer_workspace.py | 16 +++++++++++++++- tools/downstream_tests/test_baselibs.py | 2 ++ tools/downstream_tests/test_lifecycle.py | 2 ++ 4 files changed, 20 insertions(+), 2 deletions(-) diff --git a/.github/workflows/downstream_tests.yml b/.github/workflows/downstream_tests.yml index 68c5b77..21145ef 100644 --- a/.github/workflows/downstream_tests.yml +++ b/.github/workflows/downstream_tests.yml @@ -59,6 +59,6 @@ jobs: name: downstream-${{ matrix.consumer }}-results path: | bazel-testlogs/tools/downstream_tests/${{ matrix.consumer }}/test.xml - bazel-testlogs/tools/downstream_tests/${{ matrix.consumer }}/test.outputs/outputs.zip + bazel-testlogs/tools/downstream_tests/${{ matrix.consumer }}/test.outputs/coverage-report.zip if-no-files-found: ignore retention-days: 3 diff --git a/tools/downstream_tests/_consumer_workspace.py b/tools/downstream_tests/_consumer_workspace.py index 5df272d..4fd47fb 100644 --- a/tools/downstream_tests/_consumer_workspace.py +++ b/tools/downstream_tests/_consumer_workspace.py @@ -109,6 +109,10 @@ def publish_coverage_results(workspace: Path, archive_directory: str) -> None: shutil.copytree(html_report, retained_report / "coverage_linux") shutil.copy2(lcov_report, retained_report / lcov_report.name) + summary_markdown = workspace / "coverage_summary.md" + assert summary_markdown.is_file(), "Markdown coverage summary was not generated" + shutil.copy2(summary_markdown, retained_report / summary_markdown.name) + for name in ("justification_report", "unmapped_files.txt"): result = archive / name if result.is_dir(): @@ -116,8 +120,18 @@ def publish_coverage_results(workspace: Path, archive_directory: str) -> None: elif result.is_file(): shutil.copy2(result, retained_report / name) + report_archive = shutil.make_archive(str(retained_report), "zip", retained_report) + shutil.rmtree(retained_report) + + step_summary = os.environ.get("GITHUB_STEP_SUMMARY") + if step_summary: + with Path(step_summary).open("a", encoding="utf-8") as summary_file: + summary_file.write(summary_markdown.read_text(encoding="utf-8")) + summary_file.write("\n") + coverage_percent = 100 * lines_hit / lines_found print( f"Coverage report contains {source_files} source files and " - f"{lines_hit}/{lines_found} covered lines ({coverage_percent:.2f}%)." + f"{lines_hit}/{lines_found} covered lines ({coverage_percent:.2f}%). " + f"Report retained at {report_archive}." ) diff --git a/tools/downstream_tests/test_baselibs.py b/tools/downstream_tests/test_baselibs.py index e087fb0..f59d5f7 100644 --- a/tools/downstream_tests/test_baselibs.py +++ b/tools/downstream_tests/test_baselibs.py @@ -44,6 +44,8 @@ def test_baselibs_coverage_workflow_reports_measured_coverage(tmp_path: Path) -> "--", "--yaml", "tools/coverage/coverage_justifications.yaml", + "--summary-md", + "coverage_summary.md", "--testlogs-subdir", "score", "--archive-dir", diff --git a/tools/downstream_tests/test_lifecycle.py b/tools/downstream_tests/test_lifecycle.py index d29afc5..68ec6f9 100644 --- a/tools/downstream_tests/test_lifecycle.py +++ b/tools/downstream_tests/test_lifecycle.py @@ -36,6 +36,8 @@ def test_lifecycle_coverage_workflow_reports_measured_coverage(tmp_path: Path) - "--", "--yaml", "quality/coverage/coverage_justifications.yaml", + "--summary-md", + "coverage_summary.md", "--archive-dir", "coverage_artifacts", extra_environment={"COVERAGE_THRESHOLD": "66"}, From ba7829427e806720c1863c439224fbd400047562 Mon Sep 17 00:00:00 2001 From: Alexander Lanin Date: Sat, 3 Oct 2026 02:59:35 +0200 Subject: [PATCH 5/7] fix: retain downstream reports on failures --- tools/downstream_tests/_consumer_workspace.py | 84 +++++++++++++------ tools/downstream_tests/test_baselibs.py | 40 +++++---- tools/downstream_tests/test_lifecycle.py | 34 ++++---- 3 files changed, 100 insertions(+), 58 deletions(-) diff --git a/tools/downstream_tests/_consumer_workspace.py b/tools/downstream_tests/_consumer_workspace.py index 4fd47fb..256e2d0 100644 --- a/tools/downstream_tests/_consumer_workspace.py +++ b/tools/downstream_tests/_consumer_workspace.py @@ -17,6 +17,7 @@ import os import shutil import subprocess +import sys from pathlib import Path _COVERAGE_TOOL_ROOT = Path(__file__).resolve().parents[2] @@ -79,14 +80,68 @@ def run_bazel( ) -def publish_coverage_results(workspace: Path, archive_directory: str) -> None: - """Check for measured coverage and retain the report as a Bazel test output.""" +def retain_coverage_results(workspace: Path, archive_directory: str) -> None: + """Keep whatever report files exist, including reports from a failed gate.""" archive = workspace / archive_directory html_report = archive / "coverage_linux" lcov_report = archive / "coverage_report.dat" + outputs_directory = os.environ.get("TEST_UNDECLARED_OUTPUTS_DIR") + if not archive.is_dir() or not outputs_directory: + return + + retained_report = Path(outputs_directory) / "coverage-report" + summary_markdown = workspace / "coverage_summary.md" + try: + retained_report.mkdir(parents=True, exist_ok=True) + if html_report.is_dir(): + shutil.copytree(html_report, retained_report / "coverage_linux") + if lcov_report.is_file(): + shutil.copy2(lcov_report, retained_report / lcov_report.name) + if summary_markdown.is_file(): + shutil.copy2(summary_markdown, retained_report / summary_markdown.name) + + for name in ("justification_report", "unmapped_files.txt"): + result = archive / name + if result.is_dir(): + shutil.copytree(result, retained_report / name) + elif result.is_file(): + shutil.copy2(result, retained_report / name) + except Exception as error: + print(f"Could not copy all coverage report files: {error}", file=sys.stderr) + + step_summary = os.environ.get("GITHUB_STEP_SUMMARY") + if step_summary and summary_markdown.is_file(): + try: + with Path(step_summary).open("a", encoding="utf-8") as summary_file: + summary_file.write(summary_markdown.read_text(encoding="utf-8")) + summary_file.write("\n") + except OSError as error: + print(f"Could not append the coverage summary: {error}", file=sys.stderr) + + try: + report_archive = Path(shutil.make_archive(str(retained_report), "zip", retained_report)) + except Exception as error: + print(f"Could not archive the coverage report: {error}", file=sys.stderr) + return + + try: + shutil.rmtree(retained_report) + except OSError as error: + print(f"Could not remove the unpacked coverage report: {error}", file=sys.stderr) + + print(f"Coverage report retained at {report_archive}.") + + +def verify_coverage_results(workspace: Path, archive_directory: str) -> None: + """Require measurable coverage after retaining all available report files.""" + archive = workspace / archive_directory + html_report = archive / "coverage_linux" + lcov_report = archive / "coverage_report.dat" + summary_markdown = workspace / "coverage_summary.md" assert (html_report / "index.html").is_file(), "Coverage HTML index was not generated" assert lcov_report.is_file(), "LCOV coverage report was not generated" + assert summary_markdown.is_file(), "Markdown coverage summary was not generated" source_files = 0 lines_found = 0 @@ -105,29 +160,8 @@ def publish_coverage_results(workspace: Path, archive_directory: str) -> None: outputs_directory = os.environ.get("TEST_UNDECLARED_OUTPUTS_DIR") assert outputs_directory, "Bazel did not provide TEST_UNDECLARED_OUTPUTS_DIR" - retained_report = Path(outputs_directory) / "coverage-report" - shutil.copytree(html_report, retained_report / "coverage_linux") - shutil.copy2(lcov_report, retained_report / lcov_report.name) - - summary_markdown = workspace / "coverage_summary.md" - assert summary_markdown.is_file(), "Markdown coverage summary was not generated" - shutil.copy2(summary_markdown, retained_report / summary_markdown.name) - - for name in ("justification_report", "unmapped_files.txt"): - result = archive / name - if result.is_dir(): - shutil.copytree(result, retained_report / name) - elif result.is_file(): - shutil.copy2(result, retained_report / name) - - report_archive = shutil.make_archive(str(retained_report), "zip", retained_report) - shutil.rmtree(retained_report) - - step_summary = os.environ.get("GITHUB_STEP_SUMMARY") - if step_summary: - with Path(step_summary).open("a", encoding="utf-8") as summary_file: - summary_file.write(summary_markdown.read_text(encoding="utf-8")) - summary_file.write("\n") + report_archive = Path(outputs_directory) / "coverage-report.zip" + assert report_archive.is_file(), "Coverage report was not retained for Bazel" coverage_percent = 100 * lines_hit / lines_found print( diff --git a/tools/downstream_tests/test_baselibs.py b/tools/downstream_tests/test_baselibs.py index f59d5f7..e0e66c9 100644 --- a/tools/downstream_tests/test_baselibs.py +++ b/tools/downstream_tests/test_baselibs.py @@ -14,7 +14,7 @@ from pathlib import Path -from _consumer_workspace import clone_consumer, publish_coverage_results, run_bazel +from _consumer_workspace import clone_consumer, retain_coverage_results, run_bazel, verify_coverage_results def test_baselibs_coverage_workflow_reports_measured_coverage(tmp_path: Path) -> None: @@ -36,21 +36,25 @@ def test_baselibs_coverage_workflow_reports_measured_coverage(tmp_path: Path) -> "-//score/language/safecpp/safe_math/details:floating_point_environment_test", "-//score/os/linux/utils/test:network_interface_test", ) - run_bazel( - workspace, - "run", - "--lockfile_mode=error", - "@score_coverage//:generate_coverage_html", - "--", - "--yaml", - "tools/coverage/coverage_justifications.yaml", - "--summary-md", - "coverage_summary.md", - "--testlogs-subdir", - "score", - "--archive-dir", - "coverage_artifact", - extra_environment={"COVERAGE_THRESHOLD": "0"}, - ) + # The generator assembles artifacts before returning a gate failure. + try: + run_bazel( + workspace, + "run", + "--lockfile_mode=error", + "@score_coverage//:generate_coverage_html", + "--", + "--yaml", + "tools/coverage/coverage_justifications.yaml", + "--summary-md", + "coverage_summary.md", + "--testlogs-subdir", + "score", + "--archive-dir", + "coverage_artifact", + extra_environment={"COVERAGE_THRESHOLD": "0"}, + ) + finally: + retain_coverage_results(workspace, "coverage_artifact") - publish_coverage_results(workspace, "coverage_artifact") + verify_coverage_results(workspace, "coverage_artifact") diff --git a/tools/downstream_tests/test_lifecycle.py b/tools/downstream_tests/test_lifecycle.py index 68ec6f9..ccbb158 100644 --- a/tools/downstream_tests/test_lifecycle.py +++ b/tools/downstream_tests/test_lifecycle.py @@ -14,7 +14,7 @@ from pathlib import Path -from _consumer_workspace import clone_consumer, publish_coverage_results, run_bazel +from _consumer_workspace import clone_consumer, retain_coverage_results, run_bazel, verify_coverage_results def test_lifecycle_coverage_workflow_reports_measured_coverage(tmp_path: Path) -> None: @@ -29,18 +29,22 @@ def test_lifecycle_coverage_workflow_reports_measured_coverage(tmp_path: Path) - "--lockfile_mode=error", "--build_tests_only", ) - run_bazel( - workspace, - "run", - "@score_coverage//:generate_coverage_html", - "--", - "--yaml", - "quality/coverage/coverage_justifications.yaml", - "--summary-md", - "coverage_summary.md", - "--archive-dir", - "coverage_artifacts", - extra_environment={"COVERAGE_THRESHOLD": "66"}, - ) + # The generator assembles artifacts before returning a gate failure. + try: + run_bazel( + workspace, + "run", + "@score_coverage//:generate_coverage_html", + "--", + "--yaml", + "quality/coverage/coverage_justifications.yaml", + "--summary-md", + "coverage_summary.md", + "--archive-dir", + "coverage_artifacts", + extra_environment={"COVERAGE_THRESHOLD": "66"}, + ) + finally: + retain_coverage_results(workspace, "coverage_artifacts") - publish_coverage_results(workspace, "coverage_artifacts") + verify_coverage_results(workspace, "coverage_artifacts") From b16d3f61c8afec30cd971046075ec877193c7cc2 Mon Sep 17 00:00:00 2001 From: Alexander Lanin Date: Sat, 3 Oct 2026 03:03:35 +0200 Subject: [PATCH 6/7] fix: append downstream summaries in workflow step --- .github/workflows/downstream_tests.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/downstream_tests.yml b/.github/workflows/downstream_tests.yml index 21145ef..7e2d962 100644 --- a/.github/workflows/downstream_tests.yml +++ b/.github/workflows/downstream_tests.yml @@ -50,7 +50,16 @@ jobs: disk-cache-key: downstream-${{ matrix.consumer }} - name: Run the consumer's coverage workflow - run: bazel test --lockfile_mode=error --test_env=GITHUB_STEP_SUMMARY //tools/downstream_tests:${{ matrix.consumer }} + run: bazel test --lockfile_mode=error //tools/downstream_tests:${{ matrix.consumer }} + + - name: Publish coverage summary + if: always() + env: + REPORT_ARCHIVE: bazel-testlogs/tools/downstream_tests/${{ matrix.consumer }}/test.outputs/coverage-report.zip + run: | + if [ -f "$REPORT_ARCHIVE" ] && unzip -Z1 "$REPORT_ARCHIVE" | grep -x coverage_summary.md > /dev/null; then + unzip -p "$REPORT_ARCHIVE" coverage_summary.md >> "$GITHUB_STEP_SUMMARY" + fi - name: Upload pytest results if: always() From 13361094408a1aa2da8bf3232aba49da1f546e76 Mon Sep 17 00:00:00 2001 From: Alexander Lanin Date: Sat, 3 Oct 2026 03:03:48 +0200 Subject: [PATCH 7/7] fix: let workflow publish coverage summary --- tools/downstream_tests/_consumer_workspace.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/tools/downstream_tests/_consumer_workspace.py b/tools/downstream_tests/_consumer_workspace.py index 256e2d0..5752890 100644 --- a/tools/downstream_tests/_consumer_workspace.py +++ b/tools/downstream_tests/_consumer_workspace.py @@ -109,15 +109,6 @@ def retain_coverage_results(workspace: Path, archive_directory: str) -> None: except Exception as error: print(f"Could not copy all coverage report files: {error}", file=sys.stderr) - step_summary = os.environ.get("GITHUB_STEP_SUMMARY") - if step_summary and summary_markdown.is_file(): - try: - with Path(step_summary).open("a", encoding="utf-8") as summary_file: - summary_file.write(summary_markdown.read_text(encoding="utf-8")) - summary_file.write("\n") - except OSError as error: - print(f"Could not append the coverage summary: {error}", file=sys.stderr) - try: report_archive = Path(shutil.make_archive(str(retained_report), "zip", retained_report)) except Exception as error: