Add traceable pytest black-box coverage scenarios

.github/workflows/tests.yml

@@ -45,7 +45,7 @@ jobs:
       - name: Build everything
         run: bazel build --lockfile_mode=error //...
       - name: Run unit and Starlark analysis tests
-        run: bazel test --lockfile_mode=error //score_coverage/... //tools/...
+        run: bazel test --lockfile_mode=error --test_tag_filters=-integration //score_coverage/... //tools/...
       - name: Static analysis of the Python (ruff, pylint, ty; findings fail the build)
         run: bazel build --lockfile_mode=error --config=lint //score_coverage/... //tools/...
       - name: Measure structural coverage of the tool itself (coverage.py)
@@ -67,7 +67,7 @@ jobs:
           echo "collected $(find tests-report -name test.xml | wc -l) test.xml files"
       - uses: actions/upload-artifact@v4
         with:
-          name: tests-report
+          name: unit-tests-report
           path: tests-report
           if-no-files-found: error
           retention-days: 3
@@ -83,18 +83,54 @@ jobs:
       - uses: eclipse-score/cicd-actions/setup-bazel-cache@setup-bazel-cache/v0.1.0
         with:
           disk-cache-key: integration_tests
-      - name: Run the end-to-end pipeline test (C++ + Rust consumer workspace)
-        run: integration_tests/run_integration_test.sh
+      - name: Run traceable pytest black-box scenarios
+        run: bazel test --lockfile_mode=error //tools/integration_tests:blackbox_test
+      - name: Collect pytest JUnit results for the documentation build
+        if: always()
+        run: |
+          mkdir -p tests-report/tools/integration_tests/blackbox_test
+          if [ -f bazel-testlogs/tools/integration_tests/blackbox_test/test.xml ]; then
+            cp bazel-testlogs/tools/integration_tests/blackbox_test/test.xml \
+              tests-report/tools/integration_tests/blackbox_test/test.xml
+          fi
+      - name: Upload integration pytest results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: integration-tests-report
+          path: tests-report
+          if-no-files-found: ignore
+          retention-days: 3
       - name: Upload coverage report of the integration workspace
         if: always()
         uses: actions/upload-artifact@v4
         with:
           name: integration_coverage_report
-          path: integration_tests/coverage_artifact
+          path: bazel-testlogs/tools/integration_tests/blackbox_test/test.outputs/coverage_artifact
           if-no-files-found: ignore
           retention-days: 10
+  test_reports:
+    needs: [unit_tests, integration_tests]
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Download unit test results
+        uses: actions/download-artifact@v4
+        with:
+          name: unit-tests-report
+          path: tests-report
+      - name: Download integration test results
+        uses: actions/download-artifact@v4
+        with:
+          name: integration-tests-report
+          path: tests-report
+      - uses: actions/upload-artifact@v4
+        with:
+          name: tests-report
+          path: tests-report
+          if-no-files-found: error
+          retention-days: 3
   docs:
-    needs: unit_tests
+    needs: test_reports
     uses: eclipse-score/cicd-workflows/.github/workflows/docs.yml@d2083d5dac0e309643d9f495e61342dfbaf07ed5 # main, 2026-09-24
     permissions:
       contents: read

MODULE.bazel

@@ -59,6 +59,7 @@ use_repo(pip, "pip_score_coverage")
 # Development-only dependencies (repository hygiene: copyright, formatting)
 ###############################################################################
 bazel_dep(name = "score_tooling", version = "2.2.0", dev_dependency = True)
+bazel_dep(name = "score_tools", version = "0.0.3", dev_dependency = True)
 
 # use_format_targets() (from score_tooling) loads these from the ROOT module's
 # repo mapping, so the root has to declare them itself.

README.md

@@ -59,19 +59,20 @@ Exit codes: `0` gate passed, `1` gate failed, `2` no verdict possible.
 - `score_coverage/` — implementation (Python report tooling, Starlark rules) and
   unit tests, including Starlark analysis tests.
 - `integration_tests/` — a self-contained consumer workspace (C++ + Rust)
-  exercised end to end by `run_integration_test.sh` against a hand-derived
-  ground truth (`expected_lcov.dat`); also the reference for the adoption guide.
+  exercised by named pytest scenarios in `//tools/integration_tests:blackbox_test`
+  against hand-derived LLVM and gcov ground truths; also the reference for the
+  adoption guide.
 - `tools/` — repository hygiene (copyright, format, lint aspects) and the
   self-coverage gate.
 - `docs/` — the docs-as-code tree.
 
 ## Development
 
 ```bash
-bazel test //score_coverage/... //tools/...          # unit + analysis tests
+bazel test --test_tag_filters=-integration //score_coverage/... //tools/... # unit + analysis tests
+bazel test //tools/integration_tests:blackbox_test   # consumer black-box scenarios
 bazel build --config=lint //score_coverage/... //tools/...   # ruff, pylint, ty
 bazel coverage --combined_report=lcov //score_coverage/tests:all
 bazel run //tools:self_coverage_gate -- --min-lines 95 --min-branches 87
-integration_tests/run_integration_test.sh            # end-to-end (downloads LLVM + Ferrocene)
 bazel run //tools:format.fix && bazel run //tools:copyright.check
 ```

docs/BUILD

@@ -23,5 +23,8 @@ docs(
     source_dir = ".",
     # Only these test.xml files become testcase needs (JUnit results under
     # bazel-testlogs/ or tests-report/ at the workspace root).
-    test_sources = ["score_coverage/tests"],
+    test_sources = [
+        "score_coverage/tests",
+        "tools/integration_tests",
+    ],
 )

docs/verification/verification_report.rst

@@ -80,39 +80,36 @@ Test inventory
    * - ``//score_coverage/tests/starlark:coverage_scope_tests`` (14 analysis tests)
      - 14
      - scope_transitive, scope_excludes, scope_baseline_objects, scope_gcno
-   * - ``integration_tests/run_integration_test.sh`` (25 end-to-end checks)
-     - 25
-     - validation_ground_truth, instrumentation_hint, report_baseline_zero, report_relative_paths,
-       report_allowlist, report_unmapped, gcov_merge, gcov_baseline, gcov_html,
-       gate_exit_codes, gate_no_verdict, just_unknown_id, artifacts,
-       summary_first
+   * - ``//tools/integration_tests:blackbox_test``
+     - 31
+     - validation_ground_truth, instrumentation_hint, report_baseline_zero,
+       report_relative_paths, report_allowlist, report_unmapped, gcov_merge,
+       gcov_baseline, gcov_html, gate_metric, gate_exit_codes, gate_no_verdict,
+       just_markers, just_unknown_id, artifacts, summary_first
 
 Requirement coverage
 --------------------
 
-The links from test cases to requirements are generated: every unit test class
-carries ``@verifies(<tool_req ids>)``, which writes ``PartiallyVerifies``,
-``TestType`` and ``DerivationTechnique`` into the JUnit XML of the test run, and
-docs-as-code turns the results into ``testcase`` needs with back-links on the
-requirements (``testlink`` column below, with the execution result of each
-case). The links reflect the test run that preceded the documentation build.
+The links from test cases to requirements are generated: each pytest function
+uses the ``score_pytest`` metadata decorator to write ``PartiallyVerifies``,
+``TestType`` and ``DerivationTechnique`` into the JUnit XML. Docs-as-code turns
+the results into ``testcase`` needs with back-links on the requirements
+(``testlink`` column below, with the execution result of each case). The links
+reflect the test run that preceded the documentation build.
 
 .. needtable:: Requirements and the tests that verify them
    :types: tool_req
    :columns: id;title;testlink
    :style: table
 
-Four requirements are verified outside the pytest suites and therefore carry no
-generated link:
+Three requirements are verified outside the pytest suites and therefore carry
+no generated link:
 
 - :need:`tool_req__coverage_scope_transitive`,
   :need:`tool_req__coverage_scope_excludes` and
-  :need:`tool_req__coverage_scope_baseline_objects` are verified by the eleven
+  :need:`tool_req__coverage_scope_baseline_objects` are verified by the fourteen
   Starlark analysis tests in ``score_coverage/tests/starlark`` (rules_testing
   produces no test properties).
-- :need:`tool_req__coverage_validation_ground_truth` is verified by the
-  end-to-end run ``integration_tests/run_integration_test.sh`` (golden LCOV
-  comparison, see below).
 
 .. needpie:: Test results of the linked test cases
    :labels: passed, failed, skipped
@@ -169,32 +166,35 @@ yamlfmt the workflows; copyright headers are checked on every file.
 End-to-end validation
 ---------------------
 
-``integration_tests/run_integration_test.sh`` builds a consumer workspace with a
-tested and an untested C++ library, a header-only library reached through
-``strip_include_prefix``, a tested Rust library and an untested Rust binary, one
-justified line, and asserts:
-
-1. the gate fails at 100 % and passes at 10 % (effective and raw mode);
-2. the HTML, the summary and the archive tree are produced, the summary also
-   when the gate fails;
-3. the untested C++ file and the untested Rust binary appear with ``LH:0``;
-4. the LCOV matches ``expected_lcov.dat``, a hand-derived ground truth, record
-   by record;
-5. the justified line raises effective above raw coverage;
-6. fault injection: a corrupt report and a non-numeric threshold exit 2, and a
-   misspelt justification id is reported and does not raise the effective
-   coverage.
+``//tools/integration_tests:blackbox_test`` runs named pytest cases against a
+copied consumer workspace with a tested and an untested C++ library, a
+header-only library reached through ``strip_include_prefix``, a tested Rust
+library and an untested Rust binary. Separate LLVM and gcov fixtures collect
+their reports before cases exercise:
+
+1. effective and raw gate thresholds, with parametrized pass and fail values;
+2. Markdown summaries, archive contents, working HTML and stylesheet links,
+   canonical source paths, and exclusion of forwarded external code;
+3. unmapped-file categories, exact zero-count baselines, and hand-derived LLVM
+   and gcov LCOV ground truths;
+4. the gcov warning and zero-count fallback when a narrow instrumentation
+   filter omits a tested library;
+5. fault injection for corrupt reports, invalid thresholds and unknown
+   justification markers.
+
+The pytest JUnit XML records a test case and requirement metadata for every
+scenario, including each parametrized gate value.
 
 Deviations
 ----------
 
 - Structural coverage of the Python is below 100 %. The remaining lines are
   error-handling and llvm-cov fallback paths in ``reporter.py`` and
   ``effective_coverage.py``; they are covered by the fault-injection checks of
-  the integration test where they are reachable and will be closed or justified
+  the black-box test where they are reachable and will be closed or justified
   before the first qualified release.
 - Starlark (``coverage_scope.bzl``, ``reporter_wrapper.bzl``) has no structural
-  coverage tooling. The rule and aspect are verified by eight analysis tests
+  coverage tooling. The rule and aspect are verified by fourteen analysis tests
   and by the end-to-end run.
 - The gcovr backend of ``effective_coverage.py`` is unit-tested against real
   gcovr 8.6 markup but is not reachable through ``generate_coverage_html`` in

integration_tests/.bazelrc

@@ -107,3 +107,7 @@ coverage:gcov --test_lang_filters=cc
 # the tests too so header-only code that only a test translation unit
 # instantiates is measured (the scope allowlist still drops the test sources).
 coverage:gcov --instrument_test_targets
+
+# Used by the black-box scenario that verifies Bazel drops gcov counters for
+# the cross-package library when the consumer filter omits //lib/.
+coverage:gcov_narrow_filter --instrumentation_filter=^//lib/test[/:],^//src[/:]