From a185f661ccf6c4ae24a6af19fb07ff5525c560d7 Mon Sep 17 00:00:00 2001 From: Marco Heinemann Date: Tue, 29 Sep 2026 10:52:30 +0200 Subject: [PATCH 1/3] feat: convert JUnit results into needs.json that every reader imports sphinx-test-reports' test-report directive creates the testfile, testsuite and testcase needs only while Sphinx runs, so ubCode and ubc, which import needs.json, never see them. Convert the same JUnit XML into the same ids, types and fields, reusing sphinx-test-reports' own parser, and carry the results links as data: one needextend per test specification instead of the Sphinx-only sple_tr_link needs function. CMake wiring behind an off-by-default setting follows. --- src/spl_core/test_report/__init__.py | 0 src/spl_core/test_report/junit_to_needs.py | 159 ++++++++++++++++++ tests/data/junit/listing.rst | 13 ++ tests/data/junit/two_suites.xml | 15 ++ tests/unit/test_report_junit_to_needs.py | 184 +++++++++++++++++++++ 5 files changed, 371 insertions(+) create mode 100644 src/spl_core/test_report/__init__.py create mode 100644 src/spl_core/test_report/junit_to_needs.py create mode 100644 tests/data/junit/listing.rst create mode 100644 tests/data/junit/two_suites.xml create mode 100644 tests/unit/test_report_junit_to_needs.py diff --git a/src/spl_core/test_report/__init__.py b/src/spl_core/test_report/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/spl_core/test_report/junit_to_needs.py b/src/spl_core/test_report/junit_to_needs.py new file mode 100644 index 00000000..258a277e --- /dev/null +++ b/src/spl_core/test_report/junit_to_needs.py @@ -0,0 +1,159 @@ +"""Turn JUnit XML results into needs that every reader can import. + +sphinx-test-reports' ``test-report`` directive creates its testfile, +testsuite and testcase needs only while Sphinx runs, so the result data +disappears for any reader that just imports ``needs.json`` (ubCode, ``ubc``). +This module reproduces exactly those needs -- same ids, types and fields -- +from the same JUnit XML, using sphinx-test-reports' own parser, and writes them +next to the generated report page. The page itself then holds a plain +``needimport`` of that JSON plus one ``needextend`` per test specification that +links the cases matching it, so the ``results`` links become data instead of a +Sphinx-only needs function. + +The ids follow sphinx-test-reports' scheme:: + + + _ + _ +""" + +from __future__ import annotations + +import argparse +import glob as glob_module +import hashlib +import json +import re +from collections.abc import Iterable, Mapping +from pathlib import Path +from typing import Any + +from sphinxcontrib.test_reports.junitparser import JUnitParser + +SUITE_LEN, CASE_LEN = 3, 5 # sphinx-test-reports' tr_suite_id_length, tr_case_id_length +NEEDS_FILE_NAME = "unit_test_results.needs.json" +DEFAULT_PROJECT = "SPL" + +_SPEC_RE = re.compile(r"^\.\. test:: (.+)\n\s+:id: (\S+)", re.M) + + +def sha(text: str, n: int) -> str: + return hashlib.sha1(text.encode("UTF-8")).hexdigest().upper()[:n] + + +def block(label: str, value: str) -> str: + return "\n\n**{}**::\n\n {}\n\n".format(label, "\n ".join(x.lstrip() for x in value.split("\n"))) + + +def case_time(value: Any) -> str: + if isinstance(value, (int, float)): + return str(float(value) if value >= 0 else 0.0) + if value is None: + return str(0.0) + try: + return str(float(value)) + except (TypeError, ValueError): + return str(0.0) + + +def convert(title: str, file_id: str, junit: str) -> dict[str, dict[str, Any]]: + """The needs sphinx-test-reports' test-report directive creates for one JUnit file.""" + suites = JUnitParser(junit).parse() + needs: dict[str, dict[str, Any]] = {} + tags = [file_id] + needs[file_id] = dict( + id=file_id, type="testfile", title=title, file=junit, tags=tags, links=[], content="[]", + suites=len(suites), cases=sum(int(s["tests"]) for s in suites), passed=sum(s["passed"] for s in suites), + skipped=sum(s["skips"] for s in suites), failed=sum(s["failures"] for s in suites), + errors=sum(s["errors"] for s in suites)) + for suite in suites: + suite_id = f"{file_id}_{sha(suite['name'], SUITE_LEN)}" + needs[suite_id] = dict( + id=suite_id, type="testsuite", title=suite["name"], suite=suite["name"], file=junit, tags=tags, + links=[file_id], content="", cases=int(suite["tests"]), passed=suite["passed"], skipped=suite["skips"], + failed=suite["failures"], errors=suite["errors"]) + for case in suite["testcases"]: + case_id = f"{suite_id}_{sha(case['classname'] + case['name'], CASE_LEN)}" + groups = re.match(r"^(?P[^\[]+)($|\[(?P.*)?\])", case["name"]) + name, param = (groups["name"], groups["param"] or "") if groups else (case["name"], "") + content = "" + for key, label in (("text", "Text"), ("message", "Message"), ("system-out", "System-out")): + if case.get(key): + content += block(label, case[key]) + needs[case_id] = dict( + id=case_id, type="testcase", title=case["name"], case=case["name"], case_name=name, + case_parameter=param, classname=case["classname"], result=case["result"], time=case_time(case["time"]), + suite=suite["name"], style="tr_" + case["result"], file=junit, tags=tags, + links=[file_id, suite_id], content=content) + return needs + + +def _iter_listing_files(listing_paths: Iterable[str | Path]) -> Iterable[Path]: + for entry in listing_paths: + text = str(entry) + matches = sorted(glob_module.glob(text, recursive=True)) + if matches: + for match in matches: + yield Path(match) + else: + yield Path(entry) + + +def collect_specs(listing_paths: Iterable[str | Path]) -> list[tuple[str, str]]: + """The (spec_id, spec_title) pairs the ``.. test::`` directives declare in RST listings.""" + specs: list[tuple[str, str]] = [] + for path in _iter_listing_files(listing_paths): + text = path.read_text(encoding="utf-8") + specs.extend((m.group(2), m.group(1).strip()) for m in _SPEC_RE.finditer(text)) + return specs + + +def results_links(needs: Mapping[str, Mapping[str, Any]], specs: Iterable[tuple[str, str]]) -> dict[str, list[str]]: + """Map each spec id to the case ids whose ``case`` equals or matches its title.""" + links: dict[str, list[str]] = {} + for need in needs.values(): + if need["type"] != "testcase": + continue + for spec_id, spec_title in specs: + if spec_title == need["case"] or ("*" in spec_title and re.match(spec_title, need["case"])): + links.setdefault(spec_id, []).append(need["id"]) + return links + + +def write_results( + page_path: str | Path, + title: str, + file_id: str, + junit_path: str | Path, + listing_paths: Iterable[str | Path], + project: str = DEFAULT_PROJECT, +) -> None: + """Write the needs JSON next to the report page and rewrite the page to import it.""" + page_path = Path(page_path) + needs = convert(title, file_id, str(junit_path)) + needs_json = {"current_version": "", "project": project, "versions": {"": {"needs": needs}}} + + page_path.parent.mkdir(parents=True, exist_ok=True) + (page_path.parent / NEEDS_FILE_NAME).write_text(json.dumps(needs_json, indent=1), encoding="utf-8") + + links = results_links(needs, collect_specs(listing_paths)) + lines = ["", title, "=" * len(title), "", f".. needimport:: {NEEDS_FILE_NAME}", ""] + for spec_id, case_ids in sorted(links.items()): + lines += [f".. needextend:: {spec_id}", f" :+results: {', '.join(case_ids)}", ""] + page_path.write_text("\n".join(lines), encoding="utf-8") + + +def main(argv: list[str] | None = None) -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--page", required=True, help="path of the report page to (re)write") + parser.add_argument("--title", required=True, help="title for the testfile need") + parser.add_argument("--id", required=True, dest="file_id", help="id for the testfile need") + parser.add_argument("--junit", required=True, help="path of the JUnit XML file") + parser.add_argument("--listings", nargs="*", default=[], metavar="GLOB", help="RST listing files that declare test specs") + parser.add_argument("--project", default=DEFAULT_PROJECT, help="project name for the needs.json envelope") + args = parser.parse_args(argv) + write_results(args.page, args.title, args.file_id, args.junit, args.listings, project=args.project) + + +if __name__ == "__main__": + main() diff --git a/tests/data/junit/listing.rst b/tests/data/junit/listing.rst new file mode 100644 index 00000000..9aa0244e --- /dev/null +++ b/tests/data/junit/listing.rst @@ -0,0 +1,13 @@ +.. This file mirrors the generated source listings that declare test specs. + +Some generated listing. +*********************** + +.. test:: test_addition + :id: TS_ADD + +.. test:: test_division + :id: TS_DIV + +.. test:: test_param* + :id: TS_PARAM diff --git a/tests/data/junit/two_suites.xml b/tests/data/junit/two_suites.xml new file mode 100644 index 00000000..797188a6 --- /dev/null +++ b/tests/data/junit/two_suites.xml @@ -0,0 +1,15 @@ + + + + + + Traceback: division by zero + + + + + + + + + diff --git a/tests/unit/test_report_junit_to_needs.py b/tests/unit/test_report_junit_to_needs.py new file mode 100644 index 00000000..23ba1443 --- /dev/null +++ b/tests/unit/test_report_junit_to_needs.py @@ -0,0 +1,184 @@ +"""Unit tests for converting JUnit XML into importable test-result needs.""" + +import hashlib +import json +from pathlib import Path + +import pytest + +from spl_core.test_report.junit_to_needs import ( + NEEDS_FILE_NAME, + collect_specs, + convert, + results_links, + write_results, +) + +DATA_DIR = Path(__file__).parent.parent / "data" / "junit" +JUNIT_FILE = DATA_DIR / "two_suites.xml" +LISTING_FILE = DATA_DIR / "listing.rst" + +FILE_ID = "TEST_RESULT_Demo" + + +def _digest(text: str, length: int) -> str: + return hashlib.sha1(text.encode("UTF-8")).hexdigest().upper()[:length] + + +MATH_SUITE_ID = f"{FILE_ID}_{_digest('MathSuite', 3)}" +PARAM_SUITE_ID = f"{FILE_ID}_{_digest('ParamSuite', 3)}" +ADDITION_CASE_ID = f"{MATH_SUITE_ID}_{_digest('MathTests' + 'test_addition', 5)}" +DIVISION_CASE_ID = f"{MATH_SUITE_ID}_{_digest('MathTests' + 'test_division', 5)}" +SKIP_CASE_ID = f"{MATH_SUITE_ID}_{_digest('MathTests' + 'test_skip_me', 5)}" +PARAM_CASE_ID = f"{PARAM_SUITE_ID}_{_digest('ParamTests' + 'test_param[1]', 5)}" + + +@pytest.fixture +def needs(): + return convert("Unit Test Results", FILE_ID, str(JUNIT_FILE)) + + +@pytest.mark.unit +def test_ids_follow_the_test_report_scheme(needs): + assert set(needs) == { + FILE_ID, + MATH_SUITE_ID, + PARAM_SUITE_ID, + ADDITION_CASE_ID, + DIVISION_CASE_ID, + SKIP_CASE_ID, + PARAM_CASE_ID, + } + assert needs[MATH_SUITE_ID]["id"] == MATH_SUITE_ID + assert needs[ADDITION_CASE_ID]["id"] == ADDITION_CASE_ID + + +@pytest.mark.unit +def test_testfile_counts_and_fields(needs): + testfile = needs[FILE_ID] + assert testfile["type"] == "testfile" + assert testfile["title"] == "Unit Test Results" + assert testfile["file"] == str(JUNIT_FILE) + assert testfile["tags"] == [FILE_ID] + assert testfile["links"] == [] + assert testfile["suites"] == 2 + assert testfile["cases"] == 4 + assert testfile["passed"] == 2 + assert testfile["skipped"] == 1 + assert testfile["failed"] == 1 + assert testfile["errors"] == 0 + + +@pytest.mark.unit +def test_testsuite_counts_and_fields(needs): + math = needs[MATH_SUITE_ID] + assert math["type"] == "testsuite" + assert math["suite"] == "MathSuite" + assert math["links"] == [FILE_ID] + assert (math["cases"], math["passed"], math["skipped"], math["failed"], math["errors"]) == (3, 1, 1, 1, 0) + + param = needs[PARAM_SUITE_ID] + assert param["suite"] == "ParamSuite" + assert (param["cases"], param["passed"], param["skipped"], param["failed"], param["errors"]) == (1, 1, 0, 0, 0) + + +@pytest.mark.unit +def test_passed_case_fields(needs): + case = needs[ADDITION_CASE_ID] + assert case["type"] == "testcase" + assert case["case"] == "test_addition" + assert case["case_name"] == "test_addition" + assert case["case_parameter"] == "" + assert case["classname"] == "MathTests" + assert case["result"] == "passed" + assert case["style"] == "tr_passed" + assert case["suite"] == "MathSuite" + assert case["links"] == [FILE_ID, MATH_SUITE_ID] + assert case["content"] == "" + + +@pytest.mark.unit +def test_failed_case_fields(needs): + case = needs[DIVISION_CASE_ID] + assert case["result"] == "failure" + assert case["style"] == "tr_failure" + assert case["time"] == "0.2" + assert "**Message**::" in case["content"] + assert "division by zero" in case["content"] + assert "**Text**::" in case["content"] + assert "Traceback: division by zero" in case["content"] + + +@pytest.mark.unit +def test_skipped_case_fields(needs): + case = needs[SKIP_CASE_ID] + assert case["result"] == "skipped" + assert case["style"] == "tr_skipped" + assert "**Message**::" in case["content"] + assert "not implemented" in case["content"] + + +@pytest.mark.unit +def test_parameterised_case_is_split(needs): + case = needs[PARAM_CASE_ID] + assert case["case"] == "test_param[1]" + assert case["case_name"] == "test_param" + assert case["case_parameter"] == "1" + + +@pytest.mark.unit +def test_collect_specs_reads_id_and_title_pairs(): + specs = collect_specs([LISTING_FILE]) + assert specs == [ + ("TS_ADD", "test_addition"), + ("TS_DIV", "test_division"), + ("TS_PARAM", "test_param*"), + ] + + +@pytest.mark.unit +def test_results_links_exact_title_and_regex(needs): + links = results_links(needs, collect_specs([LISTING_FILE])) + assert links == { + "TS_ADD": [ADDITION_CASE_ID], + "TS_DIV": [DIVISION_CASE_ID], + "TS_PARAM": [PARAM_CASE_ID], + } + + +@pytest.mark.unit +def test_write_results_writes_needs_json_and_page(tmp_path): + page = tmp_path / "reports" / "unit_test_results.rst" + write_results(page, "Unit Test Results", FILE_ID, JUNIT_FILE, [LISTING_FILE]) + + needs_json = json.loads((page.parent / NEEDS_FILE_NAME).read_text(encoding="utf-8")) + assert needs_json["current_version"] == "" + assert needs_json["project"] == "SPL" + assert set(needs_json["versions"]) == {""} + assert set(needs_json["versions"][""]["needs"]) == { + FILE_ID, + MATH_SUITE_ID, + PARAM_SUITE_ID, + ADDITION_CASE_ID, + DIVISION_CASE_ID, + SKIP_CASE_ID, + PARAM_CASE_ID, + } + + page_text = page.read_text(encoding="utf-8") + assert "Unit Test Results\n=================" in page_text + assert f".. needimport:: {NEEDS_FILE_NAME}" in page_text + # needextend blocks are emitted in sorted spec-id order + assert page_text.index(".. needextend:: TS_ADD") < page_text.index(".. needextend:: TS_DIV") + assert page_text.index(".. needextend:: TS_DIV") < page_text.index(".. needextend:: TS_PARAM") + assert f" :+results: {ADDITION_CASE_ID}" in page_text + assert f" :+results: {DIVISION_CASE_ID}" in page_text + assert f" :+results: {PARAM_CASE_ID}" in page_text + + +@pytest.mark.unit +def test_write_results_project_name_is_configurable(tmp_path): + page = tmp_path / "unit_test_results.rst" + write_results(page, "Unit Test Results", FILE_ID, JUNIT_FILE, [], project="SPLed") + needs_json = json.loads((page.parent / NEEDS_FILE_NAME).read_text(encoding="utf-8")) + assert needs_json["project"] == "SPLed" From fe269464aff0b50b7a50af7cf73113d77478e5b4 Mon Sep 17 00:00:00 2001 From: Marco Heinemann Date: Tue, 29 Sep 2026 12:31:14 +0200 Subject: [PATCH 2/3] feat(docs): let each Sphinx run take options of its own SPL_SPHINX_OPTIONS adds options to the variant docs and reports builds and SPL_SPHINX_COMPONENT_OPTIONS to the per-component ones; @SHAPE@ and @COMPONENT_PATH@ in them are filled in per run, so every run can name a file of its own, such as the selection file that names its build directory. The binary directory check only compares a path that exists: when SPL_SPHINX_BINARY_DIR does not exist on disk, the project reaches the binary directory another way, for example through a sphinx-mounts mount, and there is no link to go stale. --- docs/reference/variables.md | 29 +++++++++++++++++--- src/spl_core/check_sphinx_binary_dir.cmake | 7 ++--- src/spl_core/common.cmake | 31 +++++++++++++++++----- tests/cmake/common.cmake/CMakeLists.txt | 30 ++++++++++++++++++--- 4 files changed, 80 insertions(+), 17 deletions(-) diff --git a/docs/reference/variables.md b/docs/reference/variables.md index a7fa1caf..2181f4a7 100644 --- a/docs/reference/variables.md +++ b/docs/reference/variables.md @@ -73,6 +73,23 @@ generates, see {ref}`SPL_SPHINX_BINARY_DIR `. set(SPL_SPHINX_SOURCE_DIR docs) ``` +(SPL_SPHINX_OPTIONS)= + +## SPL_SPHINX_OPTIONS and SPL_SPHINX_COMPONENT_OPTIONS + +Options added to every `sphinx-build` spl-core runs: `SPL_SPHINX_OPTIONS` to the +variant `docs` and `reports` builds, `SPL_SPHINX_COMPONENT_OPTIONS` to the +per-component docs and report builds. In both, `@SHAPE@` becomes `docs` or +`reports` and `@COMPONENT_PATH@` the component's path relative to the project +root (empty for the variant builds), so each run can name a file of its own. + +**Default:** none + +```cmake +set(SPL_SPHINX_OPTIONS -D spl_selection=${CMAKE_BINARY_DIR}/selection/@SHAPE@.toml) +set(SPL_SPHINX_COMPONENT_OPTIONS -D spl_selection=${CMAKE_BINARY_DIR}/selection/@COMPONENT_PATH@/@SHAPE@.toml) +``` + (SPL_SPHINX_BINARY_DIR)= ## SPL_SPHINX_BINARY_DIR @@ -93,10 +110,14 @@ gcovr HTML report is written to next to its coverage page, and the report artifacts `SplBuild` looks up. A relative path is taken relative to the project root. -Because the path usually is a link the project re-points, every docs and reports -build first checks that it still leads to its own binary directory, and fails -with a message naming both paths if another build directory was configured in -the meantime. +When the path is a link the project re-points, every docs and reports build first +checks that it still leads to its own binary directory, and fails with a message +naming both paths if another build directory was configured in the meantime. +When the path does not exist on disk, the project reaches the binary directory +another way, for example by mounting it at that path with +[sphinx-mounts](https://github.com/useblocks/sphinx-mounts) and naming each +build's directory in its runs' options ({ref}`SPL_SPHINX_OPTIONS `), +and there is nothing to check. **Default:** the binary directory itself (`CMAKE_BINARY_DIR`) diff --git a/src/spl_core/check_sphinx_binary_dir.cmake b/src/spl_core/check_sphinx_binary_dir.cmake index f8979265..04df7b72 100644 --- a/src/spl_core/check_sphinx_binary_dir.cmake +++ b/src/spl_core/check_sphinx_binary_dir.cmake @@ -10,10 +10,11 @@ # # cmake -DSPL_SPHINX_BINARY_DIR= -DSPL_BINARY_DIR= -P check_sphinx_binary_dir.cmake +# A path that does not exist on disk is not a link: the project reaches the binary +# directory another way, for example through a sphinx-mounts mount that each build +# names in the options of its own runs. There is nothing to compare. if(NOT EXISTS "${SPL_SPHINX_BINARY_DIR}") - message(FATAL_ERROR - "SPL_SPHINX_BINARY_DIR ${SPL_SPHINX_BINARY_DIR} does not exist. " - "It has to lead to ${SPL_BINARY_DIR} before this build's documentation can be built.") + return() endif() file(REAL_PATH "${SPL_SPHINX_BINARY_DIR}" _resolved) diff --git a/src/spl_core/common.cmake b/src/spl_core/common.cmake index 7af24b9f..a8dfc7e2 100644 --- a/src/spl_core/common.cmake +++ b/src/spl_core/common.cmake @@ -331,7 +331,7 @@ macro(spl_create_component) # We do not know all dependencies for generating the docs (apart from the rst files). # This might cause incremental builds to not update parts of the documentation. # To avoid this the command passes -E to make sphinx-build write all files new. - _spl_sphinx_build_command(_spl_sphinx_build SHAPE docs CONFIG ${_docs_config_json} OUTPUT_DIR ${_component_docs_html_out_dir}) + _spl_sphinx_build_command(_spl_sphinx_build SHAPE docs CONFIG ${_docs_config_json} OUTPUT_DIR ${_component_docs_html_out_dir} COMPONENT ${component_path}) _spl_sphinx_binary_dir_check(_spl_sphinx_check) add_custom_target( ${component_name}_docs @@ -407,7 +407,7 @@ Code Coverage # No OUTPUT is defined to force execution of this target every time # TODO: list of dependencies is not complete - _spl_sphinx_build_command(_spl_sphinx_build SHAPE reports CONFIG ${_reports_config_json} OUTPUT_DIR ${_component_reports_html_out_dir}) + _spl_sphinx_build_command(_spl_sphinx_build SHAPE reports CONFIG ${_reports_config_json} OUTPUT_DIR ${_component_reports_html_out_dir} COMPONENT ${component_path}) _spl_sphinx_binary_dir_check(_spl_sphinx_check) add_custom_target( ${component_name}_report @@ -540,11 +540,14 @@ function(_spl_sphinx_relative_path out_var path) endfunction() # The check a docs or reports build runs before sphinx-build when SPL_SPHINX_BINARY_DIR -# is set, including its COMMAND keyword, or nothing otherwise. The path is usually -# a link the project re-points whenever it configures a build directory, so by the +# is set, including its COMMAND keyword, or nothing otherwise. The path may be a +# link the project re-points whenever it configures a build directory, so by the # time this build runs it may lead to another build's output. Sphinx would then # read that build's generated pages without a word; the check stops the build -# instead. +# instead. When the path does not exist on disk, the project reaches the binary +# directory another way, for example by mounting it there with sphinx-mounts and +# naming each build's directory in the options of its own runs, and the check has +# nothing to compare. function(_spl_sphinx_binary_dir_check out_var) _spl_sphinx_binary_dir(_binary_dir) if(_binary_dir STREQUAL CMAKE_BINARY_DIR) @@ -568,8 +571,14 @@ endfunction() # the file each shape reads, and the build gets it as `-D needs_variant_data_file=`. # sphinx-needs keeps a command-line override even when the project's # needs_from_toml names another file, so conf.py needs no code to select it. +# +# COMPONENT is the component's path for the per-component builds and empty for the +# variant builds. A project adds options of its own with SPL_SPHINX_OPTIONS (variant +# builds) and SPL_SPHINX_COMPONENT_OPTIONS (per-component builds); `@SHAPE@` in +# them becomes the shape and `@COMPONENT_PATH@` the component's path, so each run +# can name a file of its own, e.g. `-D;spl_selection=/@COMPONENT_PATH@/@SHAPE@.toml`. function(_spl_sphinx_build_command out_var) - cmake_parse_arguments(ARG "" "SHAPE;CONFIG;OUTPUT_DIR" "" ${ARGN}) + cmake_parse_arguments(ARG "" "SHAPE;CONFIG;OUTPUT_DIR;COMPONENT" "" ${ARGN}) if(ARG_SHAPE STREQUAL "docs") set(_variant_data_file "${SPL_VARIANT_DATA_FILE_DOCS}") elseif(ARG_SHAPE STREQUAL "reports") @@ -582,6 +591,16 @@ function(_spl_sphinx_build_command out_var) if(_variant_data_file) list(APPEND _options -D needs_variant_data_file=${_variant_data_file}) endif() + if(ARG_COMPONENT) + set(_extra_options ${SPL_SPHINX_COMPONENT_OPTIONS}) + else() + set(_extra_options ${SPL_SPHINX_OPTIONS}) + endif() + foreach(_option IN LISTS _extra_options) + string(REPLACE "@SHAPE@" "${ARG_SHAPE}" _option "${_option}") + string(REPLACE "@COMPONENT_PATH@" "${ARG_COMPONENT}" _option "${_option}") + list(APPEND _options "${_option}") + endforeach() _spl_sphinx_source_dir(_source_dir) set(${out_var} diff --git a/tests/cmake/common.cmake/CMakeLists.txt b/tests/cmake/common.cmake/CMakeLists.txt index 0d52bcc1..7ef4ff35 100644 --- a/tests/cmake/common.cmake/CMakeLists.txt +++ b/tests/cmake/common.cmake/CMakeLists.txt @@ -112,6 +112,28 @@ unset(AUTOCONF_JSON) unset(SPL_VARIANT_DATA_FILE_DOCS) unset(SPL_VARIANT_DATA_FILE_REPORTS) +# ## test: _spl_sphinx_build_command (extra options per run) ######### +# given options for the variant runs and for the per-component runs +set(SPL_SPHINX_OPTIONS -D spl_selection=/sel/@SHAPE@.toml) +set(SPL_SPHINX_COMPONENT_OPTIONS -D spl_selection=/sel/@COMPONENT_PATH@/@SHAPE@.toml) + +# when +_spl_sphinx_build_command(sphinx_variant_command SHAPE reports CONFIG /build/reports/config.json OUTPUT_DIR /build/reports/html) +_spl_sphinx_build_command(sphinx_component_command SHAPE docs CONFIG /build/c/docs/config.json OUTPUT_DIR /build/c/docs/html COMPONENT components/c) + +# then: a variant run gets the variant options, a component run the component options, +# each with its own shape and component path filled in +if(NOT sphinx_variant_command MATCHES ";-D;spl_selection=/sel/reports\\.toml;[^;]+;/build/reports/html$") + message(FATAL_ERROR "Failing Test case: the variant reports run must name its own file, got '${sphinx_variant_command}'.") +endif() +if(NOT sphinx_component_command MATCHES ";-D;spl_selection=/sel/components/c/docs\\.toml;[^;]+;/build/c/docs/html$" OR sphinx_component_command MATCHES "/sel/docs\\.toml") + message(FATAL_ERROR "Failing Test case: the component docs run must name the component's file only, got '${sphinx_component_command}'.") +endif() + +# cleanup +unset(SPL_SPHINX_OPTIONS) +unset(SPL_SPHINX_COMPONENT_OPTIONS) + # ## test: _spl_sphinx_source_dir and _spl_sphinx_relative_path ######### # given the default configuration @@ -218,15 +240,15 @@ if(check_other_dir_result EQUAL 0) message(FATAL_ERROR "Failing Test case: the check must fail when the path leads to another directory.") endif() -# when it does not exist +# when it does not exist on disk, as when the project mounts the binary directory there execute_process( COMMAND ${CMAKE_COMMAND} -DSPL_SPHINX_BINARY_DIR=${PROJECT_BINARY_DIR}/does_not_exist -DSPL_BINARY_DIR=${PROJECT_BINARY_DIR} -P ${check_sphinx_binary_dir_script} RESULT_VARIABLE check_missing_dir_result OUTPUT_QUIET ERROR_QUIET ) -# then -if(check_missing_dir_result EQUAL 0) - message(FATAL_ERROR "Failing Test case: the check must fail when the path does not exist.") +# then: there is no link to compare, so the build goes ahead +if(NOT check_missing_dir_result EQUAL 0) + message(FATAL_ERROR "Failing Test case: the check must pass when the path does not exist.") endif() # ## test: _spl_get_absolute_path ######### From 1181c0a636ac4e85ffab02d52ede21dc8cc7da45 Mon Sep 17 00:00:00 2001 From: Marco Heinemann Date: Tue, 29 Sep 2026 12:31:14 +0200 Subject: [PATCH 3/3] feat: write the test results as needs after each test run With SPL_TEST_RESULTS_AS_NEEDS, a component's unit_test_results page is no longer written at configure time with sphinx-test-reports' test-report directive. After each test run, spl_core.test_report.junit_to_needs converts the JUnit XML into unit_test_results.needs.json and writes the page that imports it, with the results links of the test specifications as needextend blocks. The component and variant report builds wait for it. Off by default. --- docs/reference/variables.md | 21 +++++++++++++++++++ src/spl_core/common.cmake | 42 ++++++++++++++++++++++++++++++++++--- 2 files changed, 60 insertions(+), 3 deletions(-) diff --git a/docs/reference/variables.md b/docs/reference/variables.md index 2181f4a7..b59c15a8 100644 --- a/docs/reference/variables.md +++ b/docs/reference/variables.md @@ -90,6 +90,27 @@ set(SPL_SPHINX_OPTIONS -D spl_selection=${CMAKE_BINARY_DIR}/selection/@SHAPE@.to set(SPL_SPHINX_COMPONENT_OPTIONS -D spl_selection=${CMAKE_BINARY_DIR}/selection/@COMPONENT_PATH@/@SHAPE@.toml) ``` +(SPL_TEST_RESULTS_AS_NEEDS)= + +## SPL_TEST_RESULTS_AS_NEEDS + +When `ON`, a component's unit test results page is written after each test run +from its JUnit XML, instead of at configure time with sphinx-test-reports' +`test-report` directive. `spl_core.test_report.junit_to_needs` converts the JUnit +XML into `unit_test_results.needs.json`, with the same needs, IDs and fields the +directive creates, and writes a page that imports it with `needimport`. The page +also carries the `results` links of the component's test specifications as +`needextend` blocks: a specification links every test case whose name equals its +title, or matches it as a regular expression when the title contains `*`. Every +reader of needs.json sees the results, not only Sphinx, and the project can drop +the `sphinxcontrib.test_reports` extension and the `sple_tr_link` needs function. + +**Default:** `OFF` + +```cmake +set(SPL_TEST_RESULTS_AS_NEEDS ON) +``` + (SPL_SPHINX_BINARY_DIR)= ## SPL_SPHINX_BINARY_DIR diff --git a/src/spl_core/common.cmake b/src/spl_core/common.cmake index a8dfc7e2..33bacdff 100644 --- a/src/spl_core/common.cmake +++ b/src/spl_core/common.cmake @@ -363,9 +363,11 @@ Unit Test Specification ") - # create the test results rst file + # create the test results rst file. With SPL_TEST_RESULTS_AS_NEEDS the page is + # written after the test run instead, together with its needs.json (see below). set(_unit_test_results_rst ${_component_reports_out_dir}/unit_test_results.rst) - file(WRITE ${_unit_test_results_rst} " + if(NOT SPL_TEST_RESULTS_AS_NEEDS) + file(WRITE ${_unit_test_results_rst} " Unit Test Results ================= @@ -374,6 +376,7 @@ Unit Test Results :file: ${_component_test_junit_xml} ") + endif() # create coverage rst file to be able to automatically link to the coverage/index.html set(_coverage_rst ${_component_reports_out_dir}/coverage.rst) @@ -391,7 +394,10 @@ Code Coverage }") # add the generated files as dependency to cmake configure step - set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${_reports_config_json} ${_unit_test_spec_rst} ${_unit_test_results_rst}) + set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${_reports_config_json} ${_unit_test_spec_rst}) + if(NOT SPL_TEST_RESULTS_AS_NEEDS) + set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${_unit_test_results_rst}) + endif() set(_cov_out_html reports/html/${_rel_component_reports_out_dir}/coverage/index.html) file(RELATIVE_PATH _cov_out_json ${CMAKE_CURRENT_BINARY_DIR} ${_component_coverage_json}) @@ -426,6 +432,33 @@ Code Coverage _spl_generate_clanguru_source_docs(${component_name} "${_clanguru_all_sources}") endif() + # Test results as needs: after each test run, convert the JUnit XML into the + # needs sphinx-test-reports' test-report directive would create, and write the + # results page that imports them. Every reader of needs.json sees them, not + # only Sphinx. The page also carries the `results` links of the test + # specifications, taken from the source listings, as needextend blocks. + if(SPL_TEST_RESULTS_AS_NEEDS AND TEST_SOURCES) + set(_unit_test_results_needs_json ${_component_reports_out_dir}/unit_test_results.needs.json) + add_custom_command( + OUTPUT ${_unit_test_results_rst} ${_unit_test_results_needs_json} + COMMAND ${CMAKE_COMMAND} -E make_directory ${_component_reports_out_dir} + COMMAND ${SPL_PYTHON} -m spl_core.test_report.junit_to_needs + --page ${_unit_test_results_rst} + --title "Unit Test Results" + --id TEST_RESULT_${component_name} + --junit ${_component_test_junit_xml} + --project ${PROJECT_NAME} + --listings "${_clanguru_docs_out_dir}/**/*.rst" + DEPENDS ${_component_test_junit_xml} ${_clanguru_doc_outputs} + COMMENT "Converting the test results of ${component_name} into needs ..." + VERBATIM + ) + add_custom_target(${component_name}_test_results DEPENDS ${_unit_test_results_rst} ${_unit_test_results_needs_json}) + add_dependencies(${component_name}_report ${component_name}_test_results) + list(APPEND SPL_TEST_RESULTS_TARGETS ${component_name}_test_results) + set(SPL_TEST_RESULTS_TARGETS ${SPL_TEST_RESULTS_TARGETS} PARENT_SCOPE) + endif() + # Store the source docs directory so the variant report wrapper page can # nest it under the component (see _spl_create_reports_target). if(_COMPONENT_SOURCE_DOCS_INCLUDE_PATTERN) @@ -785,6 +818,9 @@ Code Coverage BYPRODUCTS ${_reports_html_output_dir}/index.html DEPENDS ${JUNIT_OUT_VARIANT_XML} ${COV_OUT_VARIANT_JSON} _components_variant_coverage_html_target source_docs ) + if(SPL_TEST_RESULTS_TARGETS) + add_dependencies(reports ${SPL_TEST_RESULTS_TARGETS}) + endif() endmacro() macro(_spl_set_coverage_create_overall_report_is_necessary)