feat: test results as needs.json for every reader

docs/reference/variables.md

@@ -73,6 +73,44 @@ generates, see {ref}`SPL_SPHINX_BINARY_DIR <SPL_SPHINX_BINARY_DIR>`.
 set(SPL_SPHINX_SOURCE_DIR docs)
 ```
 
+(SPL_SPHINX_OPTIONS)=
+
+## SPL_SPHINX_OPTIONS and SPL_SPHINX_COMPONENT_OPTIONS
+
+Options added to every `sphinx-build` spl-core runs: `SPL_SPHINX_OPTIONS` to the
+variant `docs` and `reports` builds, `SPL_SPHINX_COMPONENT_OPTIONS` to the
+per-component docs and report builds. In both, `@SHAPE@` becomes `docs` or
+`reports` and `@COMPONENT_PATH@` the component's path relative to the project
+root (empty for the variant builds), so each run can name a file of its own.
+
+**Default:** none
+
+```cmake
+set(SPL_SPHINX_OPTIONS -D spl_selection=${CMAKE_BINARY_DIR}/selection/@SHAPE@.toml)
+set(SPL_SPHINX_COMPONENT_OPTIONS -D spl_selection=${CMAKE_BINARY_DIR}/selection/@COMPONENT_PATH@/@SHAPE@.toml)
+```
+
+(SPL_TEST_RESULTS_AS_NEEDS)=
+
+## SPL_TEST_RESULTS_AS_NEEDS
+
+When `ON`, a component's unit test results page is written after each test run
+from its JUnit XML, instead of at configure time with sphinx-test-reports'
+`test-report` directive. `spl_core.test_report.junit_to_needs` converts the JUnit
+XML into `unit_test_results.needs.json`, with the same needs, IDs and fields the
+directive creates, and writes a page that imports it with `needimport`. The page
+also carries the `results` links of the component's test specifications as
+`needextend` blocks: a specification links every test case whose name equals its
+title, or matches it as a regular expression when the title contains `*`. Every
+reader of needs.json sees the results, not only Sphinx, and the project can drop
+the `sphinxcontrib.test_reports` extension and the `sple_tr_link` needs function.
+
+**Default:** `OFF`
+
+```cmake
+set(SPL_TEST_RESULTS_AS_NEEDS ON)
+```
+
 (SPL_SPHINX_BINARY_DIR)=
 
 ## SPL_SPHINX_BINARY_DIR
@@ -93,10 +131,14 @@ gcovr HTML report is written to next to its coverage page, and the report
 artifacts `SplBuild` looks up. A relative path is taken relative to the project
 root.
 
-Because the path usually is a link the project re-points, every docs and reports
-build first checks that it still leads to its own binary directory, and fails
-with a message naming both paths if another build directory was configured in
-the meantime.
+When the path is a link the project re-points, every docs and reports build first
+checks that it still leads to its own binary directory, and fails with a message
+naming both paths if another build directory was configured in the meantime.
+When the path does not exist on disk, the project reaches the binary directory
+another way, for example by mounting it at that path with
+[sphinx-mounts](https://github.com/useblocks/sphinx-mounts) and naming each
+build's directory in its runs' options ({ref}`SPL_SPHINX_OPTIONS <SPL_SPHINX_OPTIONS>`),
+and there is nothing to check.
 
 **Default:** the binary directory itself (`CMAKE_BINARY_DIR`)
 

src/spl_core/check_sphinx_binary_dir.cmake

@@ -10,10 +10,11 @@
 #
 #     cmake -DSPL_SPHINX_BINARY_DIR=<path> -DSPL_BINARY_DIR=<binary dir> -P check_sphinx_binary_dir.cmake
 
+# A path that does not exist on disk is not a link: the project reaches the binary
+# directory another way, for example through a sphinx-mounts mount that each build
+# names in the options of its own runs. There is nothing to compare.
 if(NOT EXISTS "${SPL_SPHINX_BINARY_DIR}")
-    message(FATAL_ERROR
-        "SPL_SPHINX_BINARY_DIR ${SPL_SPHINX_BINARY_DIR} does not exist. "
-        "It has to lead to ${SPL_BINARY_DIR} before this build's documentation can be built.")
+    return()
 endif()
 
 file(REAL_PATH "${SPL_SPHINX_BINARY_DIR}" _resolved)

src/spl_core/common.cmake

@@ -331,7 +331,7 @@ macro(spl_create_component)
             # We do not know all dependencies for generating the docs (apart from the rst files).
             # This might cause incremental builds to not update parts of the documentation.
             # To avoid this the command passes -E to make sphinx-build write all files new.
-            _spl_sphinx_build_command(_spl_sphinx_build SHAPE docs CONFIG ${_docs_config_json} OUTPUT_DIR ${_component_docs_html_out_dir})
+            _spl_sphinx_build_command(_spl_sphinx_build SHAPE docs CONFIG ${_docs_config_json} OUTPUT_DIR ${_component_docs_html_out_dir} COMPONENT ${component_path})
             _spl_sphinx_binary_dir_check(_spl_sphinx_check)
             add_custom_target(
                 ${component_name}_docs
@@ -363,9 +363,11 @@ Unit Test Specification
 
 ")
 
-                # create the test results rst file
+                # create the test results rst file. With SPL_TEST_RESULTS_AS_NEEDS the page is
+                # written after the test run instead, together with its needs.json (see below).
                 set(_unit_test_results_rst ${_component_reports_out_dir}/unit_test_results.rst)
-                file(WRITE ${_unit_test_results_rst} "
+                if(NOT SPL_TEST_RESULTS_AS_NEEDS)
+                    file(WRITE ${_unit_test_results_rst} "
 Unit Test Results
 =================
 
@@ -374,6 +376,7 @@ Unit Test Results
     :file: ${_component_test_junit_xml}
 
 ")
+                endif()
 
                 # create coverage rst file to be able to automatically link to the coverage/index.html
                 set(_coverage_rst ${_component_reports_out_dir}/coverage.rst)
@@ -391,7 +394,10 @@ Code Coverage
 }")
 
                 # add the generated files as dependency to cmake configure step
-                set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${_reports_config_json} ${_unit_test_spec_rst} ${_unit_test_results_rst})
+                set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${_reports_config_json} ${_unit_test_spec_rst})
+                if(NOT SPL_TEST_RESULTS_AS_NEEDS)
+                    set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${_unit_test_results_rst})
+                endif()
 
                 set(_cov_out_html reports/html/${_rel_component_reports_out_dir}/coverage/index.html)
                 file(RELATIVE_PATH _cov_out_json ${CMAKE_CURRENT_BINARY_DIR} ${_component_coverage_json})
@@ -407,7 +413,7 @@ Code Coverage
 
                 # No OUTPUT is defined to force execution of this target every time
                 # TODO: list of dependencies is not complete
-                _spl_sphinx_build_command(_spl_sphinx_build SHAPE reports CONFIG ${_reports_config_json} OUTPUT_DIR ${_component_reports_html_out_dir})
+                _spl_sphinx_build_command(_spl_sphinx_build SHAPE reports CONFIG ${_reports_config_json} OUTPUT_DIR ${_component_reports_html_out_dir} COMPONENT ${component_path})
                 _spl_sphinx_binary_dir_check(_spl_sphinx_check)
                 add_custom_target(
                     ${component_name}_report
@@ -426,6 +432,33 @@ Code Coverage
                 _spl_generate_clanguru_source_docs(${component_name} "${_clanguru_all_sources}")
             endif()
 
+            # Test results as needs: after each test run, convert the JUnit XML into the
+            # needs sphinx-test-reports' test-report directive would create, and write the
+            # results page that imports them. Every reader of needs.json sees them, not
+            # only Sphinx. The page also carries the `results` links of the test
+            # specifications, taken from the source listings, as needextend blocks.
+            if(SPL_TEST_RESULTS_AS_NEEDS AND TEST_SOURCES)
+                set(_unit_test_results_needs_json ${_component_reports_out_dir}/unit_test_results.needs.json)
+                add_custom_command(
+                    OUTPUT ${_unit_test_results_rst} ${_unit_test_results_needs_json}
+                    COMMAND ${CMAKE_COMMAND} -E make_directory ${_component_reports_out_dir}
+                    COMMAND ${SPL_PYTHON} -m spl_core.test_report.junit_to_needs
+                        --page ${_unit_test_results_rst}
+                        --title "Unit Test Results"
+                        --id TEST_RESULT_${component_name}
+                        --junit ${_component_test_junit_xml}
+                        --project ${PROJECT_NAME}
+                        --listings "${_clanguru_docs_out_dir}/**/*.rst"
+                    DEPENDS ${_component_test_junit_xml} ${_clanguru_doc_outputs}
+                    COMMENT "Converting the test results of ${component_name} into needs ..."
+                    VERBATIM
+                )
+                add_custom_target(${component_name}_test_results DEPENDS ${_unit_test_results_rst} ${_unit_test_results_needs_json})
+                add_dependencies(${component_name}_report ${component_name}_test_results)
+                list(APPEND SPL_TEST_RESULTS_TARGETS ${component_name}_test_results)
+                set(SPL_TEST_RESULTS_TARGETS ${SPL_TEST_RESULTS_TARGETS} PARENT_SCOPE)
+            endif()
+
             # Store the source docs directory so the variant report wrapper page can
             # nest it under the component (see _spl_create_reports_target).
             if(_COMPONENT_SOURCE_DOCS_INCLUDE_PATTERN)
@@ -540,11 +573,14 @@ function(_spl_sphinx_relative_path out_var path)
 endfunction()
 
 # The check a docs or reports build runs before sphinx-build when SPL_SPHINX_BINARY_DIR
-# is set, including its COMMAND keyword, or nothing otherwise. The path is usually
-# a link the project re-points whenever it configures a build directory, so by the
+# is set, including its COMMAND keyword, or nothing otherwise. The path may be a
+# link the project re-points whenever it configures a build directory, so by the
 # time this build runs it may lead to another build's output. Sphinx would then
 # read that build's generated pages without a word; the check stops the build
-# instead.
+# instead. When the path does not exist on disk, the project reaches the binary
+# directory another way, for example by mounting it there with sphinx-mounts and
+# naming each build's directory in the options of its own runs, and the check has
+# nothing to compare.
 function(_spl_sphinx_binary_dir_check out_var)
     _spl_sphinx_binary_dir(_binary_dir)
     if(_binary_dir STREQUAL CMAKE_BINARY_DIR)
@@ -568,8 +604,14 @@ endfunction()
 # the file each shape reads, and the build gets it as `-D needs_variant_data_file=`.
 # sphinx-needs keeps a command-line override even when the project's
 # needs_from_toml names another file, so conf.py needs no code to select it.
+#
+# COMPONENT is the component's path for the per-component builds and empty for the
+# variant builds. A project adds options of its own with SPL_SPHINX_OPTIONS (variant
+# builds) and SPL_SPHINX_COMPONENT_OPTIONS (per-component builds); `@SHAPE@` in
+# them becomes the shape and `@COMPONENT_PATH@` the component's path, so each run
+# can name a file of its own, e.g. `-D;spl_selection=<dir>/@COMPONENT_PATH@/@SHAPE@.toml`.
 function(_spl_sphinx_build_command out_var)
-    cmake_parse_arguments(ARG "" "SHAPE;CONFIG;OUTPUT_DIR" "" ${ARGN})
+    cmake_parse_arguments(ARG "" "SHAPE;CONFIG;OUTPUT_DIR;COMPONENT" "" ${ARGN})
     if(ARG_SHAPE STREQUAL "docs")
         set(_variant_data_file "${SPL_VARIANT_DATA_FILE_DOCS}")
     elseif(ARG_SHAPE STREQUAL "reports")
@@ -582,6 +624,16 @@ function(_spl_sphinx_build_command out_var)
     if(_variant_data_file)
         list(APPEND _options -D needs_variant_data_file=${_variant_data_file})
     endif()
+    if(ARG_COMPONENT)
+        set(_extra_options ${SPL_SPHINX_COMPONENT_OPTIONS})
+    else()
+        set(_extra_options ${SPL_SPHINX_OPTIONS})
+    endif()
+    foreach(_option IN LISTS _extra_options)
+        string(REPLACE "@SHAPE@" "${ARG_SHAPE}" _option "${_option}")
+        string(REPLACE "@COMPONENT_PATH@" "${ARG_COMPONENT}" _option "${_option}")
+        list(APPEND _options "${_option}")
+    endforeach()
 
     _spl_sphinx_source_dir(_source_dir)
     set(${out_var}
@@ -766,6 +818,9 @@ Code Coverage
         BYPRODUCTS ${_reports_html_output_dir}/index.html
         DEPENDS ${JUNIT_OUT_VARIANT_XML} ${COV_OUT_VARIANT_JSON} _components_variant_coverage_html_target source_docs
     )
+    if(SPL_TEST_RESULTS_TARGETS)
+        add_dependencies(reports ${SPL_TEST_RESULTS_TARGETS})
+    endif()
 endmacro()
 
 macro(_spl_set_coverage_create_overall_report_is_necessary)

src/spl_core/test_report/junit_to_needs.py

@@ -0,0 +1,159 @@
+"""Turn JUnit XML results into needs that every reader can import.
+
+sphinx-test-reports' ``test-report`` directive creates its testfile,
+testsuite and testcase needs only while Sphinx runs, so the result data
+disappears for any reader that just imports ``needs.json`` (ubCode, ``ubc``).
+This module reproduces exactly those needs -- same ids, types and fields --
+from the same JUnit XML, using sphinx-test-reports' own parser, and writes them
+next to the generated report page. The page itself then holds a plain
+``needimport`` of that JSON plus one ``needextend`` per test specification that
+links the cases matching it, so the ``results`` links become data instead of a
+Sphinx-only needs function.
+
+The ids follow sphinx-test-reports' scheme::
+
+    <file_id>
+    <file_id>_<SHA1(suite name)[:3] upper>
+    <suite_id>_<SHA1(classname + name)[:5] upper>
+"""
+
+from __future__ import annotations
+
+import argparse
+import glob as glob_module
+import hashlib
+import json
+import re
+from collections.abc import Iterable, Mapping
+from pathlib import Path
+from typing import Any
+
+from sphinxcontrib.test_reports.junitparser import JUnitParser
+
+SUITE_LEN, CASE_LEN = 3, 5  # sphinx-test-reports' tr_suite_id_length, tr_case_id_length
+NEEDS_FILE_NAME = "unit_test_results.needs.json"
+DEFAULT_PROJECT = "SPL"
+
+_SPEC_RE = re.compile(r"^\.\. test:: (.+)\n\s+:id: (\S+)", re.M)
+
+
+def sha(text: str, n: int) -> str:
+    return hashlib.sha1(text.encode("UTF-8")).hexdigest().upper()[:n]
+
+
+def block(label: str, value: str) -> str:
+    return "\n\n**{}**::\n\n   {}\n\n".format(label, "\n   ".join(x.lstrip() for x in value.split("\n")))
+
+
+def case_time(value: Any) -> str:
+    if isinstance(value, (int, float)):
+        return str(float(value) if value >= 0 else 0.0)
+    if value is None:
+        return str(0.0)
+    try:
+        return str(float(value))
+    except (TypeError, ValueError):
+        return str(0.0)
+
+
+def convert(title: str, file_id: str, junit: str) -> dict[str, dict[str, Any]]:
+    """The needs sphinx-test-reports' test-report directive creates for one JUnit file."""
+    suites = JUnitParser(junit).parse()
+    needs: dict[str, dict[str, Any]] = {}
+    tags = [file_id]
+    needs[file_id] = dict(
+        id=file_id, type="testfile", title=title, file=junit, tags=tags, links=[], content="[]",
+        suites=len(suites), cases=sum(int(s["tests"]) for s in suites), passed=sum(s["passed"] for s in suites),
+        skipped=sum(s["skips"] for s in suites), failed=sum(s["failures"] for s in suites),
+        errors=sum(s["errors"] for s in suites))
+    for suite in suites:
+        suite_id = f"{file_id}_{sha(suite['name'], SUITE_LEN)}"
+        needs[suite_id] = dict(
+            id=suite_id, type="testsuite", title=suite["name"], suite=suite["name"], file=junit, tags=tags,
+            links=[file_id], content="", cases=int(suite["tests"]), passed=suite["passed"], skipped=suite["skips"],
+            failed=suite["failures"], errors=suite["errors"])
+        for case in suite["testcases"]:
+            case_id = f"{suite_id}_{sha(case['classname'] + case['name'], CASE_LEN)}"
+            groups = re.match(r"^(?P<name>[^\[]+)($|\[(?P<param>.*)?\])", case["name"])
+            name, param = (groups["name"], groups["param"] or "") if groups else (case["name"], "")
+            content = ""
+            for key, label in (("text", "Text"), ("message", "Message"), ("system-out", "System-out")):
+                if case.get(key):
+                    content += block(label, case[key])
+            needs[case_id] = dict(
+                id=case_id, type="testcase", title=case["name"], case=case["name"], case_name=name,
+                case_parameter=param, classname=case["classname"], result=case["result"], time=case_time(case["time"]),
+                suite=suite["name"], style="tr_" + case["result"], file=junit, tags=tags,
+                links=[file_id, suite_id], content=content)
+    return needs
+
+
+def _iter_listing_files(listing_paths: Iterable[str | Path]) -> Iterable[Path]:
+    for entry in listing_paths:
+        text = str(entry)
+        matches = sorted(glob_module.glob(text, recursive=True))
+        if matches:
+            for match in matches:
+                yield Path(match)
+        else:
+            yield Path(entry)
+
+
+def collect_specs(listing_paths: Iterable[str | Path]) -> list[tuple[str, str]]:
+    """The (spec_id, spec_title) pairs the ``.. test::`` directives declare in RST listings."""
+    specs: list[tuple[str, str]] = []
+    for path in _iter_listing_files(listing_paths):
+        text = path.read_text(encoding="utf-8")
+        specs.extend((m.group(2), m.group(1).strip()) for m in _SPEC_RE.finditer(text))
+    return specs
+
+
+def results_links(needs: Mapping[str, Mapping[str, Any]], specs: Iterable[tuple[str, str]]) -> dict[str, list[str]]:
+    """Map each spec id to the case ids whose ``case`` equals or matches its title."""
+    links: dict[str, list[str]] = {}
+    for need in needs.values():
+        if need["type"] != "testcase":
+            continue
+        for spec_id, spec_title in specs:
+            if spec_title == need["case"] or ("*" in spec_title and re.match(spec_title, need["case"])):
+                links.setdefault(spec_id, []).append(need["id"])
+    return links
+
+
+def write_results(
+    page_path: str | Path,
+    title: str,
+    file_id: str,
+    junit_path: str | Path,
+    listing_paths: Iterable[str | Path],
+    project: str = DEFAULT_PROJECT,
+) -> None:
+    """Write the needs JSON next to the report page and rewrite the page to import it."""
+    page_path = Path(page_path)
+    needs = convert(title, file_id, str(junit_path))
+    needs_json = {"current_version": "", "project": project, "versions": {"": {"needs": needs}}}
+
+    page_path.parent.mkdir(parents=True, exist_ok=True)
+    (page_path.parent / NEEDS_FILE_NAME).write_text(json.dumps(needs_json, indent=1), encoding="utf-8")
+
+    links = results_links(needs, collect_specs(listing_paths))
+    lines = ["", title, "=" * len(title), "", f".. needimport:: {NEEDS_FILE_NAME}", ""]
+    for spec_id, case_ids in sorted(links.items()):
+        lines += [f".. needextend:: {spec_id}", f"   :+results: {', '.join(case_ids)}", ""]
+    page_path.write_text("\n".join(lines), encoding="utf-8")
+
+
+def main(argv: list[str] | None = None) -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--page", required=True, help="path of the report page to (re)write")
+    parser.add_argument("--title", required=True, help="title for the testfile need")
+    parser.add_argument("--id", required=True, dest="file_id", help="id for the testfile need")
+    parser.add_argument("--junit", required=True, help="path of the JUnit XML file")
+    parser.add_argument("--listings", nargs="*", default=[], metavar="GLOB", help="RST listing files that declare test specs")
+    parser.add_argument("--project", default=DEFAULT_PROJECT, help="project name for the needs.json envelope")
+    args = parser.parse_args(argv)
+    write_results(args.page, args.title, args.file_id, args.junit, args.listings, project=args.project)
+
+
+if __name__ == "__main__":
+    main()