diff --git a/docs/reference/variables.md b/docs/reference/variables.md index a7fa1caf..b59c15a8 100644 --- a/docs/reference/variables.md +++ b/docs/reference/variables.md @@ -73,6 +73,44 @@ generates, see {ref}`SPL_SPHINX_BINARY_DIR `. set(SPL_SPHINX_SOURCE_DIR docs) ``` +(SPL_SPHINX_OPTIONS)= + +## SPL_SPHINX_OPTIONS and SPL_SPHINX_COMPONENT_OPTIONS + +Options added to every `sphinx-build` spl-core runs: `SPL_SPHINX_OPTIONS` to the +variant `docs` and `reports` builds, `SPL_SPHINX_COMPONENT_OPTIONS` to the +per-component docs and report builds. In both, `@SHAPE@` becomes `docs` or +`reports` and `@COMPONENT_PATH@` the component's path relative to the project +root (empty for the variant builds), so each run can name a file of its own. + +**Default:** none + +```cmake +set(SPL_SPHINX_OPTIONS -D spl_selection=${CMAKE_BINARY_DIR}/selection/@SHAPE@.toml) +set(SPL_SPHINX_COMPONENT_OPTIONS -D spl_selection=${CMAKE_BINARY_DIR}/selection/@COMPONENT_PATH@/@SHAPE@.toml) +``` + +(SPL_TEST_RESULTS_AS_NEEDS)= + +## SPL_TEST_RESULTS_AS_NEEDS + +When `ON`, a component's unit test results page is written after each test run +from its JUnit XML, instead of at configure time with sphinx-test-reports' +`test-report` directive. `spl_core.test_report.junit_to_needs` converts the JUnit +XML into `unit_test_results.needs.json`, with the same needs, IDs and fields the +directive creates, and writes a page that imports it with `needimport`. The page +also carries the `results` links of the component's test specifications as +`needextend` blocks: a specification links every test case whose name equals its +title, or matches it as a regular expression when the title contains `*`. Every +reader of needs.json sees the results, not only Sphinx, and the project can drop +the `sphinxcontrib.test_reports` extension and the `sple_tr_link` needs function. + +**Default:** `OFF` + +```cmake +set(SPL_TEST_RESULTS_AS_NEEDS ON) +``` + (SPL_SPHINX_BINARY_DIR)= ## SPL_SPHINX_BINARY_DIR @@ -93,10 +131,14 @@ gcovr HTML report is written to next to its coverage page, and the report artifacts `SplBuild` looks up. A relative path is taken relative to the project root. -Because the path usually is a link the project re-points, every docs and reports -build first checks that it still leads to its own binary directory, and fails -with a message naming both paths if another build directory was configured in -the meantime. +When the path is a link the project re-points, every docs and reports build first +checks that it still leads to its own binary directory, and fails with a message +naming both paths if another build directory was configured in the meantime. +When the path does not exist on disk, the project reaches the binary directory +another way, for example by mounting it at that path with +[sphinx-mounts](https://github.com/useblocks/sphinx-mounts) and naming each +build's directory in its runs' options ({ref}`SPL_SPHINX_OPTIONS `), +and there is nothing to check. **Default:** the binary directory itself (`CMAKE_BINARY_DIR`) diff --git a/src/spl_core/check_sphinx_binary_dir.cmake b/src/spl_core/check_sphinx_binary_dir.cmake index f8979265..04df7b72 100644 --- a/src/spl_core/check_sphinx_binary_dir.cmake +++ b/src/spl_core/check_sphinx_binary_dir.cmake @@ -10,10 +10,11 @@ # # cmake -DSPL_SPHINX_BINARY_DIR= -DSPL_BINARY_DIR= -P check_sphinx_binary_dir.cmake +# A path that does not exist on disk is not a link: the project reaches the binary +# directory another way, for example through a sphinx-mounts mount that each build +# names in the options of its own runs. There is nothing to compare. if(NOT EXISTS "${SPL_SPHINX_BINARY_DIR}") - message(FATAL_ERROR - "SPL_SPHINX_BINARY_DIR ${SPL_SPHINX_BINARY_DIR} does not exist. " - "It has to lead to ${SPL_BINARY_DIR} before this build's documentation can be built.") + return() endif() file(REAL_PATH "${SPL_SPHINX_BINARY_DIR}" _resolved) diff --git a/src/spl_core/common.cmake b/src/spl_core/common.cmake index 7af24b9f..33bacdff 100644 --- a/src/spl_core/common.cmake +++ b/src/spl_core/common.cmake @@ -331,7 +331,7 @@ macro(spl_create_component) # We do not know all dependencies for generating the docs (apart from the rst files). # This might cause incremental builds to not update parts of the documentation. # To avoid this the command passes -E to make sphinx-build write all files new. - _spl_sphinx_build_command(_spl_sphinx_build SHAPE docs CONFIG ${_docs_config_json} OUTPUT_DIR ${_component_docs_html_out_dir}) + _spl_sphinx_build_command(_spl_sphinx_build SHAPE docs CONFIG ${_docs_config_json} OUTPUT_DIR ${_component_docs_html_out_dir} COMPONENT ${component_path}) _spl_sphinx_binary_dir_check(_spl_sphinx_check) add_custom_target( ${component_name}_docs @@ -363,9 +363,11 @@ Unit Test Specification ") - # create the test results rst file + # create the test results rst file. With SPL_TEST_RESULTS_AS_NEEDS the page is + # written after the test run instead, together with its needs.json (see below). set(_unit_test_results_rst ${_component_reports_out_dir}/unit_test_results.rst) - file(WRITE ${_unit_test_results_rst} " + if(NOT SPL_TEST_RESULTS_AS_NEEDS) + file(WRITE ${_unit_test_results_rst} " Unit Test Results ================= @@ -374,6 +376,7 @@ Unit Test Results :file: ${_component_test_junit_xml} ") + endif() # create coverage rst file to be able to automatically link to the coverage/index.html set(_coverage_rst ${_component_reports_out_dir}/coverage.rst) @@ -391,7 +394,10 @@ Code Coverage }") # add the generated files as dependency to cmake configure step - set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${_reports_config_json} ${_unit_test_spec_rst} ${_unit_test_results_rst}) + set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${_reports_config_json} ${_unit_test_spec_rst}) + if(NOT SPL_TEST_RESULTS_AS_NEEDS) + set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${_unit_test_results_rst}) + endif() set(_cov_out_html reports/html/${_rel_component_reports_out_dir}/coverage/index.html) file(RELATIVE_PATH _cov_out_json ${CMAKE_CURRENT_BINARY_DIR} ${_component_coverage_json}) @@ -407,7 +413,7 @@ Code Coverage # No OUTPUT is defined to force execution of this target every time # TODO: list of dependencies is not complete - _spl_sphinx_build_command(_spl_sphinx_build SHAPE reports CONFIG ${_reports_config_json} OUTPUT_DIR ${_component_reports_html_out_dir}) + _spl_sphinx_build_command(_spl_sphinx_build SHAPE reports CONFIG ${_reports_config_json} OUTPUT_DIR ${_component_reports_html_out_dir} COMPONENT ${component_path}) _spl_sphinx_binary_dir_check(_spl_sphinx_check) add_custom_target( ${component_name}_report @@ -426,6 +432,33 @@ Code Coverage _spl_generate_clanguru_source_docs(${component_name} "${_clanguru_all_sources}") endif() + # Test results as needs: after each test run, convert the JUnit XML into the + # needs sphinx-test-reports' test-report directive would create, and write the + # results page that imports them. Every reader of needs.json sees them, not + # only Sphinx. The page also carries the `results` links of the test + # specifications, taken from the source listings, as needextend blocks. + if(SPL_TEST_RESULTS_AS_NEEDS AND TEST_SOURCES) + set(_unit_test_results_needs_json ${_component_reports_out_dir}/unit_test_results.needs.json) + add_custom_command( + OUTPUT ${_unit_test_results_rst} ${_unit_test_results_needs_json} + COMMAND ${CMAKE_COMMAND} -E make_directory ${_component_reports_out_dir} + COMMAND ${SPL_PYTHON} -m spl_core.test_report.junit_to_needs + --page ${_unit_test_results_rst} + --title "Unit Test Results" + --id TEST_RESULT_${component_name} + --junit ${_component_test_junit_xml} + --project ${PROJECT_NAME} + --listings "${_clanguru_docs_out_dir}/**/*.rst" + DEPENDS ${_component_test_junit_xml} ${_clanguru_doc_outputs} + COMMENT "Converting the test results of ${component_name} into needs ..." + VERBATIM + ) + add_custom_target(${component_name}_test_results DEPENDS ${_unit_test_results_rst} ${_unit_test_results_needs_json}) + add_dependencies(${component_name}_report ${component_name}_test_results) + list(APPEND SPL_TEST_RESULTS_TARGETS ${component_name}_test_results) + set(SPL_TEST_RESULTS_TARGETS ${SPL_TEST_RESULTS_TARGETS} PARENT_SCOPE) + endif() + # Store the source docs directory so the variant report wrapper page can # nest it under the component (see _spl_create_reports_target). if(_COMPONENT_SOURCE_DOCS_INCLUDE_PATTERN) @@ -540,11 +573,14 @@ function(_spl_sphinx_relative_path out_var path) endfunction() # The check a docs or reports build runs before sphinx-build when SPL_SPHINX_BINARY_DIR -# is set, including its COMMAND keyword, or nothing otherwise. The path is usually -# a link the project re-points whenever it configures a build directory, so by the +# is set, including its COMMAND keyword, or nothing otherwise. The path may be a +# link the project re-points whenever it configures a build directory, so by the # time this build runs it may lead to another build's output. Sphinx would then # read that build's generated pages without a word; the check stops the build -# instead. +# instead. When the path does not exist on disk, the project reaches the binary +# directory another way, for example by mounting it there with sphinx-mounts and +# naming each build's directory in the options of its own runs, and the check has +# nothing to compare. function(_spl_sphinx_binary_dir_check out_var) _spl_sphinx_binary_dir(_binary_dir) if(_binary_dir STREQUAL CMAKE_BINARY_DIR) @@ -568,8 +604,14 @@ endfunction() # the file each shape reads, and the build gets it as `-D needs_variant_data_file=`. # sphinx-needs keeps a command-line override even when the project's # needs_from_toml names another file, so conf.py needs no code to select it. +# +# COMPONENT is the component's path for the per-component builds and empty for the +# variant builds. A project adds options of its own with SPL_SPHINX_OPTIONS (variant +# builds) and SPL_SPHINX_COMPONENT_OPTIONS (per-component builds); `@SHAPE@` in +# them becomes the shape and `@COMPONENT_PATH@` the component's path, so each run +# can name a file of its own, e.g. `-D;spl_selection=/@COMPONENT_PATH@/@SHAPE@.toml`. function(_spl_sphinx_build_command out_var) - cmake_parse_arguments(ARG "" "SHAPE;CONFIG;OUTPUT_DIR" "" ${ARGN}) + cmake_parse_arguments(ARG "" "SHAPE;CONFIG;OUTPUT_DIR;COMPONENT" "" ${ARGN}) if(ARG_SHAPE STREQUAL "docs") set(_variant_data_file "${SPL_VARIANT_DATA_FILE_DOCS}") elseif(ARG_SHAPE STREQUAL "reports") @@ -582,6 +624,16 @@ function(_spl_sphinx_build_command out_var) if(_variant_data_file) list(APPEND _options -D needs_variant_data_file=${_variant_data_file}) endif() + if(ARG_COMPONENT) + set(_extra_options ${SPL_SPHINX_COMPONENT_OPTIONS}) + else() + set(_extra_options ${SPL_SPHINX_OPTIONS}) + endif() + foreach(_option IN LISTS _extra_options) + string(REPLACE "@SHAPE@" "${ARG_SHAPE}" _option "${_option}") + string(REPLACE "@COMPONENT_PATH@" "${ARG_COMPONENT}" _option "${_option}") + list(APPEND _options "${_option}") + endforeach() _spl_sphinx_source_dir(_source_dir) set(${out_var} @@ -766,6 +818,9 @@ Code Coverage BYPRODUCTS ${_reports_html_output_dir}/index.html DEPENDS ${JUNIT_OUT_VARIANT_XML} ${COV_OUT_VARIANT_JSON} _components_variant_coverage_html_target source_docs ) + if(SPL_TEST_RESULTS_TARGETS) + add_dependencies(reports ${SPL_TEST_RESULTS_TARGETS}) + endif() endmacro() macro(_spl_set_coverage_create_overall_report_is_necessary) diff --git a/src/spl_core/test_report/__init__.py b/src/spl_core/test_report/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/spl_core/test_report/junit_to_needs.py b/src/spl_core/test_report/junit_to_needs.py new file mode 100644 index 00000000..258a277e --- /dev/null +++ b/src/spl_core/test_report/junit_to_needs.py @@ -0,0 +1,159 @@ +"""Turn JUnit XML results into needs that every reader can import. + +sphinx-test-reports' ``test-report`` directive creates its testfile, +testsuite and testcase needs only while Sphinx runs, so the result data +disappears for any reader that just imports ``needs.json`` (ubCode, ``ubc``). +This module reproduces exactly those needs -- same ids, types and fields -- +from the same JUnit XML, using sphinx-test-reports' own parser, and writes them +next to the generated report page. The page itself then holds a plain +``needimport`` of that JSON plus one ``needextend`` per test specification that +links the cases matching it, so the ``results`` links become data instead of a +Sphinx-only needs function. + +The ids follow sphinx-test-reports' scheme:: + + + _ + _ +""" + +from __future__ import annotations + +import argparse +import glob as glob_module +import hashlib +import json +import re +from collections.abc import Iterable, Mapping +from pathlib import Path +from typing import Any + +from sphinxcontrib.test_reports.junitparser import JUnitParser + +SUITE_LEN, CASE_LEN = 3, 5 # sphinx-test-reports' tr_suite_id_length, tr_case_id_length +NEEDS_FILE_NAME = "unit_test_results.needs.json" +DEFAULT_PROJECT = "SPL" + +_SPEC_RE = re.compile(r"^\.\. test:: (.+)\n\s+:id: (\S+)", re.M) + + +def sha(text: str, n: int) -> str: + return hashlib.sha1(text.encode("UTF-8")).hexdigest().upper()[:n] + + +def block(label: str, value: str) -> str: + return "\n\n**{}**::\n\n {}\n\n".format(label, "\n ".join(x.lstrip() for x in value.split("\n"))) + + +def case_time(value: Any) -> str: + if isinstance(value, (int, float)): + return str(float(value) if value >= 0 else 0.0) + if value is None: + return str(0.0) + try: + return str(float(value)) + except (TypeError, ValueError): + return str(0.0) + + +def convert(title: str, file_id: str, junit: str) -> dict[str, dict[str, Any]]: + """The needs sphinx-test-reports' test-report directive creates for one JUnit file.""" + suites = JUnitParser(junit).parse() + needs: dict[str, dict[str, Any]] = {} + tags = [file_id] + needs[file_id] = dict( + id=file_id, type="testfile", title=title, file=junit, tags=tags, links=[], content="[]", + suites=len(suites), cases=sum(int(s["tests"]) for s in suites), passed=sum(s["passed"] for s in suites), + skipped=sum(s["skips"] for s in suites), failed=sum(s["failures"] for s in suites), + errors=sum(s["errors"] for s in suites)) + for suite in suites: + suite_id = f"{file_id}_{sha(suite['name'], SUITE_LEN)}" + needs[suite_id] = dict( + id=suite_id, type="testsuite", title=suite["name"], suite=suite["name"], file=junit, tags=tags, + links=[file_id], content="", cases=int(suite["tests"]), passed=suite["passed"], skipped=suite["skips"], + failed=suite["failures"], errors=suite["errors"]) + for case in suite["testcases"]: + case_id = f"{suite_id}_{sha(case['classname'] + case['name'], CASE_LEN)}" + groups = re.match(r"^(?P[^\[]+)($|\[(?P.*)?\])", case["name"]) + name, param = (groups["name"], groups["param"] or "") if groups else (case["name"], "") + content = "" + for key, label in (("text", "Text"), ("message", "Message"), ("system-out", "System-out")): + if case.get(key): + content += block(label, case[key]) + needs[case_id] = dict( + id=case_id, type="testcase", title=case["name"], case=case["name"], case_name=name, + case_parameter=param, classname=case["classname"], result=case["result"], time=case_time(case["time"]), + suite=suite["name"], style="tr_" + case["result"], file=junit, tags=tags, + links=[file_id, suite_id], content=content) + return needs + + +def _iter_listing_files(listing_paths: Iterable[str | Path]) -> Iterable[Path]: + for entry in listing_paths: + text = str(entry) + matches = sorted(glob_module.glob(text, recursive=True)) + if matches: + for match in matches: + yield Path(match) + else: + yield Path(entry) + + +def collect_specs(listing_paths: Iterable[str | Path]) -> list[tuple[str, str]]: + """The (spec_id, spec_title) pairs the ``.. test::`` directives declare in RST listings.""" + specs: list[tuple[str, str]] = [] + for path in _iter_listing_files(listing_paths): + text = path.read_text(encoding="utf-8") + specs.extend((m.group(2), m.group(1).strip()) for m in _SPEC_RE.finditer(text)) + return specs + + +def results_links(needs: Mapping[str, Mapping[str, Any]], specs: Iterable[tuple[str, str]]) -> dict[str, list[str]]: + """Map each spec id to the case ids whose ``case`` equals or matches its title.""" + links: dict[str, list[str]] = {} + for need in needs.values(): + if need["type"] != "testcase": + continue + for spec_id, spec_title in specs: + if spec_title == need["case"] or ("*" in spec_title and re.match(spec_title, need["case"])): + links.setdefault(spec_id, []).append(need["id"]) + return links + + +def write_results( + page_path: str | Path, + title: str, + file_id: str, + junit_path: str | Path, + listing_paths: Iterable[str | Path], + project: str = DEFAULT_PROJECT, +) -> None: + """Write the needs JSON next to the report page and rewrite the page to import it.""" + page_path = Path(page_path) + needs = convert(title, file_id, str(junit_path)) + needs_json = {"current_version": "", "project": project, "versions": {"": {"needs": needs}}} + + page_path.parent.mkdir(parents=True, exist_ok=True) + (page_path.parent / NEEDS_FILE_NAME).write_text(json.dumps(needs_json, indent=1), encoding="utf-8") + + links = results_links(needs, collect_specs(listing_paths)) + lines = ["", title, "=" * len(title), "", f".. needimport:: {NEEDS_FILE_NAME}", ""] + for spec_id, case_ids in sorted(links.items()): + lines += [f".. needextend:: {spec_id}", f" :+results: {', '.join(case_ids)}", ""] + page_path.write_text("\n".join(lines), encoding="utf-8") + + +def main(argv: list[str] | None = None) -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--page", required=True, help="path of the report page to (re)write") + parser.add_argument("--title", required=True, help="title for the testfile need") + parser.add_argument("--id", required=True, dest="file_id", help="id for the testfile need") + parser.add_argument("--junit", required=True, help="path of the JUnit XML file") + parser.add_argument("--listings", nargs="*", default=[], metavar="GLOB", help="RST listing files that declare test specs") + parser.add_argument("--project", default=DEFAULT_PROJECT, help="project name for the needs.json envelope") + args = parser.parse_args(argv) + write_results(args.page, args.title, args.file_id, args.junit, args.listings, project=args.project) + + +if __name__ == "__main__": + main() diff --git a/tests/cmake/common.cmake/CMakeLists.txt b/tests/cmake/common.cmake/CMakeLists.txt index 0d52bcc1..7ef4ff35 100644 --- a/tests/cmake/common.cmake/CMakeLists.txt +++ b/tests/cmake/common.cmake/CMakeLists.txt @@ -112,6 +112,28 @@ unset(AUTOCONF_JSON) unset(SPL_VARIANT_DATA_FILE_DOCS) unset(SPL_VARIANT_DATA_FILE_REPORTS) +# ## test: _spl_sphinx_build_command (extra options per run) ######### +# given options for the variant runs and for the per-component runs +set(SPL_SPHINX_OPTIONS -D spl_selection=/sel/@SHAPE@.toml) +set(SPL_SPHINX_COMPONENT_OPTIONS -D spl_selection=/sel/@COMPONENT_PATH@/@SHAPE@.toml) + +# when +_spl_sphinx_build_command(sphinx_variant_command SHAPE reports CONFIG /build/reports/config.json OUTPUT_DIR /build/reports/html) +_spl_sphinx_build_command(sphinx_component_command SHAPE docs CONFIG /build/c/docs/config.json OUTPUT_DIR /build/c/docs/html COMPONENT components/c) + +# then: a variant run gets the variant options, a component run the component options, +# each with its own shape and component path filled in +if(NOT sphinx_variant_command MATCHES ";-D;spl_selection=/sel/reports\\.toml;[^;]+;/build/reports/html$") + message(FATAL_ERROR "Failing Test case: the variant reports run must name its own file, got '${sphinx_variant_command}'.") +endif() +if(NOT sphinx_component_command MATCHES ";-D;spl_selection=/sel/components/c/docs\\.toml;[^;]+;/build/c/docs/html$" OR sphinx_component_command MATCHES "/sel/docs\\.toml") + message(FATAL_ERROR "Failing Test case: the component docs run must name the component's file only, got '${sphinx_component_command}'.") +endif() + +# cleanup +unset(SPL_SPHINX_OPTIONS) +unset(SPL_SPHINX_COMPONENT_OPTIONS) + # ## test: _spl_sphinx_source_dir and _spl_sphinx_relative_path ######### # given the default configuration @@ -218,15 +240,15 @@ if(check_other_dir_result EQUAL 0) message(FATAL_ERROR "Failing Test case: the check must fail when the path leads to another directory.") endif() -# when it does not exist +# when it does not exist on disk, as when the project mounts the binary directory there execute_process( COMMAND ${CMAKE_COMMAND} -DSPL_SPHINX_BINARY_DIR=${PROJECT_BINARY_DIR}/does_not_exist -DSPL_BINARY_DIR=${PROJECT_BINARY_DIR} -P ${check_sphinx_binary_dir_script} RESULT_VARIABLE check_missing_dir_result OUTPUT_QUIET ERROR_QUIET ) -# then -if(check_missing_dir_result EQUAL 0) - message(FATAL_ERROR "Failing Test case: the check must fail when the path does not exist.") +# then: there is no link to compare, so the build goes ahead +if(NOT check_missing_dir_result EQUAL 0) + message(FATAL_ERROR "Failing Test case: the check must pass when the path does not exist.") endif() # ## test: _spl_get_absolute_path ######### diff --git a/tests/data/junit/listing.rst b/tests/data/junit/listing.rst new file mode 100644 index 00000000..9aa0244e --- /dev/null +++ b/tests/data/junit/listing.rst @@ -0,0 +1,13 @@ +.. This file mirrors the generated source listings that declare test specs. + +Some generated listing. +*********************** + +.. test:: test_addition + :id: TS_ADD + +.. test:: test_division + :id: TS_DIV + +.. test:: test_param* + :id: TS_PARAM diff --git a/tests/data/junit/two_suites.xml b/tests/data/junit/two_suites.xml new file mode 100644 index 00000000..797188a6 --- /dev/null +++ b/tests/data/junit/two_suites.xml @@ -0,0 +1,15 @@ + + + + + + Traceback: division by zero + + + + + + + + + diff --git a/tests/unit/test_report_junit_to_needs.py b/tests/unit/test_report_junit_to_needs.py new file mode 100644 index 00000000..23ba1443 --- /dev/null +++ b/tests/unit/test_report_junit_to_needs.py @@ -0,0 +1,184 @@ +"""Unit tests for converting JUnit XML into importable test-result needs.""" + +import hashlib +import json +from pathlib import Path + +import pytest + +from spl_core.test_report.junit_to_needs import ( + NEEDS_FILE_NAME, + collect_specs, + convert, + results_links, + write_results, +) + +DATA_DIR = Path(__file__).parent.parent / "data" / "junit" +JUNIT_FILE = DATA_DIR / "two_suites.xml" +LISTING_FILE = DATA_DIR / "listing.rst" + +FILE_ID = "TEST_RESULT_Demo" + + +def _digest(text: str, length: int) -> str: + return hashlib.sha1(text.encode("UTF-8")).hexdigest().upper()[:length] + + +MATH_SUITE_ID = f"{FILE_ID}_{_digest('MathSuite', 3)}" +PARAM_SUITE_ID = f"{FILE_ID}_{_digest('ParamSuite', 3)}" +ADDITION_CASE_ID = f"{MATH_SUITE_ID}_{_digest('MathTests' + 'test_addition', 5)}" +DIVISION_CASE_ID = f"{MATH_SUITE_ID}_{_digest('MathTests' + 'test_division', 5)}" +SKIP_CASE_ID = f"{MATH_SUITE_ID}_{_digest('MathTests' + 'test_skip_me', 5)}" +PARAM_CASE_ID = f"{PARAM_SUITE_ID}_{_digest('ParamTests' + 'test_param[1]', 5)}" + + +@pytest.fixture +def needs(): + return convert("Unit Test Results", FILE_ID, str(JUNIT_FILE)) + + +@pytest.mark.unit +def test_ids_follow_the_test_report_scheme(needs): + assert set(needs) == { + FILE_ID, + MATH_SUITE_ID, + PARAM_SUITE_ID, + ADDITION_CASE_ID, + DIVISION_CASE_ID, + SKIP_CASE_ID, + PARAM_CASE_ID, + } + assert needs[MATH_SUITE_ID]["id"] == MATH_SUITE_ID + assert needs[ADDITION_CASE_ID]["id"] == ADDITION_CASE_ID + + +@pytest.mark.unit +def test_testfile_counts_and_fields(needs): + testfile = needs[FILE_ID] + assert testfile["type"] == "testfile" + assert testfile["title"] == "Unit Test Results" + assert testfile["file"] == str(JUNIT_FILE) + assert testfile["tags"] == [FILE_ID] + assert testfile["links"] == [] + assert testfile["suites"] == 2 + assert testfile["cases"] == 4 + assert testfile["passed"] == 2 + assert testfile["skipped"] == 1 + assert testfile["failed"] == 1 + assert testfile["errors"] == 0 + + +@pytest.mark.unit +def test_testsuite_counts_and_fields(needs): + math = needs[MATH_SUITE_ID] + assert math["type"] == "testsuite" + assert math["suite"] == "MathSuite" + assert math["links"] == [FILE_ID] + assert (math["cases"], math["passed"], math["skipped"], math["failed"], math["errors"]) == (3, 1, 1, 1, 0) + + param = needs[PARAM_SUITE_ID] + assert param["suite"] == "ParamSuite" + assert (param["cases"], param["passed"], param["skipped"], param["failed"], param["errors"]) == (1, 1, 0, 0, 0) + + +@pytest.mark.unit +def test_passed_case_fields(needs): + case = needs[ADDITION_CASE_ID] + assert case["type"] == "testcase" + assert case["case"] == "test_addition" + assert case["case_name"] == "test_addition" + assert case["case_parameter"] == "" + assert case["classname"] == "MathTests" + assert case["result"] == "passed" + assert case["style"] == "tr_passed" + assert case["suite"] == "MathSuite" + assert case["links"] == [FILE_ID, MATH_SUITE_ID] + assert case["content"] == "" + + +@pytest.mark.unit +def test_failed_case_fields(needs): + case = needs[DIVISION_CASE_ID] + assert case["result"] == "failure" + assert case["style"] == "tr_failure" + assert case["time"] == "0.2" + assert "**Message**::" in case["content"] + assert "division by zero" in case["content"] + assert "**Text**::" in case["content"] + assert "Traceback: division by zero" in case["content"] + + +@pytest.mark.unit +def test_skipped_case_fields(needs): + case = needs[SKIP_CASE_ID] + assert case["result"] == "skipped" + assert case["style"] == "tr_skipped" + assert "**Message**::" in case["content"] + assert "not implemented" in case["content"] + + +@pytest.mark.unit +def test_parameterised_case_is_split(needs): + case = needs[PARAM_CASE_ID] + assert case["case"] == "test_param[1]" + assert case["case_name"] == "test_param" + assert case["case_parameter"] == "1" + + +@pytest.mark.unit +def test_collect_specs_reads_id_and_title_pairs(): + specs = collect_specs([LISTING_FILE]) + assert specs == [ + ("TS_ADD", "test_addition"), + ("TS_DIV", "test_division"), + ("TS_PARAM", "test_param*"), + ] + + +@pytest.mark.unit +def test_results_links_exact_title_and_regex(needs): + links = results_links(needs, collect_specs([LISTING_FILE])) + assert links == { + "TS_ADD": [ADDITION_CASE_ID], + "TS_DIV": [DIVISION_CASE_ID], + "TS_PARAM": [PARAM_CASE_ID], + } + + +@pytest.mark.unit +def test_write_results_writes_needs_json_and_page(tmp_path): + page = tmp_path / "reports" / "unit_test_results.rst" + write_results(page, "Unit Test Results", FILE_ID, JUNIT_FILE, [LISTING_FILE]) + + needs_json = json.loads((page.parent / NEEDS_FILE_NAME).read_text(encoding="utf-8")) + assert needs_json["current_version"] == "" + assert needs_json["project"] == "SPL" + assert set(needs_json["versions"]) == {""} + assert set(needs_json["versions"][""]["needs"]) == { + FILE_ID, + MATH_SUITE_ID, + PARAM_SUITE_ID, + ADDITION_CASE_ID, + DIVISION_CASE_ID, + SKIP_CASE_ID, + PARAM_CASE_ID, + } + + page_text = page.read_text(encoding="utf-8") + assert "Unit Test Results\n=================" in page_text + assert f".. needimport:: {NEEDS_FILE_NAME}" in page_text + # needextend blocks are emitted in sorted spec-id order + assert page_text.index(".. needextend:: TS_ADD") < page_text.index(".. needextend:: TS_DIV") + assert page_text.index(".. needextend:: TS_DIV") < page_text.index(".. needextend:: TS_PARAM") + assert f" :+results: {ADDITION_CASE_ID}" in page_text + assert f" :+results: {DIVISION_CASE_ID}" in page_text + assert f" :+results: {PARAM_CASE_ID}" in page_text + + +@pytest.mark.unit +def test_write_results_project_name_is_configurable(tmp_path): + page = tmp_path / "unit_test_results.rst" + write_results(page, "Unit Test Results", FILE_ID, JUNIT_FILE, [], project="SPLed") + needs_json = json.loads((page.parent / NEEDS_FILE_NAME).read_text(encoding="utf-8")) + assert needs_json["project"] == "SPLed"