docs: fix test execution via pytest from venv (#954)

* docs: fix test execution via pytest from venv
* fix: store only string in config
* misc: cleanup
* fix: handle empty filter in parse_checks_filter and improve error reporting in extract_test_data
* chore: remove unused apply_enabled_check_filter function

And add CHECK option for rst-test files to run a single check

Co-authored-by: Maximilian Sören Pollak <maximilian.pollak@expleogroup.com>
Co-authored-by: Michael Müller <42868757+mmr1909@users.noreply.github.com>

Author: 3 people

tooling/docs/_tooling/extensions/score_metamodel/__init__.py

@@ -24,8 +24,21 @@
 
 logger = logging.get_logger(__name__)
 
-local_checks: list[Callable[[Sphinx, NeedsInfoType, CheckLogger], None]] = []
-graph_checks: list[Callable[[Sphinx, list[NeedsInfoType], CheckLogger], None]] = []
+local_check_function = Callable[[Sphinx, NeedsInfoType, CheckLogger], None]
+graph_check_function = Callable[[Sphinx, list[NeedsInfoType], CheckLogger], None]
+
+local_checks: list[local_check_function] = []
+graph_checks: list[graph_check_function] = []
+
+
+def parse_checks_filter(filter: str) -> list[str]:
+    """
+    Parse the checks filter string into a list of individual checks.
+    When empty, an empty list is returned = all checks are enabled.
+    """
+    if not filter:
+        return []
+    return [check.strip() for check in filter.split(",")]
 
 
 def discover_checks():
@@ -41,14 +54,14 @@ def discover_checks():
         importlib.import_module(f"{package_name}.{module_name}", __package__)
 
 
-def local_check(func: Callable[[Sphinx, NeedsInfoType, CheckLogger], None]):
+def local_check(func: local_check_function):
     """Use this decorator to mark a function as a local check."""
     logger.debug(f"new local_check: {func}")
     local_checks.append(func)
     return func
 
 
-def graph_check(func: Callable[[Sphinx, list[NeedsInfoType], CheckLogger], None]):
+def graph_check(func: graph_check_function):
     """Use this decorator to mark a function as a graph check."""
     logger.debug(f"new graph_check: {func}")
     graph_checks.append(func)
@@ -68,21 +81,27 @@ def _run_checks(app: Sphinx, exception: Exception | None) -> None:
 
     log = CheckLogger(logger, prefix)
 
+    checks_filter = parse_checks_filter(app.config.score_metamodel_checks)
+
+    def is_check_enabled(check: local_check_function | graph_check_function):
+        return not checks_filter or check.__name__ in checks_filter
+
     # Need-Local checks: checks which can be checked file-local, without a
     # graph of other needs.
     for need in needs_all_needs.values():
-        for check in local_checks:
+        for check in [c for c in local_checks if is_check_enabled(c)]:
+            logger.info(f"Running local check {check} for need {need['id']}")
             check(app, need, log)
 
     # Graph-Based checks: These warnings require a graph of all other needs to
     # be checked.
     needs = list(needs_all_needs.values())
-    for check in graph_checks:
+    for check in [c for c in graph_checks if is_check_enabled(c)]:
+        logger.info(f"Running graph check {check} for all needs")
         check(app, needs, log)
 
     if log.has_warnings:
         log.warning("Some needs have issues. See the log for more information.")
-        # TODO: exit code
 
 
 def load_metamodel_data():
@@ -247,6 +266,15 @@ def setup(app: Sphinx) -> dict[str, str | bool]:
 
     discover_checks()
 
+    app.add_config_value(
+        "score_metamodel_checks",
+        "",
+        rebuild="env",
+        description=(
+            "Comma separated list of enabled checks. When empty, all checks are enabled"
+        ),
+    )
+
     app.connect("build-finished", _run_checks)
 
     return {

tooling/docs/_tooling/extensions/score_metamodel/tests/README.md

@@ -16,11 +16,19 @@ To add a new test case create a new rst file in the rst directory.
 The test files can also be organized in a subfolder structure below directory rst.
 The test files are expected to contain the following format:
 
+    #CHECK: <check functions>
+
     #EXPECT: <warning message>
     #EXPECT-NOT: <warning message>
 
     <need information>
 
+**\<check functions>**<br>
+Check functions (comma separated) to be used for the Sphinx build. Following
+                    warnings will only be generated by these check functions.
+                    Only one CHECK statement per file. Usually at the very top.
+                    If CHECK is not provided, all checks are enabled.
+
 **\<warning message>**<br>
 Message text which is expected/not expected during the
                     Sphinx build to be shown.
@@ -33,11 +41,12 @@ One or more Sphinx-Needs directives needed for the
 
 **Example:**
 
-    EXPECT: std_wp__test__abcd: is missing required option: `status`.
+    #CHECK: check_options
+    #EXPECT: std_wp__test__abcd: is missing required option: `status`.
 
     .. std_wp:: Test requirement
         :id: std_wp__test__abcd
 
 This example verifies that the warning message
-*std_wp__test__abcd: is missing required option: `status`*
-is shown during the Sphinx build.
+*std_wp__test__abcd: is missing required option: \`status\`*
+is shown during the Sphinx build. Only the *check_options* check is enabled.

tooling/docs/_tooling/extensions/score_metamodel/tests/rst/conf.py

@@ -0,0 +1,22 @@
+# *******************************************************************************
+# Copyright (c) 2024 Contributors to the Eclipse Foundation
+#
+# See the NOTICE file(s) distributed with this work for additional
+# information regarding copyright ownership.
+#
+# This program and the accompanying materials are made available under the
+# terms of the Apache License Version 2.0 which is available at
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# SPDX-License-Identifier: Apache-2.0
+# *******************************************************************************
+
+# Configuration file for the Sphinx documentation builder.
+#
+# For the full list of built-in configuration values, see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+extensions = [
+    "sphinx_needs",
+    "score_metamodel",
+]

tooling/docs/_tooling/extensions/score_metamodel/tests/rst/graph/test_metamodel_graph.rst

@@ -11,6 +11,7 @@
    #
    # SPDX-License-Identifier: Apache-2.0
    # *******************************************************************************
+#CHECK: check_metamodel_graph
 
 .. feat_req:: Parent requirement
    :id: feat_req__parent__abcd

tooling/docs/_tooling/extensions/score_metamodel/tests/rst/options/test_options_extra_option.rst

@@ -11,6 +11,7 @@
    #
    # SPDX-License-Identifier: Apache-2.0
    # *******************************************************************************
+#CHECK: check_extra_options
 
 #EXPECT: std_wp__test__abcd: has these extra options: `safety`.
 

tooling/docs/_tooling/extensions/score_metamodel/tests/rst/options/test_options_options.rst

@@ -11,6 +11,7 @@
    #
    # SPDX-License-Identifier: Apache-2.0
    # *******************************************************************************
+#CHECK: check_options
 
 #EXPECT: std_wp__test__abcd: is missing required option: `status`.
 

tooling/docs/_tooling/extensions/score_metamodel/tests/test_rules_file_based.py

@@ -20,6 +20,11 @@
 import pytest
 from sphinx.testing.util import SphinxTestApp
 
+from docs._tooling.extensions.score_metamodel import (
+    graph_check_function,
+    local_check_function,
+)
+
 RST_DIR = Path(__file__).absolute().parent / "rst"
 DOCS_DIR = Path(__file__).absolute().parent.parent.parent.parent.parent
 TOOLING_DIR_NAME = "_tooling"
@@ -32,7 +37,7 @@
 def sphinx_base_dir(tmp_path_factory: pytest.TempPathFactory) -> Path:
     ### Create a temporary directory for Sphinx and copy all necessary files.
     base_dir: Path = tmp_path_factory.mktemp("docs")
-    shutil.copy(DOCS_DIR / "conf.py", base_dir)
+    shutil.copy(RST_DIR / "conf.py", base_dir)
     shutil.copytree(
         DOCS_DIR / TOOLING_DIR_NAME,
         base_dir / TOOLING_DIR_NAME,
@@ -82,36 +87,43 @@ def _create_app(rst_file: Path) -> SphinxTestApp:
 @dataclass
 class WarningInfo:
     #### Class to hold information about warnings
-    # The class contains the filename of the rst file,
-    # line number and the expected and not expected warnings.
-    filename: str = ""
+    # Contains the line number and the expected and not expected warnings.
     lineno: int = 0
     expected: list[str] = field(default_factory=list)
     not_expected: list[str] = field(default_factory=list)
 
 
-def extract_warning(line: str) -> str:
+@dataclass
+class RstData:
+    #### Holds filename, all infos about warnings and
+    # which checks to enable if not all
+    filename: str
+    enabled_checks: str = ""
+    warning_infos: list[WarningInfo] = field(default_factory=list)
+
+
+def parse_line_for_message(line: str) -> str:
     #### Extract the warning message from the line
     # The line format is "#EXPECT: <warning message>"
     # or "#EXPECT-NOT: <warning message>"
+    # or "#CHECK: <checks>"
     return line.split(": ", 1)[1].strip()
 
 
-def extract_test_data(rst_file: Path) -> list[WarningInfo] | None:
+def extract_test_data(rst_file: Path) -> RstData | None:
     ### Extract test data from the given rst file
     # The function returns a list of WarningInfo objects
     # containing the line number and the expected and not expected warnings.
     # If no test data is found, it returns None.
+    rst_data = RstData(filename=rst_file.name)
     with open(rst_file) as f:
-        statements: list[WarningInfo] = []
         test_info: WarningInfo | None = None
         for no, line in enumerate(f, start=1):
             if line.startswith(".. "):  # Beginning of new need
                 if test_info:
-                    test_info.filename = rst_file.name
                     test_info.lineno = no
-                    statements.append(test_info)
-                test_info = None
+                    rst_data.warning_infos.append(test_info)
+                    test_info = None
             elif line.startswith("#EXPECT:") or line.startswith("#EXPECT-NOT:"):
                 if test_info is None:
                     test_info = WarningInfo()
@@ -120,47 +132,64 @@ def extract_test_data(rst_file: Path) -> list[WarningInfo] | None:
                     if line.startswith("#EXPECT:")
                     else test_info.not_expected
                 )
-                target_list.append(extract_warning(line))
+                target_list.append(parse_line_for_message(line))
+            elif line.startswith("#CHECK:"):
+                assert not rst_data.enabled_checks, "only one CHECK per file allowed"
+                rst_data.enabled_checks = parse_line_for_message(line)
         # Check last InfoElement
         if test_info:
-            print("ERROR: Teststatement without according need found")
-        return statements
+            raise AssertionError(
+                "Last EXPECT/EXPECT-NOT line is not followed by a need."
+            )
+        return rst_data
 
 
 def warning_matches(
-    warning_info: WarningInfo, expected_message: str, warnings: list[str]
+    rst_data: RstData,
+    warning_info: WarningInfo,
+    expected_message: str,
+    warnings: list[str],
 ) -> bool:
     ### Checks if any element of the warning list is includes the given warning info.
     # It returns True if found otherwise False.
     for warning in warnings:
         if (
-            f"{warning_info.filename}:{str(warning_info.lineno)}" in warning
+            f"{rst_data.filename}:{str(warning_info.lineno)}" in warning
             and expected_message in warning
         ):
             return True
     return False
 
 
 @pytest.mark.parametrize("rst_file", RST_FILES)
-def test_check_rules(
+def test_rst_files(
     rst_file: str, sphinx_app_setup: Callable[[Path], SphinxTestApp]
 ) -> None:
     ### Test function to check rules in the given rst file
     # The function uses the SphinxTestApp to build the documentation
     # and checks for the expected/unexpected warnings.
-    assert (
-        test_data := extract_test_data(RST_DIR / rst_file)
-    ), "Unable to extract test data"
+    rst_data = extract_test_data(RST_DIR / rst_file)
+    if not rst_data:
+        raise AssertionError(
+            "Unable to extract test data from the rst file: "
+            f"{rst_file}. Please check the file for the correct format."
+        )
+
     app: SphinxTestApp = sphinx_app_setup(RST_DIR / rst_file)
     os.chdir(app.srcdir)  # Change working directory to the source directory
+
+    # Build the documentation with the enabled checks
+    app.config.score_metamodel_checks = rst_data.enabled_checks
     app.build()
+
+    # Collect the warnings
     warnings = app.warning.getvalue().splitlines()
-    for test in test_data:
-        for expected in test.expected:
-            assert warning_matches(
-                test, expected, warnings
-            ), f"Expected warning: {expected} not found"
-        for not_expected in test.not_expected:
-            assert not warning_matches(
-                test, not_expected, warnings
-            ), f"Unexpected warning: {not_expected} found"
+
+    # Check if the expected warnings are present
+    for warning_info in rst_data.warning_infos:
+        for w in warning_info.expected:
+            if not warning_matches(rst_data, warning_info, w, warnings):
+                raise AssertionError(f"Expected warning: '{w}' not found")
+        for w in warning_info.not_expected:
+            if warning_matches(rst_data, warning_info, w, warnings):
+                raise AssertionError(f"Unexpected warning: '{w}' found")