tooling: Add new test files for check rule tests (#980)

* Add new test files
* Comment test files
* Optimize check of enabled checks

Author: mmr1909

tooling/docs/_tooling/extensions/score_metamodel/__init__.py

@@ -86,10 +86,12 @@ def _run_checks(app: Sphinx, exception: Exception | None) -> None:
     def is_check_enabled(check: local_check_function | graph_check_function):
         return not checks_filter or check.__name__ in checks_filter
 
+    enabled_local_checks = [c for c in local_checks if is_check_enabled(c)]
+
     # Need-Local checks: checks which can be checked file-local, without a
     # graph of other needs.
     for need in needs_all_needs.values():
-        for check in [c for c in local_checks if is_check_enabled(c)]:
+        for check in enabled_local_checks:
             logger.info(f"Running local check {check} for need {need['id']}")
             check(app, need, log)
 

tooling/docs/_tooling/extensions/score_metamodel/tests/rst/attributes/test_attributes_format_description.rst

@@ -11,18 +11,28 @@
    #
    # SPDX-License-Identifier: Apache-2.0
    # *******************************************************************************
+#CHECK: check_description
 
-.. Test xy
+.. Description contains a weak word
 #EXPECT: stkh_req__test__abcd.content (This should really work): contains a weak word: `really`.
 
 .. stkh_req:: This is a test
    :id: stkh_req__test__abcd
 
    This should really work
 
-#EXPECT-NOT: stkh_req__test__abce.title (This should work): contains a weak word
+.. Description contains no weak word
+#EXPECT-NOT: contains a weak word
 
 .. stkh_req:: This is a test
    :id: stkh_req__test__abce
 
    This should work
+
+.. Description of requirement of type std_wp is not checked for weak words
+#EXPECT-NOT: contains a weak word
+
+.. std_wp:: This is a test
+   :id: stdwp__test__abce
+
+   This should really work

tooling/docs/_tooling/extensions/score_metamodel/tests/rst/attributes/test_attributes_format_id_format.rst

@@ -11,23 +11,28 @@
    #
    # SPDX-License-Identifier: Apache-2.0
    # *******************************************************************************
+#CHECK: check_id_format
 
+.. Id does not consists of 3 parts
 #EXPECT: std_wp__test__test__abcd.id (std_wp__test__test__abcd): expected to consisting of this format: `<Req Type>__<Abbreviations>__<Architectural Element>`.
 
 .. std_wp:: This is a test
    :id: std_wp__test__test__abcd
 
-#EXPECT-NOT: std_wp__test__abce.id (std_wp__test__abce): expected to consisting of this format: `<Req Type>__<Abbreviations>__<Architectural Element>`.
+.. Id follows pattern
+#EXPECT-NOT: expected to consisting of this format: `<Req Type>__<Abbreviations>__<Architectural Element>`.
 
 .. std_wp:: This is a test
    :id: std_wp__test__abce
 
+.. Id starts with wp and number of parth is neither 2 nor 3
 #EXPECT: wp__test__test__abcd.id (wp__test__test__abcd): expected to consisting of one of these 2 formats:`<Req Type>__<Abbreviations>` or `<Req Type>__<Abbreviations>__<Architectural Element>`.
 
 .. std_wp:: This is a test
    :id: wp__test__test__abcd
 
-#EXPECT-NOT: wp__test__abce.id (wp__test__abce): expected to consisting of one of these 2 formats:`<Req Type>__<Abbreviations>` or `<Req Type>__<Abbreviations>__<Architectural Element>`.
+.. Id is valid, because it starts with wp and contains 3 parts
+#EXPECT-NOT: expected to consisting of one of these 2 formats:`<Req Type>__<Abbreviations>` or `<Req Type>__<Abbreviations>__<Architectural Element>`.
 
 .. std_wp:: This is a test
    :id: wp__test__abce

tooling/docs/_tooling/extensions/score_metamodel/tests/rst/attributes/test_attributes_format_id_length.rst

@@ -11,13 +11,16 @@
    #
    # SPDX-License-Identifier: Apache-2.0
    # *******************************************************************************
+#CHECK: check_id_length
 
+.. Id contains too many characters
 #EXPECT: std_wp__testabcdefghijklmnopqrstuvwxyz123__abcd.id (std_wp__testabcdefghijklmnopqrstuvwxyz123__abcd): exceeds the maximum allowed length of 45 characters (current length: 47).
 
 .. std_wp:: This is a test
    :id: std_wp__testabcdefghijklmnopqrstuvwxyz123__abcd
 
-#EXPECT-NOT: std_wp__test__abce.id (std_wp__test__abce): exceeds the maximum allowed length of 45 characters
+.. Id has correct length
+#EXPECT-NOT: exceeds the maximum allowed length of 45 characters
 
 .. std_wp:: This is a test
    :id: std_wp__test__abce

tooling/docs/_tooling/extensions/score_metamodel/tests/rst/attributes/test_attributes_format_title.rst

@@ -11,13 +11,22 @@
    #
    # SPDX-License-Identifier: Apache-2.0
    # *******************************************************************************
+#CHECK: check_title
 
+.. Title contains a stop word
 #EXPECT: feat_req__test__abcd.title (This must work): contains a stop word: `must`.
 
 .. feat_req:: This must work
    :id: feat_req__test__abcd
 
+.. Title contains no stop word
 #EXPECT-NOT: feat_req__test__abce.title (This is a test): contains a stop word
 
 .. feat_req:: This is a test
    :id: feat_req__test__abce
+
+.. Title of requirement of type std_wp is not checked for stop words
+#EXPECT-NOT: std_wp__test__abce.title (This must work): contains a stop word: `must`.
+
+.. std_wp:: This must work
+   :id: std_wp__test__abce

tooling/docs/_tooling/extensions/score_metamodel/tests/rst/graph/test_metamodel_graph.rst

@@ -18,6 +18,7 @@
    :safety: QM
    :status: valid
 
+.. Parent requirement has not the correct safety level
 #EXPECT: feat_req__child__abce: parent need `feat_req__parent__abcd` does not fulfill condition `{'and': ['safety != QM', 'status == valid']}`.
 
 .. feat_req:: Child requirement
@@ -26,16 +27,37 @@
    :status: valid
    :satisfies: feat_req__parent__abcd
 
-
 .. feat_req:: Parent requirement 2
    :id: feat_req__parent2__abcd
    :safety: ASIL_B
    :status: valid
 
+.. Parent requirement has the correct safety level
 #EXPECT-NOT: feat_req__child2__abce: parent need `feat_req__parent2__abcd` does not fulfill condition
 
 .. feat_req:: Child requirement 2
    :id: feat_req__child2__abce
    :safety: ASIL_B
    :status: valid
    :satisfies: feat_req__parent__abcd
+
+.. Parent requirement does not exist
+#EXPECT: feat_req__child3__abce: Parent need `feat_req__parent0__abcd` not found in needs_dict.
+
+.. feat_req:: Child requirement 3
+   :id: feat_req__child3__abce
+   :safety: ASIL_B
+   :status: valid
+   :satisfies: feat_req__parent0__abcd
+
+.. feat_req:: Parent requirement 3
+   :id: feat_req__parent3__abcd
+   :status: invalid
+
+.. Graph check without combined condition (no and or or)
+#EXPECT: comp_req__parent4__abcd: parent need `feat_req__parent3__abcd` does not fulfill condition `status == valid`.
+
+.. comp_req:: Child requirement 4
+   :id: comp_req__parent4__abcd
+   :status: valid
+   :satisfies: feat_req__parent3__abcd

tooling/docs/_tooling/extensions/score_metamodel/tests/rst/id_contains_feature/test_id_contains_feature.rst

@@ -0,0 +1,40 @@
+..
+   # *******************************************************************************
+   # Copyright (c) 2025 Contributors to the Eclipse Foundation
+   #
+   # See the NOTICE file(s) distributed with this work for additional
+   # information regarding copyright ownership.
+   #
+   # This program and the accompanying materials are made available under the
+   # terms of the Apache License Version 2.0 which is available at
+   # https://www.apache.org/licenses/LICENSE-2.0
+   #
+   # SPDX-License-Identifier: Apache-2.0
+   # *******************************************************************************
+#CHECK: id_contains_feature
+
+.. Feature is not in the path of the RST file
+#EXPECT: std_wp__test__abcd.id (std_wp__test__abcd): Feature 'test' not in path
+
+.. std_wp:: This is a test
+   :id: std_wp__test__abcd
+
+.. Feature is in the path of the RST file
+#EXPECT-NOT: Feature 'id_contains_feature' not in path
+
+.. std_wp:: This is a test
+   :id: std_wp__id_contains_feature__abce
+
+.. Check if the feature is in the path of the RST file is skipped,
+   because the id contains 4 parts
+#EXPECT-NOT: not in path
+
+.. std_wp:: This is a test
+   :id: std_wp__test1__test2__abce
+
+.. Check if the feature is in the path of the RST file is skipped,
+   because the requirement type is stkh_req
+#EXPECT-NOT: Feature 'test' not in path
+
+.. stkh_req:: This is a test
+   :id: stkh_req__test__abce

tooling/docs/_tooling/extensions/score_metamodel/tests/rst/options/test_options_extra_option.rst

@@ -13,13 +13,15 @@
    # *******************************************************************************
 #CHECK: check_extra_options
 
+.. Invalid option: `safety` is not allowed
 #EXPECT: std_wp__test__abcd: has these extra options: `safety`.
 
 .. std_wp:: This is a test
    :id: std_wp__test__abcd
    :safety: QM
 
-#EXPECT-NOT: std_wp__test__abce: has these extra options
+.. No invalid extra options are present
+#EXPECT-NOT: has these extra options
 
 .. std_wp:: This is a test
    :id: std_wp__test__abce

tooling/docs/_tooling/extensions/score_metamodel/tests/rst/options/test_options_options.rst

@@ -13,13 +13,22 @@
    # *******************************************************************************
 #CHECK: check_options
 
+.. Required option: `status` is missing
 #EXPECT: std_wp__test__abcd: is missing required option: `status`.
 
 .. std_wp:: This is a test
    :id: std_wp__test__abcd
 
-#EXPECT-NOT: std_wp__test__abce: is missing required option: `status`.
+.. All required options are present
+#EXPECT-NOT: is missing required option
 
 .. std_wp:: This is a test
    :id: std_wp__test__abce
    :status: active
+
+.. Satisfies link refers to wrong requirement type
+#EXPECT: feat_req__abce.satisfies (['std_wp__test__abce']): does not follow pattern `^stkh_req__.*$`.
+
+.. feat_req:: Child requirement
+   :id: feat_req__abce
+   :satisfies: std_wp__test__abce

tooling/docs/_tooling/extensions/score_metamodel/tests/test_rules_file_based.py

@@ -55,7 +55,7 @@ def _create_rst_file(rst_file: Path) -> str:
         # that refers to the given rst file.
         index_rst: str = f"""
 .. toctree::
-   {rst_file.stem}
+   {rst_file.relative_to(RST_DIR)}
 """
         return index_rst
 
@@ -70,7 +70,11 @@ def sphinx_app_setup(
     def _create_app(rst_file: Path) -> SphinxTestApp:
         ### Create a SphinxTestApp instance.
         # The source directory is set to the temporary directory.
-        shutil.copy(rst_file, sphinx_base_dir)
+        # Create folder structure of rst file in the temporary directory.
+        rst_folder = sphinx_base_dir / rst_file.parent.relative_to(RST_DIR)
+        rst_folder.mkdir(parents=True, exist_ok=True)
+        # Copy the rst file to the temporary directory.
+        shutil.copy(str(rst_file), rst_folder)
         index_context: str = index_file(rst_file)
         (sphinx_base_dir / "index.rst").write_text(index_context)
         app: SphinxTestApp = SphinxTestApp(
@@ -174,7 +178,7 @@ def test_rst_files(
             "Unable to extract test data from the rst file: "
             f"{rst_file}. Please check the file for the correct format."
         )
-
+    print(f"RST Data: {rst_data}")
     app: SphinxTestApp = sphinx_app_setup(RST_DIR / rst_file)
     os.chdir(app.srcdir)  # Change working directory to the source directory
 
@@ -184,6 +188,7 @@ def test_rst_files(
 
     # Collect the warnings
     warnings = app.warning.getvalue().splitlines()
+    print(f"Warnings: {warnings}")
 
     # Check if the expected warnings are present
     for warning_info in rst_data.warning_infos: