Skip to content

Commit 105f054

Browse files
mmr1909mmr1909
authored
tooling: Add file based rule check tests (#906)
file based unit tests to verify the functionality of the rule checks with test rst files. Add example test rst files for the file-based rule check. Verifies the following checks: attribute_format, check_options, graph_checks.
1 parent 3572002 commit 105f054

9 files changed

Lines changed: 346 additions & 1 deletion

‎tooling/docs/_tooling/docs.bzl‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -149,7 +149,7 @@ def _docs():
149149
"**/*.yaml",
150150
"**/*.json",
151151
"**/*.csv",
152-
]),
152+
], exclude = ["**/tests/rst/**/*.rst"]),
153153
config = ":conf.py",
154154
extra_opts = [
155155
"-W",

‎tooling/docs/_tooling/extensions/score_metamodel/tests/rst/attributes/test_attributes_format_description.rst‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
..
2+
# *******************************************************************************
3+
# Copyright (c) 2025 Contributors to the Eclipse Foundation
4+
#
5+
# See the NOTICE file(s) distributed with this work for additional
6+
# information regarding copyright ownership.
7+
#
8+
# This program and the accompanying materials are made available under the
9+
# terms of the Apache License Version 2.0 which is available at
10+
# https://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# SPDX-License-Identifier: Apache-2.0
13+
# *******************************************************************************
14+
15+
.. Test xy
16+
#EXPECT: stkh_req__test__abcd.content (This should really work): contains a weak word: `really`.
17+
18+
.. stkh_req:: This is a test
19+
:id: stkh_req__test__abcd
20+
21+
This should really work
22+
23+
#EXPECT-NOT: stkh_req__test__abce.title (This should work): contains a weak word
24+
25+
.. stkh_req:: This is a test
26+
:id: stkh_req__test__abce
27+
28+
This should work

‎tooling/docs/_tooling/extensions/score_metamodel/tests/rst/attributes/test_attributes_format_id_format.rst‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
..
2+
# *******************************************************************************
3+
# Copyright (c) 2025 Contributors to the Eclipse Foundation
4+
#
5+
# See the NOTICE file(s) distributed with this work for additional
6+
# information regarding copyright ownership.
7+
#
8+
# This program and the accompanying materials are made available under the
9+
# terms of the Apache License Version 2.0 which is available at
10+
# https://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# SPDX-License-Identifier: Apache-2.0
13+
# *******************************************************************************
14+
15+
#EXPECT: std_wp__test__test__abcd.id (std_wp__test__test__abcd): expected to consisting of this format: `<Req Type>__<Abbreviations>__<Architectural Element>`.
16+
17+
.. std_wp:: This is a test
18+
:id: std_wp__test__test__abcd
19+
20+
#EXPECT-NOT: std_wp__test__abce.id (std_wp__test__abce): expected to consisting of this format: `<Req Type>__<Abbreviations>__<Architectural Element>`.
21+
22+
.. std_wp:: This is a test
23+
:id: std_wp__test__abce
24+
25+
#EXPECT: wp__test__test__abcd.id (wp__test__test__abcd): expected to consisting of one of these 2 formats:`<Req Type>__<Abbreviations>` or `<Req Type>__<Abbreviations>__<Architectural Element>`.
26+
27+
.. std_wp:: This is a test
28+
:id: wp__test__test__abcd
29+
30+
#EXPECT-NOT: wp__test__abce.id (wp__test__abce): expected to consisting of one of these 2 formats:`<Req Type>__<Abbreviations>` or `<Req Type>__<Abbreviations>__<Architectural Element>`.
31+
32+
.. std_wp:: This is a test
33+
:id: wp__test__abce

‎tooling/docs/_tooling/extensions/score_metamodel/tests/rst/attributes/test_attributes_format_id_length.rst‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
..
2+
# *******************************************************************************
3+
# Copyright (c) 2025 Contributors to the Eclipse Foundation
4+
#
5+
# See the NOTICE file(s) distributed with this work for additional
6+
# information regarding copyright ownership.
7+
#
8+
# This program and the accompanying materials are made available under the
9+
# terms of the Apache License Version 2.0 which is available at
10+
# https://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# SPDX-License-Identifier: Apache-2.0
13+
# *******************************************************************************
14+
15+
#EXPECT: std_wp__testabcdefghijklmnopqrstuvwxyz123__abcd.id (std_wp__testabcdefghijklmnopqrstuvwxyz123__abcd): exceeds the maximum allowed length of 45 characters (current length: 47).
16+
17+
.. std_wp:: This is a test
18+
:id: std_wp__testabcdefghijklmnopqrstuvwxyz123__abcd
19+
20+
#EXPECT-NOT: std_wp__test__abce.id (std_wp__test__abce): exceeds the maximum allowed length of 45 characters
21+
22+
.. std_wp:: This is a test
23+
:id: std_wp__test__abce

‎tooling/docs/_tooling/extensions/score_metamodel/tests/rst/attributes/test_attributes_format_title.rst‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
..
2+
# *******************************************************************************
3+
# Copyright (c) 2025 Contributors to the Eclipse Foundation
4+
#
5+
# See the NOTICE file(s) distributed with this work for additional
6+
# information regarding copyright ownership.
7+
#
8+
# This program and the accompanying materials are made available under the
9+
# terms of the Apache License Version 2.0 which is available at
10+
# https://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# SPDX-License-Identifier: Apache-2.0
13+
# *******************************************************************************
14+
15+
#EXPECT: feat_req__test__abcd.title (This must work): contains a stop word: `must`.
16+
17+
.. feat_req:: This must work
18+
:id: feat_req__test__abcd
19+
20+
#EXPECT-NOT: feat_req__test__abce.title (This is a test): contains a stop word
21+
22+
.. feat_req:: This is a test
23+
:id: feat_req__test__abce

‎tooling/docs/_tooling/extensions/score_metamodel/tests/rst/graph/test_metamodel_graph.rst‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
..
2+
# *******************************************************************************
3+
# Copyright (c) 2025 Contributors to the Eclipse Foundation
4+
#
5+
# See the NOTICE file(s) distributed with this work for additional
6+
# information regarding copyright ownership.
7+
#
8+
# This program and the accompanying materials are made available under the
9+
# terms of the Apache License Version 2.0 which is available at
10+
# https://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# SPDX-License-Identifier: Apache-2.0
13+
# *******************************************************************************
14+
15+
.. feat_req:: Parent requirement
16+
:id: feat_req__parent__abcd
17+
:safety: QM
18+
:status: valid
19+
20+
#EXPECT: feat_req__child__abce: parent need `feat_req__parent__abcd` does not fulfill condition `{'and': ['safety != QM', 'status == valid']}`.
21+
22+
.. feat_req:: Child requirement
23+
:id: feat_req__child__abce
24+
:safety: ASIL_B
25+
:status: valid
26+
:satisfies: feat_req__parent__abcd
27+
28+
29+
.. feat_req:: Parent requirement 2
30+
:id: feat_req__parent2__abcd
31+
:safety: ASIL_B
32+
:status: valid
33+
34+
#EXPECT-NOT: feat_req__child2__abce: parent need `feat_req__parent2__abcd` does not fulfill condition
35+
36+
.. feat_req:: Child requirement 2
37+
:id: feat_req__child2__abce
38+
:safety: ASIL_B
39+
:status: valid
40+
:satisfies: feat_req__parent__abcd

‎tooling/docs/_tooling/extensions/score_metamodel/tests/rst/options/test_options_extra_option.rst‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
..
2+
# *******************************************************************************
3+
# Copyright (c) 2025 Contributors to the Eclipse Foundation
4+
#
5+
# See the NOTICE file(s) distributed with this work for additional
6+
# information regarding copyright ownership.
7+
#
8+
# This program and the accompanying materials are made available under the
9+
# terms of the Apache License Version 2.0 which is available at
10+
# https://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# SPDX-License-Identifier: Apache-2.0
13+
# *******************************************************************************
14+
15+
#EXPECT: std_wp__test__abcd: has these extra options: `safety`.
16+
17+
.. std_wp:: This is a test
18+
:id: std_wp__test__abcd
19+
:safety: QM
20+
21+
#EXPECT-NOT: std_wp__test__abce: has these extra options
22+
23+
.. std_wp:: This is a test
24+
:id: std_wp__test__abce

‎tooling/docs/_tooling/extensions/score_metamodel/tests/rst/options/test_options_options.rst‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
..
2+
# *******************************************************************************
3+
# Copyright (c) 2025 Contributors to the Eclipse Foundation
4+
#
5+
# See the NOTICE file(s) distributed with this work for additional
6+
# information regarding copyright ownership.
7+
#
8+
# This program and the accompanying materials are made available under the
9+
# terms of the Apache License Version 2.0 which is available at
10+
# https://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# SPDX-License-Identifier: Apache-2.0
13+
# *******************************************************************************
14+
15+
#EXPECT: std_wp__test__abcd: is missing required option: `status`.
16+
17+
.. std_wp:: This is a test
18+
:id: std_wp__test__abcd
19+
20+
#EXPECT-NOT: std_wp__test__abce: is missing required option: `status`.
21+
22+
.. std_wp:: This is a test
23+
:id: std_wp__test__abce
24+
:status: active

‎tooling/docs/_tooling/extensions/score_metamodel/tests/test_rules_file_based.py‎

Lines changed: 150 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,150 @@
1+
# *******************************************************************************
2+
# Copyright (c) 2025 Contributors to the Eclipse Foundation
3+
#
4+
# See the NOTICE file(s) distributed with this work for additional
5+
# information regarding copyright ownership.
6+
#
7+
# This program and the accompanying materials are made available under the
8+
# terms of the Apache License Version 2.0 which is available at
9+
# https://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# SPDX-License-Identifier: Apache-2.0
12+
# *******************************************************************************
13+
14+
import os
15+
import shutil
16+
from collections.abc import Callable
17+
from dataclasses import dataclass, field
18+
from pathlib import Path
19+
20+
import pytest
21+
from sphinx.testing.util import SphinxTestApp
22+
23+
RST_DIR = Path(__file__).absolute().parent / "rst"
24+
DOCS_DIR = Path(__file__).absolute().parent.parent.parent.parent.parent
25+
TOOLING_DIR_NAME = "_tooling"
26+
27+
### List of relative paths of all rst files in RST_DIR
28+
RST_FILES = [str(f.relative_to(RST_DIR)) for f in Path(RST_DIR).rglob("*.rst")]
29+
30+
31+
@pytest.fixture
32+
def sphinx_base_dir(tmp_path_factory: pytest.TempPathFactory) -> Path:
33+
### Create a temporary directory for Sphinx and copy all necessary files.
34+
base_dir: Path = tmp_path_factory.mktemp("docs")
35+
shutil.copy(DOCS_DIR / "conf.py", base_dir)
36+
shutil.copytree(
37+
DOCS_DIR / TOOLING_DIR_NAME,
38+
base_dir / TOOLING_DIR_NAME,
39+
dirs_exist_ok=True,
40+
ignore=shutil.ignore_patterns("*.rst"),
41+
)
42+
return base_dir
43+
44+
45+
@pytest.fixture
46+
def index_file() -> Callable[[Path], str]:
47+
### Returns a function that creates an index.rst file.
48+
def _create_rst_file(rst_file: Path) -> str:
49+
### returns an index.rst file with a toctree
50+
# that refers to the given rst file.
51+
index_rst: str = f"""
52+
.. toctree::
53+
{rst_file.stem}
54+
"""
55+
return index_rst
56+
57+
return _create_rst_file
58+
59+
60+
@pytest.fixture
61+
def sphinx_app_setup(
62+
sphinx_base_dir: Path, index_file: Callable[[Path], str]
63+
) -> Callable[[Path], SphinxTestApp]:
64+
### Returns a function that creates a SphinxTestApp instance.
65+
def _create_app(rst_file: Path) -> SphinxTestApp:
66+
### Create a SphinxTestApp instance.
67+
# The source directory is set to the temporary directory.
68+
shutil.copy(rst_file, sphinx_base_dir)
69+
index_context: str = index_file(rst_file)
70+
(sphinx_base_dir / "index.rst").write_text(index_context)
71+
app: SphinxTestApp = SphinxTestApp(
72+
freshenv=True,
73+
srcdir=sphinx_base_dir,
74+
outdir=sphinx_base_dir / "out",
75+
buildername="html",
76+
)
77+
return app
78+
79+
return _create_app
80+
81+
82+
@dataclass
83+
class WarningInfo:
84+
#### Class to hold information about warnings
85+
# The class contains the line number and the expected and not expected warnings.
86+
lineno: int = 0
87+
expected: list[str] = field(default_factory=list)
88+
not_expected: list[str] = field(default_factory=list)
89+
90+
91+
def extract_warning(line: str) -> str:
92+
#### Extract the warning message from the line
93+
# The line format is "#EXPECT: <warning message>"
94+
# or "#EXPECT-NOT: <warning message>"
95+
return line.split(": ", 1)[1].strip()
96+
97+
98+
def extract_test_data(rst_file: Path) -> list[WarningInfo] | None:
99+
### Extract test data from the given rst file
100+
# The function returns a list of WarningInfo objects
101+
# containing the line number and the expected and not expected warnings.
102+
# If no test data is found, it returns None.
103+
with open(rst_file) as f:
104+
statements: list[WarningInfo] = []
105+
test_info: WarningInfo | None = None
106+
for no, line in enumerate(f, start=1):
107+
if line.startswith(".. "): # Beginning of new need
108+
if test_info:
109+
test_info.lineno = no
110+
statements.append(test_info)
111+
test_info = None
112+
elif line.startswith("#EXPECT:") or line.startswith("#EXPECT-NOT:"):
113+
if test_info is None:
114+
test_info = WarningInfo()
115+
target_list = (
116+
test_info.expected
117+
if line.startswith("#EXPECT:")
118+
else test_info.not_expected
119+
)
120+
target_list.append(extract_warning(line))
121+
# Check last InfoElement
122+
if test_info:
123+
print("ERROR: Teststatement without according need found")
124+
return statements
125+
126+
127+
@pytest.mark.parametrize("rst_file", RST_FILES)
128+
def test_check_rules(
129+
rst_file: str, sphinx_app_setup: Callable[[Path], SphinxTestApp]
130+
) -> None:
131+
### Test function to check rules in the given rst file
132+
# The function uses the SphinxTestApp to build the documentation
133+
# and checks for the expected/unexpected warnings.
134+
assert (
135+
test_data := extract_test_data(RST_DIR / rst_file)
136+
), "Unable to extract test data"
137+
app: SphinxTestApp = sphinx_app_setup(RST_DIR / rst_file)
138+
os.chdir(app.srcdir) # Change working directory to the source directory
139+
app.build()
140+
warn_text: str = app.warning.getvalue()
141+
for test in test_data:
142+
for expected in test.expected:
143+
assert (
144+
f"{Path(rst_file).name}:{test.lineno}: WARNING: {expected}" in warn_text
145+
), f"Expected warning: {[expected]} not found"
146+
for not_expected in test.not_expected:
147+
assert (
148+
f"{Path(rst_file).name}:{test.lineno}: WARNING: {not_expected}"
149+
not in warn_text
150+
), f"Unexpected warning: {[not_expected]} found"

0 commit comments

Comments
 (0)