store conformance test renders at the module suite root with a two-tier guard

The client now requests module-root-relative output paths (tests in the
functionality's subfolder, shared setup files allowed at the suite root)
and stores render responses relative to the module's conformance tests
folder. Existing-tests context sent to the API now includes shared root
files. A two-tier guard validates every response before storing: files in
other functionalities' subfolders (or required-module copies) and changes
to the definition file are rejected, and existing shared root files may
only be extended by inserting lines (difflib opcode check). One retry on
violation, then the render fails with a clear error.

Author: pedjaradenkovic

codeplain_REST_api.py

@@ -335,6 +335,7 @@ def render_conformance_tests(
             "conformance_tests_json": conformance_tests_json,
             "all_acceptance_tests": all_acceptance_tests,
             "existing_conformance_tests_files": existing_conformance_tests_files,
+            "conformance_tests_paths_relative_to_module_root": True,
         }
 
         response = self.post_request(endpoint_url, headers, payload, run_state)

render_machine/actions/render_conformance_tests.py

@@ -8,11 +8,12 @@
 from render_machine.actions.base_action import BaseAction
 from render_machine.implementation_code_helpers import ImplementationCodeHelpers
 from render_machine.render_context import RenderContext
-from render_machine.render_types import AcceptanceTestPhase, TestExecutionPhase
+from render_machine.render_types import AcceptanceTestPhase, RenderError, TestExecutionPhase
 
 
 class RenderConformanceTests(BaseAction):
     SUCCESSFUL_OUTCOME = "conformance_test_rendered"
+    RESPONSE_VALIDATION_FAILED_OUTCOME = "conformance_test_response_validation_failed"
 
     def execute(self, render_context: RenderContext, _previous_action_payload: Any | None):
         if self._should_render_conformance_tests(render_context):
@@ -123,33 +124,64 @@ def _render_conformance_tests(self, render_context: RenderContext):
             if not file_name.startswith(current_subfolder_prefix)
         }
 
-        response_files, implementation_plan_summary = render_context.codeplain_api.render_conformance_tests(
-            render_context.frid_context.frid,
-            render_context.conformance_tests_running_context.current_testing_frid,
-            render_context.plain_source_tree,
-            render_context.frid_context.linked_resources,
-            existing_files_content,
-            memory_files_content,
-            render_context.module_name,
-            render_context.get_required_modules_functionalities(),
-            conformance_tests_folder_name,
-            render_context.conformance_tests_running_context.get_conformance_tests_json(
-                render_context.conformance_tests_running_context.current_testing_module_name
-            ),
-            all_acceptance_tests,
-            existing_conformance_tests_files,
-            run_state=render_context.run_state,
+        current_subfolder_name = os.path.basename(conformance_tests_folder_name)
+        module_conformance_tests_folder = render_context.conformance_tests.get_module_conformance_tests_folder(
+            render_context.module_name
         )
 
+        for attempt in range(2):
+            response_files, implementation_plan_summary = render_context.codeplain_api.render_conformance_tests(
+                render_context.frid_context.frid,
+                render_context.conformance_tests_running_context.current_testing_frid,
+                render_context.plain_source_tree,
+                render_context.frid_context.linked_resources,
+                existing_files_content,
+                memory_files_content,
+                render_context.module_name,
+                render_context.get_required_modules_functionalities(),
+                conformance_tests_folder_name,
+                render_context.conformance_tests_running_context.get_conformance_tests_json(
+                    render_context.conformance_tests_running_context.current_testing_module_name
+                ),
+                all_acceptance_tests,
+                existing_conformance_tests_files,
+                run_state=render_context.run_state,
+            )
+
+            violations = render_context.conformance_tests.find_response_file_violations(
+                render_context.module_name,
+                current_subfolder_name,
+                response_files,
+            )
+
+            if not violations:
+                break
+
+            console.warning(
+                "Generated conformance test files violate the suite layout rules:\n  "
+                + "\n  ".join(violations)
+                + ("\nRetrying the generation." if attempt == 0 else "")
+            )
+        else:
+            return (
+                self.RESPONSE_VALIDATION_FAILED_OUTCOME,
+                RenderError.encode(
+                    message="Generated conformance test files repeatedly violated the suite layout rules "
+                    "(files outside the functionality's subfolder or invalid changes to shared setup files).",
+                    error_type="CONFORMANCE_TESTS_VALIDATION_ERROR",
+                    violations="\n".join(violations),
+                ).to_payload(),
+            )
+
         render_context.conformance_tests_running_context.current_testing_frid_high_level_implementation_plan = (
             implementation_plan_summary
         )
 
-        file_utils.store_response_files(conformance_tests_folder_name, response_files, [])
+        file_utils.store_response_files(module_conformance_tests_folder, response_files, [])
 
         console.print_files(
             "Conformance test files generated:",
-            conformance_tests_folder_name,
+            module_conformance_tests_folder,
             response_files,
             style=console.OUTPUT_STYLE,
         )

render_machine/conformance_tests.py

@@ -1,3 +1,4 @@
+import difflib
 import json
 import os
 
@@ -9,6 +10,18 @@
 CONFORMANCE_TESTS_DEFINITION_FILE_NAME = "conformance_tests.json"
 
 
+def _is_insertion_only_change(existing_content: str, new_content: str) -> bool:
+    """Check that new_content only inserts lines into existing_content.
+
+    Every existing line must survive, in order - the diff opcodes may contain only
+    "equal" and "insert" operations.
+    """
+    matcher = difflib.SequenceMatcher(
+        None, existing_content.splitlines(keepends=True), new_content.splitlines(keepends=True), autojunk=False
+    )
+    return all(opcode in ("equal", "insert") for opcode, *_ in matcher.get_opcodes())
+
+
 class ConformanceTests:
     """Manages the state of conformance tests."""
 
@@ -177,9 +190,10 @@ def fetch_all_existing_conformance_test_files(self, module_name: str) -> dict[st
 
         Files are collected from every conformance test subfolder of the module (one subfolder
         per functional requirement) and keyed as "<subfolder>/<relative file path>" so each
-        file's suite remains identifiable. Hidden subfolders (copies of required modules' tests)
-        and the conformance tests definition file (stored at the module folder root) are not
-        included. Returns an empty dict when the module has no conformance tests yet.
+        file's suite remains identifiable, plus any shared setup files at the module folder
+        root (keyed by their bare file name). Hidden subfolders (copies of required modules'
+        tests) and the conformance tests definition file are not included. Returns an empty
+        dict when the module has no conformance tests yet.
         """
         all_files_content: dict[str, str] = {}
         module_folder = self.get_module_conformance_tests_folder(module_name)
@@ -190,8 +204,66 @@ def fetch_all_existing_conformance_test_files(self, module_name: str) -> dict[st
             for file_name, content in files_content.items():
                 all_files_content[os.path.join(folder_name, file_name)] = content
 
+        if os.path.isdir(module_folder):
+            root_file_names = [
+                entry.name
+                for entry in os.scandir(module_folder)
+                if entry.is_file() and entry.name != self.conformance_tests_definition_file_name
+            ]
+            root_files_content = file_utils.get_existing_files_content(module_folder, sorted(root_file_names))
+            all_files_content.update(root_files_content)
+
         return all_files_content
 
+    def find_response_file_violations(
+        self,
+        module_name: str,
+        current_subfolder_name: str,
+        response_files: dict[str, str],
+    ) -> list[str]:
+        """Check a conformance tests render response against the two-tier immutability rule.
+
+        Paths are relative to the module's conformance tests folder. Allowed: any file under
+        the current functionality's subfolder, new files anywhere outside other
+        functionalities' subfolders, and extensions of existing shared setup files at the
+        suite root that only insert lines. Violations: files in other functionalities'
+        subfolders (or required-module copies), the conformance tests definition file, and
+        root-file changes that delete or modify existing lines.
+        """
+        violations = []
+        module_folder = self.get_module_conformance_tests_folder(module_name)
+        known_suite_folders = set(self.fetch_existing_conformance_test_folder_names(module_name))
+
+        for file_name, content in response_files.items():
+            path_parts = file_name.replace(os.sep, "/").split("/")
+            top_level_name = path_parts[0]
+
+            if top_level_name == current_subfolder_name:
+                continue
+
+            if len(path_parts) > 1:
+                if top_level_name.startswith("."):
+                    violations.append(f"{file_name}: files of required modules' test copies must not be changed")
+                elif top_level_name in known_suite_folders:
+                    violations.append(f"{file_name}: belongs to another functionality's test suite")
+                continue
+
+            if file_name == self.conformance_tests_definition_file_name:
+                violations.append(f"{file_name}: the conformance tests definition file must not be changed")
+                continue
+
+            existing_file_path = os.path.join(module_folder, file_name)
+            if os.path.exists(existing_file_path):
+                with open(existing_file_path, "r") as f:
+                    existing_content = f.read()
+                if content is None or not _is_insertion_only_change(existing_content, content):
+                    violations.append(
+                        f"{file_name}: shared setup files may only be extended by adding lines - "
+                        "existing lines must not be changed or removed"
+                    )
+
+        return violations
+
     def fetch_existing_conformance_test_files(
         self,
         module_name: str,

render_machine/state_machine_config.py

@@ -85,6 +85,7 @@ def get_action_result_triggers_map(self) -> Dict[str, str]:
             FinishFunctionalRequirement.SUCCESSFUL_OUTCOME: triggers.PROCEED_FRID_PROCESSING,
             CreateDist.SUCCESSFUL_OUTCOME: triggers.FINISH_RENDER,
             RenderConformanceTests.SUCCESSFUL_OUTCOME: triggers.MARK_CONFORMANCE_TESTS_READY,
+            RenderConformanceTests.RESPONSE_VALIDATION_FAILED_OUTCOME: triggers.HANDLE_ERROR,
             PrepareTestingEnvironment.SUCCESSFUL_OUTCOME: triggers.MARK_TESTING_ENVIRONMENT_PREPARED,
             PrepareTestingEnvironment.FAILED_OUTCOME: triggers.HANDLE_ERROR,
             RunConformanceTests.SUCCESSFUL_OUTCOME: triggers.MOVE_TO_NEXT_CONFORMANCE_TEST,

tests/test_conformance_tests.py

@@ -106,3 +106,77 @@ def test_get_module_suite_run_folder_required_module_without_copy(conformance_te
     )
 
     assert folder == os.path.join(conformance_tests_dir, "required_module")
+
+
+def test_fetch_all_existing_conformance_test_files_includes_shared_root_files(conformance_tests, conformance_tests_dir):
+    module_folder = os.path.join(conformance_tests_dir, MODULE_NAME)
+    _write_file(module_folder, os.path.join("some_functionality", "test_some.py"), "some test")
+    _write_file(module_folder, "shared_helpers.py", "shared helper")
+    _write_file(module_folder, CONFORMANCE_TESTS_DEFINITION_FILE_NAME, json.dumps({}))
+
+    files_content = conformance_tests.fetch_all_existing_conformance_test_files(MODULE_NAME)
+
+    assert files_content == {
+        os.path.join("some_functionality", "test_some.py"): "some test",
+        "shared_helpers.py": "shared helper",
+    }
+
+
+def test_find_response_file_violations_allows_own_subfolder_and_new_root_files(
+    conformance_tests, conformance_tests_dir
+):
+    module_folder = os.path.join(conformance_tests_dir, MODULE_NAME)
+    _write_file(module_folder, os.path.join("earlier_suite", "test_earlier.py"), "earlier")
+
+    violations = conformance_tests.find_response_file_violations(
+        MODULE_NAME,
+        "current_suite",
+        {
+            os.path.join("current_suite", "test_new.py"): "new test",
+            "shared_helpers.py": "brand new helper",
+            os.path.join("support", "util.py"): "new shared dir file",
+        },
+    )
+
+    assert violations == []
+
+
+def test_find_response_file_violations_rejects_other_suites_and_hidden_copies(conformance_tests, conformance_tests_dir):
+    module_folder = os.path.join(conformance_tests_dir, MODULE_NAME)
+    _write_file(module_folder, os.path.join("earlier_suite", "test_earlier.py"), "earlier")
+
+    violations = conformance_tests.find_response_file_violations(
+        MODULE_NAME,
+        "current_suite",
+        {
+            os.path.join("earlier_suite", "test_earlier.py"): "rewritten",
+            os.path.join(".required_module", "suite", "test_x.py"): "copied",
+            CONFORMANCE_TESTS_DEFINITION_FILE_NAME: "{}",
+        },
+    )
+
+    assert len(violations) == 3
+
+
+def test_find_response_file_violations_shared_root_file_insertion_only(conformance_tests, conformance_tests_dir):
+    module_folder = os.path.join(conformance_tests_dir, MODULE_NAME)
+    _write_file(module_folder, "shared_helpers.py", "line one\nline two\n")
+
+    extended = "line one\nnew line in between\nline two\nnew line at end\n"
+    assert (
+        conformance_tests.find_response_file_violations(MODULE_NAME, "current_suite", {"shared_helpers.py": extended})
+        == []
+    )
+
+    modified = "line one CHANGED\nline two\n"
+    violations = conformance_tests.find_response_file_violations(
+        MODULE_NAME, "current_suite", {"shared_helpers.py": modified}
+    )
+    assert len(violations) == 1
+    assert "adding lines" in violations[0]
+
+    truncated = "line one\n"
+    violations = conformance_tests.find_response_file_violations(
+        MODULE_NAME, "current_suite", {"shared_helpers.py": truncated}
+    )
+    assert len(violations) == 1