Hand the conformance tests fix to the unit tests fixer

When the conformance tests fixer changes implementation code, the unit
tests are re-run and, if they fail, fixed next. The unit tests fixer had
no idea the implementation was just deliberately changed, so it repaired
the implementation back to what the stale unit tests asserted. The
conformance fixer then re-applied the same change, and the render looped.

The conformance tests running context now records every implementation
change the conformance fixer made (hypothesis, approach and per-file
diff). FixUnitTests sends that list as conformance_tests_fixes with the
/fix_unittests_issue request, so the fixer preserves the change and
adjusts the unit tests instead. The field is only sent when unit tests
are processed inside the conformance phase.

Author: VitjanZ

codeplain_REST_api.py

@@ -237,6 +237,7 @@ def fix_unittests_issue(
         required_modules,
         unittests_issue,
         run_state: RunState,
+        conformance_tests_fixes: list[dict] | None = None,
     ):
         endpoint_url = f"{self.api_url}/fix_unittests_issue"
         headers = {"X-API-Key": self.api_key, "Content-Type": "application/json"}
@@ -251,6 +252,10 @@ def fix_unittests_issue(
             "unittests_issue": unittests_issue,
             "unittest_batch_id": run_state.unittest_batch_id,
         }
+        # Implementation code changes made by the conformance tests fixer right before this unit tests run;
+        # only sent when unit tests are processed inside the conformance tests phase.
+        if conformance_tests_fixes is not None:
+            payload["conformance_tests_fixes"] = conformance_tests_fixes
 
         return self.post_request(endpoint_url, headers, payload, run_state)
 

render_machine/actions/fix_conformance_test.py

@@ -189,6 +189,17 @@ def execute(self, render_context: RenderContext, previous_action_payload: Any |
                 ctx.test_that_triggered_code_change = (ctx.current_testing_module_name, ctx.current_testing_frid)
                 ctx.execution_phase = TestExecutionPhase.RETRYING_AFTER_CODE_CHANGE
 
+                # Remember the change so that the unit tests fixer (which runs next) preserves it instead of
+                # reverting it - see FixUnitTests.
+                summary = fix_attempt_summary if isinstance(fix_attempt_summary, dict) else {}
+                ctx.implementation_code_fixes.append(
+                    {
+                        "hypothesis": summary.get("hypothesis"),
+                        "approach": summary.get("approach"),
+                        "code_diff": code_diff_files_content,
+                    }
+                )
+
                 journal.record_fix(
                     ctx.current_testing_module_name,
                     ctx.current_testing_frid,

render_machine/actions/fix_unit_tests.py

@@ -32,6 +32,13 @@ def execute(self, render_context: RenderContext, previous_action_payload: Any |
 
         render_utils.print_inputs(render_context, existing_files_content, "Files sent as input to unit tests fixing:")
 
+        conformance_tests_fixes = self._get_conformance_tests_fixes(render_context)
+        if conformance_tests_fixes:
+            console.info(
+                f"Unit tests are fixed while preserving {len(conformance_tests_fixes)} implementation code change(s) "
+                "made to fix the conformance tests."
+            )
+
         response_files = render_context.codeplain_api.fix_unittests_issue(
             render_context.frid_context.frid,
             render_context.plain_source_tree,
@@ -41,6 +48,7 @@ def execute(self, render_context: RenderContext, previous_action_payload: Any |
             render_context.get_required_modules_functionalities(),
             previous_unittests_issue,
             run_state=render_context.run_state,
+            conformance_tests_fixes=conformance_tests_fixes,
         )
 
         _, changed_files = file_utils.update_build_folder_with_rendered_files(
@@ -52,3 +60,20 @@ def execute(self, render_context: RenderContext, previous_action_payload: Any |
         console.print_files("Files fixed:", render_context.build_folder, response_files, style=console.OUTPUT_STYLE)
 
         return self.SUCCESSFUL_OUTCOME, None
+
+    @staticmethod
+    def _get_conformance_tests_fixes(render_context: RenderContext) -> list[dict] | None:
+        """Implementation code changes the conformance tests fixer made before these unit tests were run.
+
+        Only present when unit tests are processed inside the conformance tests phase - the implementation
+        and refactoring unit test passes have no conformance tests running context.
+        """
+        conformance_tests_running_context = getattr(render_context, "conformance_tests_running_context", None)
+        if conformance_tests_running_context is None:
+            return None
+
+        implementation_code_fixes = getattr(conformance_tests_running_context, "implementation_code_fixes", None)
+        if not implementation_code_fixes:
+            return None
+
+        return list(implementation_code_fixes)

render_machine/render_types.py

@@ -91,6 +91,11 @@ def __init__(
 
         self.current_testing_frid_high_level_implementation_plan: Optional[str] = None
 
+        # Implementation code changes made by the conformance tests fixer during this conformance phase, in
+        # order. Each entry is {"hypothesis": str | None, "approach": str | None, "code_diff": {file: diff}}.
+        # Handed to the unit tests fixer so it adjusts the unit tests instead of reverting these changes.
+        self.implementation_code_fixes: list[dict] = []
+
     def get_conformance_tests_json(self, module_name: str) -> dict:
         return self._conformance_tests_json[module_name]
 

tests/test_fix_unit_tests_conformance_context.py

@@ -0,0 +1,230 @@
+"""Tests for handing the conformance tests fixer's implementation changes to the unit tests fixer.
+
+When the conformance tests fixer changes implementation code, the unit tests are re-run and fixed next.
+The unit tests fixer must know about that change so it adjusts the unit tests instead of reverting it.
+"""
+
+import os
+import tempfile
+from types import SimpleNamespace
+
+import pytest
+
+import plain_spec
+from memory_management import MemoryManager
+from render_machine.actions import fix_unit_tests as fix_unit_tests_module
+from render_machine.actions.fix_conformance_test import FixConformanceTest
+from render_machine.actions.fix_unit_tests import FixUnitTests
+from render_machine.implementation_code_helpers import ImplementationCodeHelpers
+from render_machine.render_types import ConformanceTestsRunningContext, UnitTestsRunningContext
+
+
+class FakeCodeplainAPI:
+    def __init__(self, conformance_fix_response=None, unittests_fix_response=None):
+        self.conformance_fix_response = conformance_fix_response
+        self.unittests_fix_response = unittests_fix_response or {}
+        self.unittests_fix_calls = []
+
+    def fix_conformance_tests_issue(self, *args, **kwargs):
+        return self.conformance_fix_response
+
+    def fix_unittests_issue(self, *args, **kwargs):
+        self.unittests_fix_calls.append(kwargs)
+        return self.unittests_fix_response
+
+
+class FakeConformanceTests:
+    def fetch_existing_conformance_test_files(self, *args):
+        return {}, {}
+
+    def store_conformance_tests_files(self, *args):
+        pass
+
+
+@pytest.fixture
+def build_folder():
+    with tempfile.TemporaryDirectory() as folder:
+        with open(os.path.join(folder, "app.py"), "w", encoding="utf-8") as source_file:
+            source_file.write("def add(a, b):\n    return a + b\n")
+        yield folder
+
+
+@pytest.fixture
+def memory_folder():
+    with tempfile.TemporaryDirectory() as folder:
+        yield folder
+
+
+@pytest.fixture(autouse=True)
+def isolate_from_git_and_console(monkeypatch):
+    monkeypatch.setattr(ImplementationCodeHelpers, "get_code_diff", staticmethod(lambda *args: {}))
+    monkeypatch.setattr(plain_spec, "collect_linked_resources", lambda *args: None)
+    monkeypatch.setattr(fix_unit_tests_module.render_utils, "print_inputs", lambda *args: None)
+
+
+def make_conformance_context():
+    return ConformanceTestsRunningContext(
+        current_testing_module_name="mod",
+        current_testing_frid="1",
+        fix_attempts=0,
+        conformance_tests_json={"1": {"folder_name": "conformance_1"}},
+        conformance_tests_render_attempts=0,
+        current_testing_frid_specifications={},
+        should_prepare_testing_environment=False,
+        frid_being_implemented="1",
+    )
+
+
+def make_render_context(api, build_folder, memory_folder, conformance_tests_running_context):
+    return SimpleNamespace(
+        codeplain_api=api,
+        build_folder=build_folder,
+        memory_manager=MemoryManager(api, memory_folder),
+        conformance_tests=FakeConformanceTests(),
+        conformance_tests_running_context=conformance_tests_running_context,
+        unit_tests_running_context=UnitTestsRunningContext(fix_attempts=1),
+        plain_source_tree={},
+        module_name="mod",
+        required_modules=None,
+        frid_context=SimpleNamespace(frid="1", linked_resources={}),
+        get_required_modules_functionalities=lambda: {},
+        run_state=SimpleNamespace(render_id="test-render-id", unittest_batch_id=1),
+    )
+
+
+def run_conformance_fix(render_context):
+    return FixConformanceTest().execute(render_context, {"previous_conformance_tests_issue": "tests failed"})
+
+
+def test_conformance_fix_of_implementation_code_is_remembered(build_folder, memory_folder):
+    api = FakeCodeplainAPI(
+        conformance_fix_response=[
+            FixConformanceTest.ISSUE_REASON_CODE_IMPLEMENTATION_CODE,
+            {"app.py": "def add(a, b):\n    return a + b + 1\n"},
+            {"hypothesis": "off by one", "approach": "add one"},
+            "prepared issue",
+        ]
+    )
+    ctx = make_conformance_context()
+    render_context = make_render_context(api, build_folder, memory_folder, ctx)
+
+    outcome, _ = run_conformance_fix(render_context)
+
+    assert outcome == FixConformanceTest.IMPLEMENTATION_CODE_UPDATED
+    assert len(ctx.implementation_code_fixes) == 1
+    recorded_fix = ctx.implementation_code_fixes[0]
+    assert recorded_fix["hypothesis"] == "off by one"
+    assert recorded_fix["approach"] == "add one"
+    assert list(recorded_fix["code_diff"]) == ["app.py"]
+    assert "+    return a + b + 1" in recorded_fix["code_diff"]["app.py"]
+
+
+def test_consecutive_implementation_fixes_accumulate(build_folder, memory_folder):
+    ctx = make_conformance_context()
+    for attempt in range(2):
+        api = FakeCodeplainAPI(
+            conformance_fix_response=[
+                FixConformanceTest.ISSUE_REASON_CODE_IMPLEMENTATION_CODE,
+                {"app.py": f"def add(a, b):\n    return a + b + {attempt + 1}\n"},
+                {"hypothesis": f"hypothesis {attempt}", "approach": f"approach {attempt}"},
+                "prepared issue",
+            ]
+        )
+        render_context = make_render_context(api, build_folder, memory_folder, ctx)
+        run_conformance_fix(render_context)
+
+    assert [fix["hypothesis"] for fix in ctx.implementation_code_fixes] == ["hypothesis 0", "hypothesis 1"]
+
+
+def test_conformance_fix_without_summary_is_still_remembered(build_folder, memory_folder):
+    api = FakeCodeplainAPI(
+        conformance_fix_response=[
+            FixConformanceTest.ISSUE_REASON_CODE_IMPLEMENTATION_CODE,
+            {"app.py": "def add(a, b):\n    return a + b + 1\n"},
+            None,
+            "prepared issue",
+        ]
+    )
+    ctx = make_conformance_context()
+    render_context = make_render_context(api, build_folder, memory_folder, ctx)
+
+    run_conformance_fix(render_context)
+
+    assert ctx.implementation_code_fixes[0]["hypothesis"] is None
+    assert ctx.implementation_code_fixes[0]["approach"] is None
+    assert "app.py" in ctx.implementation_code_fixes[0]["code_diff"]
+
+
+def test_conformance_fix_of_conformance_tests_is_not_remembered(build_folder, memory_folder):
+    api = FakeCodeplainAPI(
+        conformance_fix_response=[
+            FixConformanceTest.ISSUE_REASON_CODE_CONFORMANCE_TESTS,
+            {"test_app.py": "def test_add(): pass\n"},
+            {"hypothesis": "wrong assertion", "approach": "fix test"},
+            "prepared issue",
+        ]
+    )
+    ctx = make_conformance_context()
+    render_context = make_render_context(api, build_folder, memory_folder, ctx)
+
+    outcome, _ = run_conformance_fix(render_context)
+
+    assert outcome == FixConformanceTest.IMPLEMENTATION_CODE_NOT_UPDATED
+    assert ctx.implementation_code_fixes == []
+
+
+def test_implementation_fix_with_no_files_is_not_remembered(build_folder, memory_folder):
+    api = FakeCodeplainAPI(
+        conformance_fix_response=[
+            FixConformanceTest.ISSUE_REASON_CODE_IMPLEMENTATION_CODE,
+            {},
+            {"hypothesis": "nothing", "approach": "nothing"},
+            "prepared issue",
+        ]
+    )
+    ctx = make_conformance_context()
+    render_context = make_render_context(api, build_folder, memory_folder, ctx)
+
+    outcome, _ = run_conformance_fix(render_context)
+
+    assert outcome == FixConformanceTest.IMPLEMENTATION_CODE_NOT_UPDATED
+    assert ctx.implementation_code_fixes == []
+
+
+def run_unit_tests_fix(render_context):
+    return FixUnitTests().execute(render_context, {"previous_unittests_issue": "1 failed"})
+
+
+def test_unit_tests_fix_forwards_conformance_fixes(build_folder, memory_folder):
+    api = FakeCodeplainAPI(unittests_fix_response={"test_app.py": "def test_add(): pass\n"})
+    ctx = make_conformance_context()
+    ctx.implementation_code_fixes.append(
+        {"hypothesis": "off by one", "approach": "add one", "code_diff": {"app.py": "+    return a + b + 1"}}
+    )
+    render_context = make_render_context(api, build_folder, memory_folder, ctx)
+
+    outcome, _ = run_unit_tests_fix(render_context)
+
+    assert outcome == FixUnitTests.SUCCESSFUL_OUTCOME
+    assert len(api.unittests_fix_calls) == 1
+    assert api.unittests_fix_calls[0]["conformance_tests_fixes"] == ctx.implementation_code_fixes
+    # The forwarded list is a copy, so later conformance fixes do not mutate what was sent.
+    assert api.unittests_fix_calls[0]["conformance_tests_fixes"] is not ctx.implementation_code_fixes
+
+
+def test_unit_tests_fix_outside_conformance_phase_sends_no_fixes(build_folder, memory_folder):
+    api = FakeCodeplainAPI()
+    render_context = make_render_context(api, build_folder, memory_folder, None)
+
+    run_unit_tests_fix(render_context)
+
+    assert api.unittests_fix_calls[0]["conformance_tests_fixes"] is None
+
+
+def test_unit_tests_fix_in_conformance_phase_without_implementation_changes_sends_no_fixes(build_folder, memory_folder):
+    api = FakeCodeplainAPI()
+    render_context = make_render_context(api, build_folder, memory_folder, make_conformance_context())
+
+    run_unit_tests_fix(render_context)
+
+    assert api.unittests_fix_calls[0]["conformance_tests_fixes"] is None