From 3341396c7048f257cce9bc3db75420e8a44ddd96 Mon Sep 17 00:00:00 2001 From: VitjanZ Date: Thu, 3 Sep 2026 11:04:28 +0200 Subject: [PATCH] Hand the conformance tests fix to the unit tests fixer When the conformance tests fixer changes implementation code, the unit tests are re-run and, if they fail, fixed next. The unit tests fixer had no idea the implementation was just deliberately changed, so it repaired the implementation back to what the stale unit tests asserted. The conformance fixer then re-applied the same change, and the render looped. The conformance tests running context now records every implementation change the conformance fixer made (hypothesis, approach and per-file diff). FixUnitTests sends that list as conformance_tests_fixes with the /fix_unittests_issue request, so the fixer preserves the change and adjusts the unit tests instead. The field is only sent when unit tests are processed inside the conformance phase. --- codeplain_REST_api.py | 5 + .../actions/fix_conformance_test.py | 11 + render_machine/actions/fix_unit_tests.py | 25 ++ render_machine/render_types.py | 5 + ...test_fix_unit_tests_conformance_context.py | 230 ++++++++++++++++++ 5 files changed, 276 insertions(+) create mode 100644 tests/test_fix_unit_tests_conformance_context.py diff --git a/codeplain_REST_api.py b/codeplain_REST_api.py index 7f676c66..e911e525 100644 --- a/codeplain_REST_api.py +++ b/codeplain_REST_api.py @@ -237,6 +237,7 @@ def fix_unittests_issue( required_modules, unittests_issue, run_state: RunState, + conformance_tests_fixes: list[dict] | None = None, ): endpoint_url = f"{self.api_url}/fix_unittests_issue" headers = {"X-API-Key": self.api_key, "Content-Type": "application/json"} @@ -251,6 +252,10 @@ def fix_unittests_issue( "unittests_issue": unittests_issue, "unittest_batch_id": run_state.unittest_batch_id, } + # Implementation code changes made by the conformance tests fixer right before this unit tests run; + # only sent when unit tests are processed inside the conformance tests phase. + if conformance_tests_fixes is not None: + payload["conformance_tests_fixes"] = conformance_tests_fixes return self.post_request(endpoint_url, headers, payload, run_state) diff --git a/render_machine/actions/fix_conformance_test.py b/render_machine/actions/fix_conformance_test.py index 882632d6..73e5ca0e 100644 --- a/render_machine/actions/fix_conformance_test.py +++ b/render_machine/actions/fix_conformance_test.py @@ -189,6 +189,17 @@ def execute(self, render_context: RenderContext, previous_action_payload: Any | ctx.test_that_triggered_code_change = (ctx.current_testing_module_name, ctx.current_testing_frid) ctx.execution_phase = TestExecutionPhase.RETRYING_AFTER_CODE_CHANGE + # Remember the change so that the unit tests fixer (which runs next) preserves it instead of + # reverting it - see FixUnitTests. + summary = fix_attempt_summary if isinstance(fix_attempt_summary, dict) else {} + ctx.implementation_code_fixes.append( + { + "hypothesis": summary.get("hypothesis"), + "approach": summary.get("approach"), + "code_diff": code_diff_files_content, + } + ) + journal.record_fix( ctx.current_testing_module_name, ctx.current_testing_frid, diff --git a/render_machine/actions/fix_unit_tests.py b/render_machine/actions/fix_unit_tests.py index c6cc4008..50a54b6a 100644 --- a/render_machine/actions/fix_unit_tests.py +++ b/render_machine/actions/fix_unit_tests.py @@ -32,6 +32,13 @@ def execute(self, render_context: RenderContext, previous_action_payload: Any | render_utils.print_inputs(render_context, existing_files_content, "Files sent as input to unit tests fixing:") + conformance_tests_fixes = self._get_conformance_tests_fixes(render_context) + if conformance_tests_fixes: + console.info( + f"Unit tests are fixed while preserving {len(conformance_tests_fixes)} implementation code change(s) " + "made to fix the conformance tests." + ) + response_files = render_context.codeplain_api.fix_unittests_issue( render_context.frid_context.frid, render_context.plain_source_tree, @@ -41,6 +48,7 @@ def execute(self, render_context: RenderContext, previous_action_payload: Any | render_context.get_required_modules_functionalities(), previous_unittests_issue, run_state=render_context.run_state, + conformance_tests_fixes=conformance_tests_fixes, ) _, changed_files = file_utils.update_build_folder_with_rendered_files( @@ -52,3 +60,20 @@ def execute(self, render_context: RenderContext, previous_action_payload: Any | console.print_files("Files fixed:", render_context.build_folder, response_files, style=console.OUTPUT_STYLE) return self.SUCCESSFUL_OUTCOME, None + + @staticmethod + def _get_conformance_tests_fixes(render_context: RenderContext) -> list[dict] | None: + """Implementation code changes the conformance tests fixer made before these unit tests were run. + + Only present when unit tests are processed inside the conformance tests phase - the implementation + and refactoring unit test passes have no conformance tests running context. + """ + conformance_tests_running_context = getattr(render_context, "conformance_tests_running_context", None) + if conformance_tests_running_context is None: + return None + + implementation_code_fixes = getattr(conformance_tests_running_context, "implementation_code_fixes", None) + if not implementation_code_fixes: + return None + + return list(implementation_code_fixes) diff --git a/render_machine/render_types.py b/render_machine/render_types.py index 6e653b02..75a2f9b6 100644 --- a/render_machine/render_types.py +++ b/render_machine/render_types.py @@ -91,6 +91,11 @@ def __init__( self.current_testing_frid_high_level_implementation_plan: Optional[str] = None + # Implementation code changes made by the conformance tests fixer during this conformance phase, in + # order. Each entry is {"hypothesis": str | None, "approach": str | None, "code_diff": {file: diff}}. + # Handed to the unit tests fixer so it adjusts the unit tests instead of reverting these changes. + self.implementation_code_fixes: list[dict] = [] + def get_conformance_tests_json(self, module_name: str) -> dict: return self._conformance_tests_json[module_name] diff --git a/tests/test_fix_unit_tests_conformance_context.py b/tests/test_fix_unit_tests_conformance_context.py new file mode 100644 index 00000000..a2fe3a1e --- /dev/null +++ b/tests/test_fix_unit_tests_conformance_context.py @@ -0,0 +1,230 @@ +"""Tests for handing the conformance tests fixer's implementation changes to the unit tests fixer. + +When the conformance tests fixer changes implementation code, the unit tests are re-run and fixed next. +The unit tests fixer must know about that change so it adjusts the unit tests instead of reverting it. +""" + +import os +import tempfile +from types import SimpleNamespace + +import pytest + +import plain_spec +from memory_management import MemoryManager +from render_machine.actions import fix_unit_tests as fix_unit_tests_module +from render_machine.actions.fix_conformance_test import FixConformanceTest +from render_machine.actions.fix_unit_tests import FixUnitTests +from render_machine.implementation_code_helpers import ImplementationCodeHelpers +from render_machine.render_types import ConformanceTestsRunningContext, UnitTestsRunningContext + + +class FakeCodeplainAPI: + def __init__(self, conformance_fix_response=None, unittests_fix_response=None): + self.conformance_fix_response = conformance_fix_response + self.unittests_fix_response = unittests_fix_response or {} + self.unittests_fix_calls = [] + + def fix_conformance_tests_issue(self, *args, **kwargs): + return self.conformance_fix_response + + def fix_unittests_issue(self, *args, **kwargs): + self.unittests_fix_calls.append(kwargs) + return self.unittests_fix_response + + +class FakeConformanceTests: + def fetch_existing_conformance_test_files(self, *args): + return {}, {} + + def store_conformance_tests_files(self, *args): + pass + + +@pytest.fixture +def build_folder(): + with tempfile.TemporaryDirectory() as folder: + with open(os.path.join(folder, "app.py"), "w", encoding="utf-8") as source_file: + source_file.write("def add(a, b):\n return a + b\n") + yield folder + + +@pytest.fixture +def memory_folder(): + with tempfile.TemporaryDirectory() as folder: + yield folder + + +@pytest.fixture(autouse=True) +def isolate_from_git_and_console(monkeypatch): + monkeypatch.setattr(ImplementationCodeHelpers, "get_code_diff", staticmethod(lambda *args: {})) + monkeypatch.setattr(plain_spec, "collect_linked_resources", lambda *args: None) + monkeypatch.setattr(fix_unit_tests_module.render_utils, "print_inputs", lambda *args: None) + + +def make_conformance_context(): + return ConformanceTestsRunningContext( + current_testing_module_name="mod", + current_testing_frid="1", + fix_attempts=0, + conformance_tests_json={"1": {"folder_name": "conformance_1"}}, + conformance_tests_render_attempts=0, + current_testing_frid_specifications={}, + should_prepare_testing_environment=False, + frid_being_implemented="1", + ) + + +def make_render_context(api, build_folder, memory_folder, conformance_tests_running_context): + return SimpleNamespace( + codeplain_api=api, + build_folder=build_folder, + memory_manager=MemoryManager(api, memory_folder), + conformance_tests=FakeConformanceTests(), + conformance_tests_running_context=conformance_tests_running_context, + unit_tests_running_context=UnitTestsRunningContext(fix_attempts=1), + plain_source_tree={}, + module_name="mod", + required_modules=None, + frid_context=SimpleNamespace(frid="1", linked_resources={}), + get_required_modules_functionalities=lambda: {}, + run_state=SimpleNamespace(render_id="test-render-id", unittest_batch_id=1), + ) + + +def run_conformance_fix(render_context): + return FixConformanceTest().execute(render_context, {"previous_conformance_tests_issue": "tests failed"}) + + +def test_conformance_fix_of_implementation_code_is_remembered(build_folder, memory_folder): + api = FakeCodeplainAPI( + conformance_fix_response=[ + FixConformanceTest.ISSUE_REASON_CODE_IMPLEMENTATION_CODE, + {"app.py": "def add(a, b):\n return a + b + 1\n"}, + {"hypothesis": "off by one", "approach": "add one"}, + "prepared issue", + ] + ) + ctx = make_conformance_context() + render_context = make_render_context(api, build_folder, memory_folder, ctx) + + outcome, _ = run_conformance_fix(render_context) + + assert outcome == FixConformanceTest.IMPLEMENTATION_CODE_UPDATED + assert len(ctx.implementation_code_fixes) == 1 + recorded_fix = ctx.implementation_code_fixes[0] + assert recorded_fix["hypothesis"] == "off by one" + assert recorded_fix["approach"] == "add one" + assert list(recorded_fix["code_diff"]) == ["app.py"] + assert "+ return a + b + 1" in recorded_fix["code_diff"]["app.py"] + + +def test_consecutive_implementation_fixes_accumulate(build_folder, memory_folder): + ctx = make_conformance_context() + for attempt in range(2): + api = FakeCodeplainAPI( + conformance_fix_response=[ + FixConformanceTest.ISSUE_REASON_CODE_IMPLEMENTATION_CODE, + {"app.py": f"def add(a, b):\n return a + b + {attempt + 1}\n"}, + {"hypothesis": f"hypothesis {attempt}", "approach": f"approach {attempt}"}, + "prepared issue", + ] + ) + render_context = make_render_context(api, build_folder, memory_folder, ctx) + run_conformance_fix(render_context) + + assert [fix["hypothesis"] for fix in ctx.implementation_code_fixes] == ["hypothesis 0", "hypothesis 1"] + + +def test_conformance_fix_without_summary_is_still_remembered(build_folder, memory_folder): + api = FakeCodeplainAPI( + conformance_fix_response=[ + FixConformanceTest.ISSUE_REASON_CODE_IMPLEMENTATION_CODE, + {"app.py": "def add(a, b):\n return a + b + 1\n"}, + None, + "prepared issue", + ] + ) + ctx = make_conformance_context() + render_context = make_render_context(api, build_folder, memory_folder, ctx) + + run_conformance_fix(render_context) + + assert ctx.implementation_code_fixes[0]["hypothesis"] is None + assert ctx.implementation_code_fixes[0]["approach"] is None + assert "app.py" in ctx.implementation_code_fixes[0]["code_diff"] + + +def test_conformance_fix_of_conformance_tests_is_not_remembered(build_folder, memory_folder): + api = FakeCodeplainAPI( + conformance_fix_response=[ + FixConformanceTest.ISSUE_REASON_CODE_CONFORMANCE_TESTS, + {"test_app.py": "def test_add(): pass\n"}, + {"hypothesis": "wrong assertion", "approach": "fix test"}, + "prepared issue", + ] + ) + ctx = make_conformance_context() + render_context = make_render_context(api, build_folder, memory_folder, ctx) + + outcome, _ = run_conformance_fix(render_context) + + assert outcome == FixConformanceTest.IMPLEMENTATION_CODE_NOT_UPDATED + assert ctx.implementation_code_fixes == [] + + +def test_implementation_fix_with_no_files_is_not_remembered(build_folder, memory_folder): + api = FakeCodeplainAPI( + conformance_fix_response=[ + FixConformanceTest.ISSUE_REASON_CODE_IMPLEMENTATION_CODE, + {}, + {"hypothesis": "nothing", "approach": "nothing"}, + "prepared issue", + ] + ) + ctx = make_conformance_context() + render_context = make_render_context(api, build_folder, memory_folder, ctx) + + outcome, _ = run_conformance_fix(render_context) + + assert outcome == FixConformanceTest.IMPLEMENTATION_CODE_NOT_UPDATED + assert ctx.implementation_code_fixes == [] + + +def run_unit_tests_fix(render_context): + return FixUnitTests().execute(render_context, {"previous_unittests_issue": "1 failed"}) + + +def test_unit_tests_fix_forwards_conformance_fixes(build_folder, memory_folder): + api = FakeCodeplainAPI(unittests_fix_response={"test_app.py": "def test_add(): pass\n"}) + ctx = make_conformance_context() + ctx.implementation_code_fixes.append( + {"hypothesis": "off by one", "approach": "add one", "code_diff": {"app.py": "+ return a + b + 1"}} + ) + render_context = make_render_context(api, build_folder, memory_folder, ctx) + + outcome, _ = run_unit_tests_fix(render_context) + + assert outcome == FixUnitTests.SUCCESSFUL_OUTCOME + assert len(api.unittests_fix_calls) == 1 + assert api.unittests_fix_calls[0]["conformance_tests_fixes"] == ctx.implementation_code_fixes + # The forwarded list is a copy, so later conformance fixes do not mutate what was sent. + assert api.unittests_fix_calls[0]["conformance_tests_fixes"] is not ctx.implementation_code_fixes + + +def test_unit_tests_fix_outside_conformance_phase_sends_no_fixes(build_folder, memory_folder): + api = FakeCodeplainAPI() + render_context = make_render_context(api, build_folder, memory_folder, None) + + run_unit_tests_fix(render_context) + + assert api.unittests_fix_calls[0]["conformance_tests_fixes"] is None + + +def test_unit_tests_fix_in_conformance_phase_without_implementation_changes_sends_no_fixes(build_folder, memory_folder): + api = FakeCodeplainAPI() + render_context = make_render_context(api, build_folder, memory_folder, make_conformance_context()) + + run_unit_tests_fix(render_context) + + assert api.unittests_fix_calls[0]["conformance_tests_fixes"] is None