Consolidate the fix journal into lessons instead of rebuilding memory each round

Removes the per-run memory call and replaces it with a single consolidation
when a functionality's conformance tests pass, reading the journal that the fix
loop has been accumulating.

The lessons land where memory files are already read from, so they flow into
every later prompt through the existing path - only the producer changes. A
functionality that passed without any fixing has an empty journal and costs no
call at all.

The journal is discarded once its lessons have been taken. The three
previous_conformance_tests_issue fields on the running context go with it; they
existed only to feed the per-run memory call.

Author: VitjanZ

codeplain_REST_api.py

@@ -254,40 +254,34 @@ def fix_unittests_issue(
 
         return self.post_request(endpoint_url, headers, payload, run_state)
 
-    def create_conformance_test_memory(
+    def consolidate_conformance_test_lessons(
         self,
         frid,
         plain_source_tree,
         linked_resources,
-        existing_files_content,
         memory_files_content,
         module_name,
         required_modules,
-        code_diff,
         conformance_tests_files,
         acceptance_tests,
-        conformance_tests_issue,
         conformance_tests_folder_name,
-        previous_conformance_tests_issue_old,
+        conformance_test_journal,
         run_state: RunState,
     ):
-        endpoint_url = f"{self.api_url}/create_conformance_test_memory"
+        endpoint_url = f"{self.api_url}/consolidate_conformance_test_lessons"
         headers = {"X-API-Key": self.api_key, "Content-Type": "application/json"}
 
         payload = {
             "frid": frid,
             "plain_source_tree": plain_source_tree,
             "linked_resources": linked_resources,
-            "existing_files_content": existing_files_content,
             "memory_files_content": memory_files_content,
             "module_name": module_name,
             "required_modules": required_modules,
-            "code_diff": code_diff,
             "conformance_tests_files": conformance_tests_files,
             "acceptance_tests": acceptance_tests,
-            "conformance_tests_issue": conformance_tests_issue,
             "conformance_tests_folder_name": conformance_tests_folder_name,
-            "previous_conformance_tests_issue": previous_conformance_tests_issue_old,
+            "conformance_test_journal": conformance_test_journal,
         }
 
         return self.post_request(endpoint_url, headers, payload, run_state)

memory_management.py

@@ -1,9 +1,24 @@
-import json
+"""What the renderer remembers about conformance tests, and when.
+
+Two artifacts, with different lifetimes:
+
+* The **journal**, written mechanically as a functionality's fix loop runs, holding every attempt made against
+  it. See :mod:`conformance_test_journal`. It is discarded once that functionality's tests pass.
+
+* The **lessons**, extracted from the journal at that point and kept for the rest of the project. These are
+  the memory files, and they are fed into every later prompt - which is why the set is capped and why a lesson
+  has to be a transferable constraint rather than the story of one bug.
+
+Consolidation runs once per functionality. It replaces an earlier arrangement that rebuilt a memory after
+every conformance test run and, being told not to duplicate itself, deleted the previous one each time - so a
+twenty round fix loop remembered only its most recent round.
+"""
+
 import os
 
 import file_utils
+from conformance_test_journal import ConformanceTestJournal
 from plain2code_console import console
-from render_machine.implementation_code_helpers import ImplementationCodeHelpers
 from render_machine.render_context import RenderContext
 
 CONFORMANCE_TESTS_SUCCESS_EXIT_CODE = 0
@@ -27,106 +42,58 @@ def __init__(self, codeplain_api, memory_folder: str):
         self.codeplain_api = codeplain_api
         self.memory_folder = memory_folder
 
-    def create_conformance_tests_memory(
-        self, render_context: RenderContext, exit_code: int, conformance_tests_issue: str
-    ):
+    def consolidate_lessons(self, render_context: RenderContext) -> None:
+        """Extract what transfers to later functionalities, then discard the journal it came from.
 
-        current_conformance_tests_issue_frid = render_context.conformance_tests_running_context.current_testing_frid
-        current_conformance_tests_issue_module = (
-            render_context.conformance_tests_running_context.current_testing_module_name
-        )
-        old_conformance_tests_issue_frid = (
-            render_context.conformance_tests_running_context.previous_conformance_tests_issue_frid
-        )
-        old_conformance_tests_issue_module = (
-            render_context.conformance_tests_running_context.previous_conformance_tests_issue_module
-        )
-
-        old_conformance_tests_issue = (
-            render_context.conformance_tests_running_context.previous_conformance_tests_issue_old
-        )
-
-        is_first_time_running_conformance_tests = (
-            old_conformance_tests_issue_frid is None
-            or old_conformance_tests_issue_frid == ""
-            or old_conformance_tests_issue_module != current_conformance_tests_issue_module
-        )
-        is_same_frid_as_previous_failing_test = (
-            current_conformance_tests_issue_frid == old_conformance_tests_issue_frid
-            and current_conformance_tests_issue_module == old_conformance_tests_issue_module
-        )
-        is_conformance_test_failed = exit_code != CONFORMANCE_TESTS_SUCCESS_EXIT_CODE
+        Called when a functionality's own conformance tests pass. A functionality whose tests passed without
+        any fixing has an empty journal and teaches nothing, so no call is made for it.
+        """
+        ctx = render_context.conformance_tests_running_context
+        if ctx.current_testing_frid is None:
+            return
 
-        should_create_memory = not is_first_time_running_conformance_tests and (
-            is_same_frid_as_previous_failing_test or is_conformance_test_failed
+        journal = ConformanceTestJournal.load(
+            self.memory_folder, ctx.current_testing_module_name, ctx.current_testing_frid
         )
-        code_diff_files = render_context.conformance_tests_running_context.code_diff_files
-
-        if not should_create_memory or code_diff_files is None:
-            console.debug(
-                "Skipping creation of conformance test memory because the conditions for creating memories are not met."
-            )
+        previous_attempts = journal.render_for_prompt()
+        if not previous_attempts:
+            console.debug("No conformance test fixes were needed, so there are no lessons to consolidate.")
             return
 
-        existing_files, existing_files_content = ImplementationCodeHelpers.fetch_existing_files(
-            render_context.build_folder
-        )
         memory_files, memory_files_content = MemoryManager.fetch_memory_files(self.memory_folder)
-
-        conformance_tests_folder_name = (
-            render_context.conformance_tests_running_context.get_current_conformance_test_folder_name()
-        )
-
+        conformance_tests_folder_name = ctx.get_current_conformance_test_folder_name()
         (
             _,
-            existing_conformance_test_files_content,
+            conformance_tests_files_content,
         ) = render_context.conformance_tests.fetch_existing_conformance_test_files(
             render_context.module_name,
             render_context.required_modules,
-            render_context.conformance_tests_running_context.current_testing_module_name,
+            ctx.current_testing_module_name,
             conformance_tests_folder_name,
         )
-        acceptance_tests = render_context.conformance_tests_running_context.get_current_acceptance_tests()
 
-        response_files = render_context.codeplain_api.create_conformance_test_memory(
+        console.info(
+            f"Consolidating what fixing the conformance tests for functionality {ctx.current_testing_frid} "
+            f"in module {ctx.current_testing_module_name} has taught."
+        )
+
+        response_files = render_context.codeplain_api.consolidate_conformance_test_lessons(
             render_context.frid_context.frid,
             render_context.plain_source_tree,
             render_context.frid_context.linked_resources,
-            existing_files_content,
             memory_files_content,
             render_context.module_name,
             render_context.get_required_modules_functionalities(),
-            code_diff_files,
-            existing_conformance_test_files_content,
-            acceptance_tests,
-            conformance_tests_issue,
+            conformance_tests_files_content,
+            ctx.get_current_acceptance_tests(),
             conformance_tests_folder_name,
-            old_conformance_tests_issue,
+            previous_attempts,
             run_state=render_context.run_state,
         )
-        if len(response_files) > 0:
+
+        if response_files:
             memory_folder_path = os.path.join(self.memory_folder, CONFORMANCE_TEST_MEMORY_SUBFOLDER)
             file_utils.store_response_files(memory_folder_path, response_files, memory_files)
 
-    def delete_unresolved_memory_files(self):
-        """Delete memory files whose resolution_status is not 'RESOLVED'."""
-        memory_path = os.path.join(self.memory_folder, CONFORMANCE_TEST_MEMORY_SUBFOLDER)
-        if not os.path.exists(memory_path):
-            return
-
-        memory_files = file_utils.list_all_text_files(memory_path)
-        for file_name in memory_files:
-            file_path = os.path.join(memory_path, file_name)
-            try:
-                with open(file_path, "r") as f:
-                    content = json.load(f)
-                if content.get("resolution_status") == "RESOLVED":
-                    continue
-                else:
-                    os.remove(file_path)
-            except (json.JSONDecodeError, OSError):
-                # Not a valid JSON file, unlikely to be a valid memory file, delete it
-                console.error(f"Memory file is not a valid JSON file: {file_name}. Deleting it.")
-                os.remove(file_path)
-
-            console.debug(f"Deleted temporary memory file: {file_name}")
+        # The journal has served its purpose for this functionality; anything worth keeping is now a lesson.
+        journal.delete(self.memory_folder)

render_machine/actions/fix_conformance_test.py

@@ -91,16 +91,6 @@ def execute(self, render_context: RenderContext, previous_action_payload: Any |
             )
         previous_conformance_tests_issue = previous_action_payload["previous_conformance_tests_issue"]
 
-        render_context.conformance_tests_running_context.previous_conformance_tests_issue_old = (
-            previous_conformance_tests_issue
-        )
-        render_context.conformance_tests_running_context.previous_conformance_tests_issue_frid = (
-            render_context.conformance_tests_running_context.current_testing_frid
-        )
-        render_context.conformance_tests_running_context.previous_conformance_tests_issue_module = (
-            render_context.conformance_tests_running_context.current_testing_module_name
-        )
-
         existing_files, existing_files_content = ImplementationCodeHelpers.fetch_existing_files(
             render_context.build_folder
         )

render_machine/actions/run_conformance_tests.py

@@ -84,18 +84,14 @@ def execute(self, render_context: RenderContext, _previous_action_payload: Any |
 
         self._fingerprint_run(render_context, exit_code, conformance_tests_issue)
 
-        render_context.memory_manager.create_conformance_tests_memory(
-            render_context, exit_code, conformance_tests_issue
-        )
-
         if exit_code == 0:
             if (
                 render_context.conformance_tests_running_context.current_testing_module_name
                 == render_context.module_name
                 and render_context.conformance_tests_running_context.current_testing_frid
                 == render_context.frid_context.frid
             ):
-                render_context.memory_manager.delete_unresolved_memory_files()
+                render_context.memory_manager.consolidate_lessons(render_context)
             return self.SUCCESSFUL_OUTCOME, None
 
         if exit_code in UNRECOVERABLE_ERROR_EXIT_CODES:

render_machine/render_types.py

@@ -96,9 +96,6 @@ def __init__(
         # repeat of an earlier one.
         self.last_failure_signature: Optional[str] = None
         self.last_failure_excerpt: Optional[str] = None
-        self.previous_conformance_tests_issue_old: Optional[str] = None
-        self.previous_conformance_tests_issue_frid: Optional[str] = None
-        self.previous_conformance_tests_issue_module: Optional[str] = None
         self.code_diff_files: Optional[dict[str, str]] = None
 
     def get_conformance_tests_json(self, module_name: str) -> dict:

tests/test_lesson_consolidation.py

@@ -0,0 +1,125 @@
+"""Tests for consolidating a functionality's fix journal into lessons once its tests pass."""
+
+import os
+from unittest.mock import MagicMock
+
+import pytest
+
+from conformance_test_journal import TARGET_IMPLEMENTATION, ConformanceTestJournal
+from memory_management import CONFORMANCE_TEST_MEMORY_SUBFOLDER, MemoryManager
+from render_machine.render_types import ConformanceTestsRunningContext
+
+LESSONS_FILE_NAME = "conformance_test_lessons.json"
+
+
+@pytest.fixture
+def render_context(tmp_path):
+    context = MagicMock()
+    context.memory_manager.memory_folder = str(tmp_path)
+    context.module_name = "module"
+    context.frid_context.frid = "1"
+    context.conformance_tests_running_context = ConformanceTestsRunningContext(
+        current_testing_module_name="module",
+        current_testing_frid="1",
+        fix_attempts=0,
+        conformance_tests_json={"1": {"folder_name": "tests_for_1"}},
+        conformance_tests_render_attempts=0,
+        current_testing_frid_specifications=None,
+        should_prepare_testing_environment=False,
+    )
+    context.conformance_tests.fetch_existing_conformance_test_files.return_value = ([], {})
+    context.codeplain_api.consolidate_conformance_test_lessons.return_value = {
+        LESSONS_FILE_NAME: '{"lessons": [{"lesson": "a rule"}]}'
+    }
+    return context
+
+
+@pytest.fixture
+def memory_manager(render_context, tmp_path):
+    return MemoryManager(render_context.codeplain_api, str(tmp_path))
+
+
+def _write_journal(tmp_path, rounds=2):
+    journal = ConformanceTestJournal("module", "1")
+    for index in range(rounds):
+        journal.record_attempt(TARGET_IMPLEMENTATION, [f"file_{index}.py"], issue_excerpt="E assert 3 == 4")
+    journal.save(str(tmp_path))
+    return journal
+
+
+def test_lessons_are_consolidated_from_the_journal(memory_manager, render_context, tmp_path):
+    _write_journal(tmp_path)
+
+    memory_manager.consolidate_lessons(render_context)
+
+    render_context.codeplain_api.consolidate_conformance_test_lessons.assert_called_once()
+    journal_argument = render_context.codeplain_api.consolidate_conformance_test_lessons.call_args[0][-1]
+    assert "Attempt 1" in journal_argument and "Attempt 2" in journal_argument
+
+
+def test_the_consolidated_lessons_are_stored_where_memory_files_are_read_from(memory_manager, render_context, tmp_path):
+    _write_journal(tmp_path)
+
+    memory_manager.consolidate_lessons(render_context)
+
+    _, memory_files_content = MemoryManager.fetch_memory_files(str(tmp_path))
+    assert LESSONS_FILE_NAME in memory_files_content
+
+
+def test_the_journal_is_discarded_once_its_lessons_have_been_taken(memory_manager, render_context, tmp_path):
+    _write_journal(tmp_path)
+
+    memory_manager.consolidate_lessons(render_context)
+
+    assert not os.path.exists(ConformanceTestJournal.journal_path(str(tmp_path), "module", "1"))
+
+
+def test_the_lessons_already_on_record_are_sent_so_they_can_be_reconciled(memory_manager, render_context, tmp_path):
+    _write_journal(tmp_path)
+    memory_path = os.path.join(str(tmp_path), CONFORMANCE_TEST_MEMORY_SUBFOLDER)
+    os.makedirs(memory_path, exist_ok=True)
+    with open(os.path.join(memory_path, LESSONS_FILE_NAME), "w", encoding="utf-8") as recorded:
+        recorded.write('{"lessons": [{"lesson": "an earlier rule"}]}')
+
+    memory_manager.consolidate_lessons(render_context)
+
+    sent_memory_files = render_context.codeplain_api.consolidate_conformance_test_lessons.call_args[0][3]
+    assert "an earlier rule" in sent_memory_files[LESSONS_FILE_NAME]
+
+
+def test_a_functionality_that_needed_no_fixing_costs_no_call(memory_manager, render_context):
+    """An empty journal means the tests passed first time, so there is nothing to have learned."""
+    memory_manager.consolidate_lessons(render_context)
+
+    render_context.codeplain_api.consolidate_conformance_test_lessons.assert_not_called()
+
+
+def test_nothing_is_consolidated_when_no_functionality_is_under_test(memory_manager, render_context, tmp_path):
+    _write_journal(tmp_path)
+    render_context.conformance_tests_running_context.current_testing_frid = None
+
+    memory_manager.consolidate_lessons(render_context)
+
+    render_context.codeplain_api.consolidate_conformance_test_lessons.assert_not_called()
+
+
+def test_the_journal_survives_a_consolidation_that_returned_nothing(memory_manager, render_context, tmp_path):
+    """Returning no lessons is a valid outcome; it must not be confused with a failure to run."""
+    _write_journal(tmp_path)
+    render_context.codeplain_api.consolidate_conformance_test_lessons.return_value = {}
+
+    memory_manager.consolidate_lessons(render_context)
+
+    _, memory_files_content = MemoryManager.fetch_memory_files(str(tmp_path))
+    assert memory_files_content == {}
+
+
+def test_only_the_journal_of_the_functionality_under_test_is_harvested(memory_manager, render_context, tmp_path):
+    """Journals are per functionality, so another functionality's record is not swept up by this one."""
+    _write_journal(tmp_path)
+    render_context.conformance_tests_running_context.current_testing_frid = "2"
+
+    memory_manager.consolidate_lessons(render_context)
+
+    render_context.codeplain_api.consolidate_conformance_test_lessons.assert_not_called()
+    assert os.path.exists(ConformanceTestJournal.journal_path(str(tmp_path), "module", "1"))