Stop telling the user to rewrite the specification when a fix loop gives up

Both client-side fix loops ended with "Please review and rewrite the
specification.", which asserts a cause nothing in the render
establishes: a conformance loop has been observed patching thirteen
times against a failure it was not moving, which tells us we did not
converge, not that the input was wrong. The server said the same and
no longer does.

Each reason now names the loop, the functionality and the attempt
count, and points at the log that holds the detail. The conformance
loop also gains the error type its unit-test counterpart already had.

Author: gorandodig

render_machine/actions/fix_conformance_test.py

@@ -9,7 +9,7 @@
 from render_machine.actions.base_action import BaseAction
 from render_machine.implementation_code_helpers import ImplementationCodeHelpers
 from render_machine.render_context import RenderContext
-from render_machine.render_types import RenderError, TestExecutionPhase
+from render_machine.render_types import FIX_LOOP_EXHAUSTED_HINT, RenderError, TestExecutionPhase
 
 MAX_CONFORMANCE_TEST_FIX_ATTEMPTS = 20
 MAX_CONFORMANCE_TEST_RERENDER_ATTEMPTS = 1
@@ -32,11 +32,20 @@ def execute(self, render_context: RenderContext, previous_action_payload: Any |
 
         if ctx.fix_attempts >= MAX_CONFORMANCE_TEST_FIX_ATTEMPTS:
             if ctx.conformance_tests_render_attempts >= MAX_CONFORMANCE_TEST_RERENDER_ATTEMPTS:
-                error_msg = f"The renderer was unable to produce an implementation that passes conformance tests for functionality '{render_context.frid_context.frid}' after many attempts. Please review and rewrite the specification. (Render ID: {render_context.run_state.render_id})"
+                error_msg = (
+                    f"Rendering stopped: the conformance tests for functionality "
+                    f"'{render_context.frid_context.frid}' still failed after "
+                    f"{MAX_CONFORMANCE_TEST_FIX_ATTEMPTS} attempts at fixing them. {FIX_LOOP_EXHAUSTED_HINT} "
+                    f"(Render ID: {render_context.run_state.render_id})"
+                )
                 render_context.last_error_message = error_msg
                 return (
                     self.LIMIT_EXCEEDED_OUTCOME,
-                    RenderError.encode(message=error_msg).to_payload(),
+                    RenderError.encode(
+                        message=error_msg,
+                        error_type="CONFORMANCE_TESTS_FIX_EXHAUSTED",
+                        frid=render_context.frid_context.frid,
+                    ).to_payload(),
                 )
             else:
                 ctx.regenerating_conformance_tests = True

render_machine/actions/render_functional_requirement.py

@@ -8,7 +8,7 @@
 from render_machine.actions.base_action import BaseAction
 from render_machine.implementation_code_helpers import ImplementationCodeHelpers
 from render_machine.render_context import RenderContext
-from render_machine.render_types import RenderError
+from render_machine.render_types import FIX_LOOP_EXHAUSTED_HINT, RenderError
 
 MAX_CODE_GENERATION_RETRIES = 2
 
@@ -21,9 +21,9 @@ class RenderFunctionalRequirement(BaseAction):
     def execute(self, render_context: RenderContext, _previous_action_payload: Any | None):
         if render_context.frid_context.functional_requirement_render_attempts >= MAX_CODE_GENERATION_RETRIES:
             error_msg = (
-                f"The renderer was unable to produce an implementation whose unit tests pass for functionality "
-                f"'{render_context.frid_context.frid}' after rendering it from scratch {MAX_CODE_GENERATION_RETRIES} "
-                f"times. Please review and rewrite the specification."
+                f"Rendering stopped: the unit tests for functionality '{render_context.frid_context.frid}' still "
+                f"failed after rendering it from scratch {MAX_CODE_GENERATION_RETRIES} times. "
+                f"{FIX_LOOP_EXHAUSTED_HINT}"
             )
             render_context.last_error_message = error_msg
             return (

render_machine/render_types.py

@@ -148,6 +148,14 @@ class ScriptExecutionHistory:
     should_update_script_outputs: bool = False
 
 
+# Both fix loops end the same way when they run out of attempts, and neither ending tells us
+# the specification is at fault - only that we did not converge. The reason points at the
+# evidence instead of assigning blame.
+FIX_LOOP_EXHAUSTED_HINT = (
+    "The render log records what each attempt failed on. " "If this repeats, send the log to support@codeplain.ai."
+)
+
+
 @dataclass
 class RenderError:
     """Standardized error format for all render failures."""

tests/test_render_failure_payloads.py

@@ -9,11 +9,17 @@
 
 from unittest.mock import MagicMock
 
+from render_machine.actions.fix_conformance_test import (
+    MAX_CONFORMANCE_TEST_FIX_ATTEMPTS,
+    MAX_CONFORMANCE_TEST_RERENDER_ATTEMPTS,
+    FixConformanceTest,
+)
 from render_machine.actions.refactor_code import MAX_REFACTORING_ITERATIONS, RefactorCode
 from render_machine.actions.render_functional_requirement import (
     MAX_CODE_GENERATION_RETRIES,
     RenderFunctionalRequirement,
 )
+from render_machine.render_types import FIX_LOOP_EXHAUSTED_HINT
 
 
 def exhausted_context(**frid_attributes):
@@ -25,6 +31,15 @@ def exhausted_context(**frid_attributes):
     return context
 
 
+def exhausted_conformance_context():
+    """A conformance loop that has spent its fix attempts and its one regeneration."""
+    context = exhausted_context()
+    ctx = context.conformance_tests_running_context
+    ctx.fix_attempts = MAX_CONFORMANCE_TEST_FIX_ATTEMPTS - 1
+    ctx.conformance_tests_render_attempts = MAX_CONFORMANCE_TEST_RERENDER_ATTEMPTS
+    return context
+
+
 def test_unit_test_exhaustion_reports_a_typed_reason():
     context = exhausted_context(functional_requirement_render_attempts=MAX_CODE_GENERATION_RETRIES)
 
@@ -43,7 +58,36 @@ def test_unit_test_exhaustion_names_the_functionality_and_what_to_do():
 
     assert "'1'" in message
     assert "unit tests" in message
-    assert "specification" in message
+    assert FIX_LOOP_EXHAUSTED_HINT in message
+
+
+def test_no_fix_loop_blames_the_specification_for_giving_up():
+    """Exhaustion tells us we did not converge, not that the input was wrong. Both loops used
+    to end with `Please review and rewrite the specification.`, which asserts a cause nothing
+    in the render establishes - a conformance loop has been observed patching thirteen times
+    against a failure it was not moving."""
+    unit = exhausted_context(functional_requirement_render_attempts=MAX_CODE_GENERATION_RETRIES)
+    _, unit_payload = RenderFunctionalRequirement().execute(unit, None)
+
+    conformance = exhausted_conformance_context()
+    _, conformance_payload = FixConformanceTest().execute(conformance, None)
+
+    for payload in (unit_payload, conformance_payload):
+        assert "rewrite the specification" not in payload["error"]["message"]
+
+
+def test_conformance_exhaustion_names_the_functionality_and_what_to_do():
+    context = exhausted_conformance_context()
+
+    outcome, payload = FixConformanceTest().execute(context, None)
+    message = payload["error"]["message"]
+
+    assert outcome == FixConformanceTest.LIMIT_EXCEEDED_OUTCOME
+    assert payload["error"]["type"] == "CONFORMANCE_TESTS_FIX_EXHAUSTED"
+    assert "'1'" in message
+    assert "conformance tests" in message
+    assert str(MAX_CONFORMANCE_TEST_FIX_ATTEMPTS) in message
+    assert FIX_LOOP_EXHAUSTED_HINT in message
 
 
 def test_unit_test_exhaustion_logs_the_same_reason_it_returns():