Show when unit test output is cut, and keep its end

docs/plain2code_cli.md

@@ -66,7 +66,10 @@ options:
   --unittests-script UNITTESTS_SCRIPT
                         Shell script to run unit tests on generated code.
                         Receives the module's code folder path as its first
-                        argument (e.g. `plain_modules/module_name/code`).
+                        argument (e.g. `plain_modules/module_name/code`). When
+                        it fails, the fixer gets at most 10,000 characters of
+                        its output: the start and the end, with the middle
+                        left out.
   --conformance-tests-script CONFORMANCE_TESTS_SCRIPT
                         Path to conformance tests shell script. Every
                         conformance test script should accept two arguments:

docs/starting_a_plain_project_from_scratch.md

@@ -72,6 +72,16 @@ cp /path/to/plain2code_client/test_scripts/run_conformance_tests_python.sh ./
 ```
 You may need to modify these scripts based on your specific project requirements.
 
+### Keep test output within the 10,000-character budget
+
+When a unit test script fails, its output goes to the fixer, which gets at most 10,000 characters of it. Longer output is cut from the middle: the fixer gets about the first and the last 4,980 characters, with a marker saying how many characters were left out. A warning in the log gives the full size each time this happens. Conformance test output over the same budget is summarized on the server instead.
+
+Output the fixer cannot act on uses up the budget and pushes the lines that matter into the middle, where they are cut. To keep the whole output within the budget:
+
+- Print a passing test run as its summary line only, and a failing one in full. A passing run still reaches the fixer when something else in the script fails, such as the linter.
+- Print lint results as counts per rule, then one line per offense. For RuboCop: `rubocop --format offenses --format emacs`.
+- Leave out coverage tables, slowest-test reports, and progress output.
+
 ## 3. Configure Parameters
 
 Create a `config.yaml` (default name, which you can change with `--config-name` argument in the file) file to configure the plain2code CLI parameters.

plain2code_arguments.py

@@ -299,7 +299,8 @@ def create_parser():
         parser,
         "--unittests-script",
         type=str,
-        help="Shell script to run unit tests on generated code. Receives the module's code folder path as its first argument (e.g. `plain_modules/module_name/code`).",
+        help="Shell script to run unit tests on generated code. Receives the module's code folder path as its first argument (e.g. `plain_modules/module_name/code`). "
+        "When it fails, the fixer gets at most 10,000 characters of its output: the start and the end, with the middle left out.",
     )
     _add_arg(
         parser,

plain2code_utils.py

@@ -35,6 +35,29 @@ def format_duration_hms(total_seconds: float) -> str:
     return f"{hours}h {minutes % 60}m"
 
 
+def _omission_marker(omitted: int) -> str:
+    return f"\n\n[... {omitted:,} characters omitted ...]\n\n"
+
+
+def truncate_middle(text: str, max_length: int) -> tuple[str, int]:
+    """Fit text into max_length characters by cutting the middle out, keeping equal shares of start and end.
+
+    Test output carries the first failures at the start and the summary lines and lint findings at the
+    end, so both ends are kept. A marker in place of the cut says how many characters were left out.
+    Returns the fitted text and the number of characters left out (0 when the text already fits).
+    """
+    if len(text) <= max_length:
+        return text, 0
+
+    # The marker's length depends on the count it reports, so leave room for the largest possible count.
+    kept_at_each_end = (max_length - len(_omission_marker(len(text)))) // 2
+    if kept_at_each_end < 1:
+        raise ValueError(f"A budget of {max_length} characters cannot hold the omission marker.")
+
+    omitted = len(text) - 2 * kept_at_each_end
+    return text[:kept_at_each_end] + _omission_marker(omitted) + text[-kept_at_each_end:], omitted
+
+
 AMBIGUITY_CAUSES = {
     "reference_resource_ambiguity": "Ambiguity is in the reference resources",
     "definition_ambiguity": "Ambiguity is in the definitions",

render_machine/actions/fix_unit_tests.py

@@ -4,10 +4,12 @@
 import render_machine.render_utils as render_utils
 from plain2code_console import console
 from plain2code_exceptions import InternalClientError
+from plain2code_utils import truncate_middle
 from render_machine.actions.base_action import BaseAction
 from render_machine.implementation_code_helpers import ImplementationCodeHelpers
 from render_machine.render_context import RenderContext
 
+# Same as MAX_ISSUE_LENGTH on the server: output within it reaches the fixer as is, without an LLM summary.
 MAX_ISSUE_LENGTH = 10000
 
 
@@ -21,9 +23,12 @@ def execute(self, render_context: RenderContext, previous_action_payload: Any |
             )
         previous_unittests_issue = previous_action_payload["previous_unittests_issue"]
 
-        if previous_unittests_issue and len(previous_unittests_issue) > MAX_ISSUE_LENGTH:
-            console.debug(
-                f"Unit tests issue text is too long and will be smartly truncated to {MAX_ISSUE_LENGTH} characters."
+        unittests_issue, omitted_characters = truncate_middle(previous_unittests_issue, MAX_ISSUE_LENGTH)
+        if omitted_characters:
+            console.warning(
+                f"Unit test output is {len(previous_unittests_issue):,} characters, over the "
+                f"{MAX_ISSUE_LENGTH:,}-character budget. The fixer gets the start and the end; "
+                f"{omitted_characters:,} characters in the middle are left out."
             )
 
         existing_files, existing_files_content = ImplementationCodeHelpers.fetch_existing_files(
@@ -46,7 +51,7 @@ def execute(self, render_context: RenderContext, previous_action_payload: Any |
             existing_files_content,
             render_context.module_name,
             render_context.get_required_modules_functionalities(),
-            previous_unittests_issue,
+            unittests_issue,
             run_state=render_context.run_state,
             conformance_tests_fixes=conformance_tests_fixes,
         )

tests/test_fix_unit_tests_output_budget.py

@@ -0,0 +1,90 @@
+"""Tests for fitting unit test script output into the fixer's character budget.
+
+Output over the budget is cut from the middle, so the start (first failures) and the end (summary
+lines, lint findings) both reach the fixer, and a warning tells the script author it happened.
+"""
+
+import os
+import tempfile
+from types import SimpleNamespace
+
+import pytest
+
+from render_machine.actions import fix_unit_tests as fix_unit_tests_module
+from render_machine.actions.fix_unit_tests import MAX_ISSUE_LENGTH, FixUnitTests
+from render_machine.render_types import UnitTestsRunningContext
+
+
+class FakeCodeplainAPI:
+    def __init__(self):
+        self.sent_issues = []
+
+    def fix_unittests_issue(
+        self,
+        frid,
+        plain_source_tree,
+        linked_resources,
+        existing_files_content,
+        module_name,
+        required_modules,
+        unittests_issue,
+        run_state,
+        conformance_tests_fixes=None,
+    ):
+        self.sent_issues.append(unittests_issue)
+        return {}
+
+
+@pytest.fixture
+def build_folder():
+    with tempfile.TemporaryDirectory() as folder:
+        with open(os.path.join(folder, "app.py"), "w", encoding="utf-8") as source_file:
+            source_file.write("def add(a, b):\n    return a + b\n")
+        yield folder
+
+
+@pytest.fixture
+def warnings(monkeypatch):
+    logged = []
+    monkeypatch.setattr(fix_unit_tests_module.render_utils, "print_inputs", lambda *args: None)
+    monkeypatch.setattr(
+        fix_unit_tests_module.console, "warning", lambda *args, **kwargs: logged.append(" ".join(map(str, args)))
+    )
+    return logged
+
+
+def run_fix(build_folder, unittests_issue):
+    api = FakeCodeplainAPI()
+    render_context = SimpleNamespace(
+        codeplain_api=api,
+        build_folder=build_folder,
+        unit_tests_running_context=UnitTestsRunningContext(fix_attempts=1),
+        plain_source_tree={},
+        module_name="mod",
+        frid_context=SimpleNamespace(frid="1", linked_resources={}),
+        get_required_modules_functionalities=lambda: {},
+        run_state=SimpleNamespace(render_id="test-render-id", unittest_batch_id=1),
+    )
+    FixUnitTests().execute(render_context, {"previous_unittests_issue": unittests_issue})
+    return api.sent_issues[0]
+
+
+def test_output_within_budget_is_sent_whole_without_a_warning(build_folder, warnings):
+    unittests_issue = "F" * MAX_ISSUE_LENGTH
+
+    assert run_fix(build_folder, unittests_issue) == unittests_issue
+    assert warnings == []
+
+
+def test_output_over_budget_keeps_start_and_end_and_warns_with_sizes(build_folder, warnings):
+    unittests_issue = "1) User is valid\n" + "." * 20_000 + "\napp/models/user.rb:3:1: C: Style/Foo: offense"
+
+    sent_issue = run_fix(build_folder, unittests_issue)
+
+    assert len(sent_issue) <= MAX_ISSUE_LENGTH
+    assert sent_issue.startswith("1) User is valid\n")
+    assert sent_issue.endswith("app/models/user.rb:3:1: C: Style/Foo: offense")
+    assert "characters omitted" in sent_issue
+    assert len(warnings) == 1
+    assert f"{len(unittests_issue):,} characters" in warnings[0]
+    assert f"{MAX_ISSUE_LENGTH:,}-character budget" in warnings[0]

tests/test_plain2code_utils.py

@@ -1,6 +1,12 @@
 """Tests for pure helpers in plain2code_utils."""
 
-from plain2code_utils import format_duration_hms
+import re
+
+import pytest
+
+from plain2code_utils import format_duration_hms, truncate_middle
+
+OMISSION_MARKER = re.compile(r"\n\n\[\.\.\. ([\d,]+) characters omitted \.\.\.\]\n\n")
 
 
 class TestFormatDurationHms:
@@ -34,3 +40,78 @@ def test_float_input_is_truncated(self):
 
     def test_negative_input_clamped_to_zero(self):
         assert format_duration_hms(-5) == "0s"
+
+
+def split_at_marker(truncated: str) -> tuple[str, int, str]:
+    """Splits truncated text into the kept start, the reported omitted count, and the kept end."""
+    match = OMISSION_MARKER.search(truncated)
+    assert match, f"no omission marker in {truncated!r}"
+    return truncated[: match.start()], int(match.group(1).replace(",", "")), truncated[match.end() :]
+
+
+def numbered_text(length: int) -> str:
+    """Text whose every position is distinguishable, so a wrong slice cannot pass by accident."""
+    return "".join(str(i % 10) if i % 7 else "\n" for i in range(length))
+
+
+class TestTruncateMiddle:
+    def test_text_under_budget_is_unchanged(self):
+        assert truncate_middle("1 failed", 100) == ("1 failed", 0)
+
+    def test_empty_text_is_unchanged(self):
+        assert truncate_middle("", 100) == ("", 0)
+
+    def test_text_exactly_at_budget_is_unchanged(self):
+        text = numbered_text(100)
+        assert truncate_middle(text, 100) == (text, 0)
+
+    def test_text_one_over_budget_fits_and_keeps_both_ends(self):
+        text = "S" + "m" * 99 + "E"
+
+        truncated, omitted = truncate_middle(text, 100)
+
+        assert len(truncated) <= 100
+        assert truncated.startswith("S")
+        assert truncated.endswith("E")
+        assert omitted > 0
+
+    def test_kept_parts_are_the_exact_start_and_end_of_the_text(self):
+        text = numbered_text(1000)
+
+        truncated, omitted = truncate_middle(text, 200)
+
+        start, reported, end = split_at_marker(truncated)
+        assert text.startswith(start)
+        assert text.endswith(end)
+        assert reported == omitted
+        assert len(start) + omitted + len(end) == len(text)
+
+    def test_start_and_end_get_equal_shares(self):
+        start, _, end = split_at_marker(truncate_middle(numbered_text(1000), 200)[0])
+
+        assert len(start) == len(end)
+
+    def test_very_long_text_fits_budget_with_a_thousands_separated_count(self):
+        text = numbered_text(5_000_000)
+
+        truncated, omitted = truncate_middle(text, 10_000)
+
+        assert len(truncated) <= 10_000
+        start, _, end = split_at_marker(truncated)
+        assert len(start) + omitted + len(end) == 5_000_000
+        assert f"[... {omitted:,} characters omitted ...]" in truncated
+        assert omitted > 4_990_000
+
+    def test_findings_printed_last_survive(self):
+        output = "".join(f"spec/models/user_spec.rb:{i} example passed\n" for i in range(1000))
+        output += "app/models/user.rb:3:1: C: Style/FrozenStringLiteralComment: Missing magic comment.\n"
+
+        truncated, _ = truncate_middle(output, 10_000)
+
+        assert truncated.endswith(
+            "app/models/user.rb:3:1: C: Style/FrozenStringLiteralComment: Missing magic comment.\n"
+        )
+
+    def test_budget_too_small_for_the_marker_is_rejected(self):
+        with pytest.raises(ValueError):
+            truncate_middle(numbered_text(1000), 20)