From 4962e05e782be52dd37d1e4f48c2be2b71244b6c Mon Sep 17 00:00:00 2001 From: Goran Dodig Date: Wed, 14 Oct 2026 11:56:36 +0200 Subject: [PATCH] feat: warn when unit test output is cut, and keep its end Unit test output over the fixer's 10,000-character budget was handed to the server whole, which summarized it with an LLM (or kept the first 85%) and left only a DEBUG line behind. Lint findings and summary lines, which scripts print last, were the part most often lost, and script authors could not see that it happened. The client now cuts output over the budget from the middle, keeping equal shares of the start and the end with a marker saying how much was left out, and logs a WARNING with the original size and the budget. The budget and how to stay within it are documented for test script authors. --- docs/plain2code_cli.md | 5 +- docs/starting_a_plain_project_from_scratch.md | 10 +++ plain2code_arguments.py | 3 +- plain2code_utils.py | 23 +++++ render_machine/actions/fix_unit_tests.py | 13 ++- tests/test_fix_unit_tests_output_budget.py | 90 +++++++++++++++++++ tests/test_plain2code_utils.py | 83 ++++++++++++++++- 7 files changed, 220 insertions(+), 7 deletions(-) create mode 100644 tests/test_fix_unit_tests_output_budget.py diff --git a/docs/plain2code_cli.md b/docs/plain2code_cli.md index 01764ad8..281c37fa 100644 --- a/docs/plain2code_cli.md +++ b/docs/plain2code_cli.md @@ -66,7 +66,10 @@ options: --unittests-script UNITTESTS_SCRIPT Shell script to run unit tests on generated code. Receives the module's code folder path as its first - argument (e.g. `plain_modules/module_name/code`). + argument (e.g. `plain_modules/module_name/code`). When + it fails, the fixer gets at most 10,000 characters of + its output: the start and the end, with the middle + left out. --conformance-tests-script CONFORMANCE_TESTS_SCRIPT Path to conformance tests shell script. Every conformance test script should accept two arguments: diff --git a/docs/starting_a_plain_project_from_scratch.md b/docs/starting_a_plain_project_from_scratch.md index c5e6225c..cfcd3752 100644 --- a/docs/starting_a_plain_project_from_scratch.md +++ b/docs/starting_a_plain_project_from_scratch.md @@ -72,6 +72,16 @@ cp /path/to/plain2code_client/test_scripts/run_conformance_tests_python.sh ./ ``` You may need to modify these scripts based on your specific project requirements. +### Keep test output within the 10,000-character budget + +When a unit test script fails, its output goes to the fixer, which gets at most 10,000 characters of it. Longer output is cut from the middle: the fixer gets about the first and the last 4,980 characters, with a marker saying how many characters were left out. A warning in the log gives the full size each time this happens. Conformance test output over the same budget is summarized on the server instead. + +Output the fixer cannot act on uses up the budget and pushes the lines that matter into the middle, where they are cut. To keep the whole output within the budget: + +- Print a passing test run as its summary line only, and a failing one in full. A passing run still reaches the fixer when something else in the script fails, such as the linter. +- Print lint results as counts per rule, then one line per offense. For RuboCop: `rubocop --format offenses --format emacs`. +- Leave out coverage tables, slowest-test reports, and progress output. + ## 3. Configure Parameters Create a `config.yaml` (default name, which you can change with `--config-name` argument in the file) file to configure the plain2code CLI parameters. diff --git a/plain2code_arguments.py b/plain2code_arguments.py index a33086b5..a2a112ff 100644 --- a/plain2code_arguments.py +++ b/plain2code_arguments.py @@ -299,7 +299,8 @@ def create_parser(): parser, "--unittests-script", type=str, - help="Shell script to run unit tests on generated code. Receives the module's code folder path as its first argument (e.g. `plain_modules/module_name/code`).", + help="Shell script to run unit tests on generated code. Receives the module's code folder path as its first argument (e.g. `plain_modules/module_name/code`). " + "When it fails, the fixer gets at most 10,000 characters of its output: the start and the end, with the middle left out.", ) _add_arg( parser, diff --git a/plain2code_utils.py b/plain2code_utils.py index 9425cf2c..bb4f56bc 100644 --- a/plain2code_utils.py +++ b/plain2code_utils.py @@ -35,6 +35,29 @@ def format_duration_hms(total_seconds: float) -> str: return f"{hours}h {minutes % 60}m" +def _omission_marker(omitted: int) -> str: + return f"\n\n[... {omitted:,} characters omitted ...]\n\n" + + +def truncate_middle(text: str, max_length: int) -> tuple[str, int]: + """Fit text into max_length characters by cutting the middle out, keeping equal shares of start and end. + + Test output carries the first failures at the start and the summary lines and lint findings at the + end, so both ends are kept. A marker in place of the cut says how many characters were left out. + Returns the fitted text and the number of characters left out (0 when the text already fits). + """ + if len(text) <= max_length: + return text, 0 + + # The marker's length depends on the count it reports, so leave room for the largest possible count. + kept_at_each_end = (max_length - len(_omission_marker(len(text)))) // 2 + if kept_at_each_end < 1: + raise ValueError(f"A budget of {max_length} characters cannot hold the omission marker.") + + omitted = len(text) - 2 * kept_at_each_end + return text[:kept_at_each_end] + _omission_marker(omitted) + text[-kept_at_each_end:], omitted + + AMBIGUITY_CAUSES = { "reference_resource_ambiguity": "Ambiguity is in the reference resources", "definition_ambiguity": "Ambiguity is in the definitions", diff --git a/render_machine/actions/fix_unit_tests.py b/render_machine/actions/fix_unit_tests.py index 50a54b6a..623a2778 100644 --- a/render_machine/actions/fix_unit_tests.py +++ b/render_machine/actions/fix_unit_tests.py @@ -4,10 +4,12 @@ import render_machine.render_utils as render_utils from plain2code_console import console from plain2code_exceptions import InternalClientError +from plain2code_utils import truncate_middle from render_machine.actions.base_action import BaseAction from render_machine.implementation_code_helpers import ImplementationCodeHelpers from render_machine.render_context import RenderContext +# Same as MAX_ISSUE_LENGTH on the server: output within it reaches the fixer as is, without an LLM summary. MAX_ISSUE_LENGTH = 10000 @@ -21,9 +23,12 @@ def execute(self, render_context: RenderContext, previous_action_payload: Any | ) previous_unittests_issue = previous_action_payload["previous_unittests_issue"] - if previous_unittests_issue and len(previous_unittests_issue) > MAX_ISSUE_LENGTH: - console.debug( - f"Unit tests issue text is too long and will be smartly truncated to {MAX_ISSUE_LENGTH} characters." + unittests_issue, omitted_characters = truncate_middle(previous_unittests_issue, MAX_ISSUE_LENGTH) + if omitted_characters: + console.warning( + f"Unit test output is {len(previous_unittests_issue):,} characters, over the " + f"{MAX_ISSUE_LENGTH:,}-character budget. The fixer gets the start and the end; " + f"{omitted_characters:,} characters in the middle are left out." ) existing_files, existing_files_content = ImplementationCodeHelpers.fetch_existing_files( @@ -46,7 +51,7 @@ def execute(self, render_context: RenderContext, previous_action_payload: Any | existing_files_content, render_context.module_name, render_context.get_required_modules_functionalities(), - previous_unittests_issue, + unittests_issue, run_state=render_context.run_state, conformance_tests_fixes=conformance_tests_fixes, ) diff --git a/tests/test_fix_unit_tests_output_budget.py b/tests/test_fix_unit_tests_output_budget.py new file mode 100644 index 00000000..7daf6234 --- /dev/null +++ b/tests/test_fix_unit_tests_output_budget.py @@ -0,0 +1,90 @@ +"""Tests for fitting unit test script output into the fixer's character budget. + +Output over the budget is cut from the middle, so the start (first failures) and the end (summary +lines, lint findings) both reach the fixer, and a warning tells the script author it happened. +""" + +import os +import tempfile +from types import SimpleNamespace + +import pytest + +from render_machine.actions import fix_unit_tests as fix_unit_tests_module +from render_machine.actions.fix_unit_tests import MAX_ISSUE_LENGTH, FixUnitTests +from render_machine.render_types import UnitTestsRunningContext + + +class FakeCodeplainAPI: + def __init__(self): + self.sent_issues = [] + + def fix_unittests_issue( + self, + frid, + plain_source_tree, + linked_resources, + existing_files_content, + module_name, + required_modules, + unittests_issue, + run_state, + conformance_tests_fixes=None, + ): + self.sent_issues.append(unittests_issue) + return {} + + +@pytest.fixture +def build_folder(): + with tempfile.TemporaryDirectory() as folder: + with open(os.path.join(folder, "app.py"), "w", encoding="utf-8") as source_file: + source_file.write("def add(a, b):\n return a + b\n") + yield folder + + +@pytest.fixture +def warnings(monkeypatch): + logged = [] + monkeypatch.setattr(fix_unit_tests_module.render_utils, "print_inputs", lambda *args: None) + monkeypatch.setattr( + fix_unit_tests_module.console, "warning", lambda *args, **kwargs: logged.append(" ".join(map(str, args))) + ) + return logged + + +def run_fix(build_folder, unittests_issue): + api = FakeCodeplainAPI() + render_context = SimpleNamespace( + codeplain_api=api, + build_folder=build_folder, + unit_tests_running_context=UnitTestsRunningContext(fix_attempts=1), + plain_source_tree={}, + module_name="mod", + frid_context=SimpleNamespace(frid="1", linked_resources={}), + get_required_modules_functionalities=lambda: {}, + run_state=SimpleNamespace(render_id="test-render-id", unittest_batch_id=1), + ) + FixUnitTests().execute(render_context, {"previous_unittests_issue": unittests_issue}) + return api.sent_issues[0] + + +def test_output_within_budget_is_sent_whole_without_a_warning(build_folder, warnings): + unittests_issue = "F" * MAX_ISSUE_LENGTH + + assert run_fix(build_folder, unittests_issue) == unittests_issue + assert warnings == [] + + +def test_output_over_budget_keeps_start_and_end_and_warns_with_sizes(build_folder, warnings): + unittests_issue = "1) User is valid\n" + "." * 20_000 + "\napp/models/user.rb:3:1: C: Style/Foo: offense" + + sent_issue = run_fix(build_folder, unittests_issue) + + assert len(sent_issue) <= MAX_ISSUE_LENGTH + assert sent_issue.startswith("1) User is valid\n") + assert sent_issue.endswith("app/models/user.rb:3:1: C: Style/Foo: offense") + assert "characters omitted" in sent_issue + assert len(warnings) == 1 + assert f"{len(unittests_issue):,} characters" in warnings[0] + assert f"{MAX_ISSUE_LENGTH:,}-character budget" in warnings[0] diff --git a/tests/test_plain2code_utils.py b/tests/test_plain2code_utils.py index d56b3d74..3bb47a44 100644 --- a/tests/test_plain2code_utils.py +++ b/tests/test_plain2code_utils.py @@ -1,6 +1,12 @@ """Tests for pure helpers in plain2code_utils.""" -from plain2code_utils import format_duration_hms +import re + +import pytest + +from plain2code_utils import format_duration_hms, truncate_middle + +OMISSION_MARKER = re.compile(r"\n\n\[\.\.\. ([\d,]+) characters omitted \.\.\.\]\n\n") class TestFormatDurationHms: @@ -34,3 +40,78 @@ def test_float_input_is_truncated(self): def test_negative_input_clamped_to_zero(self): assert format_duration_hms(-5) == "0s" + + +def split_at_marker(truncated: str) -> tuple[str, int, str]: + """Splits truncated text into the kept start, the reported omitted count, and the kept end.""" + match = OMISSION_MARKER.search(truncated) + assert match, f"no omission marker in {truncated!r}" + return truncated[: match.start()], int(match.group(1).replace(",", "")), truncated[match.end() :] + + +def numbered_text(length: int) -> str: + """Text whose every position is distinguishable, so a wrong slice cannot pass by accident.""" + return "".join(str(i % 10) if i % 7 else "\n" for i in range(length)) + + +class TestTruncateMiddle: + def test_text_under_budget_is_unchanged(self): + assert truncate_middle("1 failed", 100) == ("1 failed", 0) + + def test_empty_text_is_unchanged(self): + assert truncate_middle("", 100) == ("", 0) + + def test_text_exactly_at_budget_is_unchanged(self): + text = numbered_text(100) + assert truncate_middle(text, 100) == (text, 0) + + def test_text_one_over_budget_fits_and_keeps_both_ends(self): + text = "S" + "m" * 99 + "E" + + truncated, omitted = truncate_middle(text, 100) + + assert len(truncated) <= 100 + assert truncated.startswith("S") + assert truncated.endswith("E") + assert omitted > 0 + + def test_kept_parts_are_the_exact_start_and_end_of_the_text(self): + text = numbered_text(1000) + + truncated, omitted = truncate_middle(text, 200) + + start, reported, end = split_at_marker(truncated) + assert text.startswith(start) + assert text.endswith(end) + assert reported == omitted + assert len(start) + omitted + len(end) == len(text) + + def test_start_and_end_get_equal_shares(self): + start, _, end = split_at_marker(truncate_middle(numbered_text(1000), 200)[0]) + + assert len(start) == len(end) + + def test_very_long_text_fits_budget_with_a_thousands_separated_count(self): + text = numbered_text(5_000_000) + + truncated, omitted = truncate_middle(text, 10_000) + + assert len(truncated) <= 10_000 + start, _, end = split_at_marker(truncated) + assert len(start) + omitted + len(end) == 5_000_000 + assert f"[... {omitted:,} characters omitted ...]" in truncated + assert omitted > 4_990_000 + + def test_findings_printed_last_survive(self): + output = "".join(f"spec/models/user_spec.rb:{i} example passed\n" for i in range(1000)) + output += "app/models/user.rb:3:1: C: Style/FrozenStringLiteralComment: Missing magic comment.\n" + + truncated, _ = truncate_middle(output, 10_000) + + assert truncated.endswith( + "app/models/user.rb:3:1: C: Style/FrozenStringLiteralComment: Missing magic comment.\n" + ) + + def test_budget_too_small_for_the_marker_is_rejected(self): + with pytest.raises(ValueError): + truncate_middle(numbered_text(1000), 20)