Skip to content

Commit dc112fb

Browse files
committed
Record what has already been tried when fixing conformance tests
The fixer sees the current tests, the current implementation and the current failure. What it could not see was how any of them got that way: the changes made in earlier rounds, whether each one moved the failure, and which approaches are already exhausted. Memory was meant to carry that, but it replaced itself every round, so a twenty round loop entered each attempt knowing only about the one before it. A journal per (module, functionality) now records every round: whether the fix targeted the tests or the implementation, which files it touched, its diff, and the failure that prompted it. It is written from data the renderer already holds and read back into the fix call. No model call is involved. Failure text is stored once per distinct failure and referenced by signature, so a loop cycling through three failures stores three excerpts rather than twenty. "This is the failure we already saw in attempt 1" then falls out of the record instead of having to be inferred. Also corrects a flaw in the signature itself, found while wiring this up. The profile classified boilerplate by run frequency, but a fix loop re-runs one failure up to twenty times - so that failure reached 100% frequency and was discarded as furniture at exactly the point it was being tracked. Boilerplate is now decided by a sharper rule: a line that appears when the tests pass cannot be what identifies a failure. Frequency survives only as a fallback for projects with no green run yet, counted across functionalities rather than runs. Degrades rather than guesses: with no passing run and only one functionality seen there is no signature, so repeats go unrecognised - but every round's failure text is still kept in full.
1 parent 1454277 commit dc112fb

8 files changed

Lines changed: 997 additions & 73 deletions

‎conformance_test_journal.py‎

Lines changed: 217 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,217 @@
1+
"""A record of what has already been tried while fixing one functionality's conformance tests.
2+
3+
The fixer sees the current test output, the current test code and the current implementation. What it cannot
4+
see is everything that led to them: the changes made in earlier rounds, whether each one moved the failure,
5+
and which hypotheses are already exhausted. Without that it re-tries approaches that have already failed, and
6+
the cumulative diff of the reverted attempts is invisible because only the surviving state remains on disk.
7+
8+
The journal is written mechanically from data the renderer already holds - the diff of each fix, the reason
9+
code saying whether the fix targeted the tests or the implementation, and the failure that prompted it. No
10+
model call is involved in maintaining it.
11+
12+
Failure text is stored once per distinct failure and referenced by signature, so a twenty round loop that
13+
cycles through three failures stores three excerpts rather than twenty. That is also what makes "this is the
14+
failure we already saw in round 4" fall out of the record instead of having to be inferred.
15+
16+
The journal covers one functionality and is discarded once that functionality's tests pass, at which point
17+
the durable lessons are extracted from it. It is deliberately kept outside the folder memory files are read
18+
from, so that it is fed to a prompt only where it is wanted.
19+
"""
20+
21+
import json
22+
import os
23+
from typing import Any, Optional
24+
25+
import failure_signature
26+
from plain2code_console import console
27+
28+
JOURNAL_SUBFOLDER = "conformance_test_journal"
29+
30+
# Distinct failures whose text is retained. A fix loop that produces more distinct failures than this is
31+
# thrashing, and the oldest ones have stopped being informative.
32+
MAX_DISTINCT_ISSUES = 8
33+
34+
# Failure excerpts are held for the whole loop and several may be live at once, so they are capped tighter
35+
# than a single excerpt shown on its own would be.
36+
ISSUE_EXCERPT_MAX_LINES = 40
37+
ISSUE_EXCERPT_MAX_CHARS = 2000
38+
39+
# Rounds whose diff is kept in full. Older changes are usually still visible in the current code; what the
40+
# recent diffs uniquely preserve are the intermediate and reverted states that are not.
41+
ROUNDS_WITH_FULL_DIFF = 5
42+
DIFF_MAX_CHARS = 1500
43+
44+
TARGET_CONFORMANCE_TESTS = "conformance tests"
45+
TARGET_IMPLEMENTATION = "implementation"
46+
47+
PROMPT_FILE_NAME = "conformance_test_journal.md"
48+
49+
50+
def _safe_name(value: str) -> str:
51+
return "".join(character if character.isalnum() or character in "._-" else "_" for character in value)
52+
53+
54+
class ConformanceTestJournal:
55+
"""Append-only record of the fix rounds for one (module, functionality) pair."""
56+
57+
def __init__(self, module_name: str, frid: str, attempts: Optional[list] = None, issues: Optional[dict] = None):
58+
self.module_name = module_name
59+
self.frid = frid
60+
self.attempts: list[dict[str, Any]] = attempts or []
61+
# signature (or a per-round stand-in) -> failure excerpt
62+
self.issues: dict[str, str] = issues or {}
63+
64+
@staticmethod
65+
def journal_path(memory_folder: str, module_name: str, frid: str) -> str:
66+
return os.path.join(memory_folder, JOURNAL_SUBFOLDER, _safe_name(module_name), f"{_safe_name(frid)}.json")
67+
68+
@classmethod
69+
def load(cls, memory_folder: str, module_name: str, frid: str) -> "ConformanceTestJournal":
70+
path = cls.journal_path(memory_folder, module_name, frid)
71+
if not os.path.exists(path):
72+
return cls(module_name, frid)
73+
74+
try:
75+
with open(path, "r", encoding="utf-8") as journal_file:
76+
content = json.load(journal_file)
77+
return cls(
78+
module_name=content.get("module", module_name),
79+
frid=content.get("frid", frid),
80+
attempts=content.get("attempts", []),
81+
issues=content.get("issues", {}),
82+
)
83+
except (json.JSONDecodeError, OSError, AttributeError) as exception:
84+
console.debug(f"Could not read the conformance test journal at {path}: {exception}. Starting a new one.")
85+
return cls(module_name, frid)
86+
87+
def save(self, memory_folder: str) -> None:
88+
path = self.journal_path(memory_folder, self.module_name, self.frid)
89+
try:
90+
os.makedirs(os.path.dirname(path), exist_ok=True)
91+
with open(path, "w", encoding="utf-8") as journal_file:
92+
json.dump(
93+
{"module": self.module_name, "frid": self.frid, "issues": self.issues, "attempts": self.attempts},
94+
journal_file,
95+
indent=2,
96+
)
97+
except OSError as exception:
98+
console.debug(f"Could not write the conformance test journal to {path}: {exception}.")
99+
100+
def delete(self, memory_folder: str) -> None:
101+
path = self.journal_path(memory_folder, self.module_name, self.frid)
102+
try:
103+
if os.path.exists(path):
104+
os.remove(path)
105+
except OSError as exception:
106+
console.debug(f"Could not delete the conformance test journal at {path}: {exception}.")
107+
108+
def record_attempt(
109+
self,
110+
target: str,
111+
files_changed: list[str],
112+
diff_text: Optional[str] = None,
113+
issue_signature: Optional[str] = None,
114+
issue_excerpt: Optional[str] = None,
115+
) -> None:
116+
"""Record one fix round: the failure that prompted it and the change it made in response."""
117+
round_number = len(self.attempts) + 1
118+
119+
# Without a signature the failure cannot be recognised as one seen before, but its text is still
120+
# worth keeping, so it is filed under a key that will not collide with another round's.
121+
issue_key = issue_signature or f"round-{round_number}"
122+
if issue_excerpt:
123+
self.issues.setdefault(issue_key, issue_excerpt)
124+
125+
self.attempts.append(
126+
{
127+
"round": round_number,
128+
"target": target,
129+
"files_changed": sorted(files_changed),
130+
"diff": diff_text[:DIFF_MAX_CHARS] if diff_text else None,
131+
"issue": issue_key if issue_excerpt or issue_key in self.issues else None,
132+
"issue_identified": issue_signature is not None,
133+
}
134+
)
135+
136+
self._drop_diffs_from_older_rounds()
137+
self._evict_surplus_issues()
138+
139+
def _drop_diffs_from_older_rounds(self) -> None:
140+
for attempt in self.attempts[:-ROUNDS_WITH_FULL_DIFF]:
141+
attempt["diff"] = None
142+
143+
def _evict_surplus_issues(self) -> None:
144+
if len(self.issues) <= MAX_DISTINCT_ISSUES:
145+
return
146+
147+
# Keep the failures referenced most recently; those are the ones still in play.
148+
most_recent_use: dict[str, int] = {}
149+
for attempt in self.attempts:
150+
if attempt.get("issue"):
151+
most_recent_use[attempt["issue"]] = attempt["round"]
152+
153+
retained = sorted(self.issues, key=lambda key: most_recent_use.get(key, 0), reverse=True)
154+
self.issues = {key: self.issues[key] for key in retained[:MAX_DISTINCT_ISSUES]}
155+
156+
def first_round_with_issue(self, issue_key: str, before_round: int) -> Optional[int]:
157+
"""The earliest round that hit this same failure, if an earlier one did."""
158+
for attempt in self.attempts:
159+
if attempt["round"] >= before_round:
160+
break
161+
if attempt.get("issue") == issue_key and attempt.get("issue_identified"):
162+
return int(attempt["round"])
163+
return None
164+
165+
def render_for_prompt(self) -> Optional[str]:
166+
"""The journal as prose, or None when there is nothing worth saying yet."""
167+
if not self.attempts:
168+
return None
169+
170+
lines = [
171+
f"# Previous attempts at fixing the conformance tests for functionality {self.frid} "
172+
f"in module {self.module_name}",
173+
"",
174+
"These attempts have already been made. Approaches recorded here as having failed should not be "
175+
"repeated unless there is a specific reason to believe the circumstances have changed.",
176+
"",
177+
]
178+
179+
for attempt in self.attempts:
180+
round_number = attempt["round"]
181+
lines.append(f"## Attempt {round_number}: changed the {attempt['target']}")
182+
183+
if attempt["files_changed"]:
184+
lines.append(f"Files changed: {', '.join(attempt['files_changed'])}")
185+
else:
186+
lines.append("No files were changed.")
187+
188+
issue_key = attempt.get("issue")
189+
if issue_key:
190+
repeated_from = self.first_round_with_issue(issue_key, round_number)
191+
if repeated_from is not None:
192+
lines.append(
193+
f"The failure that prompted this attempt was the same one already seen in attempt "
194+
f"{repeated_from}, so the changes made in between did not resolve it."
195+
)
196+
excerpt = self.issues.get(issue_key)
197+
if excerpt and repeated_from is None:
198+
lines.append("")
199+
lines.append("The failure that prompted this attempt:")
200+
lines.append("```")
201+
lines.append(excerpt)
202+
lines.append("```")
203+
204+
if attempt.get("diff"):
205+
lines.append("")
206+
lines.append("The change made:")
207+
lines.append("```")
208+
lines.append(attempt["diff"])
209+
lines.append("```")
210+
211+
lines.append("")
212+
213+
return "\n".join(lines).rstrip() + "\n"
214+
215+
216+
def build_issue_excerpt(output: str) -> Optional[str]:
217+
return failure_signature.build_excerpt(output, max_lines=ISSUE_EXCERPT_MAX_LINES, max_chars=ISSUE_EXCERPT_MAX_CHARS)

0 commit comments

Comments
 (0)