|
| 1 | +"""A record of what has already been tried while fixing one functionality's conformance tests. |
| 2 | +
|
| 3 | +The fixer sees the current test output, the current test code and the current implementation. What it cannot |
| 4 | +see is everything that led to them: the changes made in earlier rounds, whether each one moved the failure, |
| 5 | +and which hypotheses are already exhausted. Without that it re-tries approaches that have already failed, and |
| 6 | +the cumulative diff of the reverted attempts is invisible because only the surviving state remains on disk. |
| 7 | +
|
| 8 | +The journal is written mechanically from data the renderer already holds - the diff of each fix, the reason |
| 9 | +code saying whether the fix targeted the tests or the implementation, and the failure that prompted it. No |
| 10 | +model call is involved in maintaining it. |
| 11 | +
|
| 12 | +Failure text is stored once per distinct failure and referenced by signature, so a twenty round loop that |
| 13 | +cycles through three failures stores three excerpts rather than twenty. That is also what makes "this is the |
| 14 | +failure we already saw in round 4" fall out of the record instead of having to be inferred. |
| 15 | +
|
| 16 | +The journal covers one functionality and is discarded once that functionality's tests pass, at which point |
| 17 | +the durable lessons are extracted from it. It is deliberately kept outside the folder memory files are read |
| 18 | +from, so that it is fed to a prompt only where it is wanted. |
| 19 | +""" |
| 20 | + |
| 21 | +import json |
| 22 | +import os |
| 23 | +from typing import Any, Optional |
| 24 | + |
| 25 | +import failure_signature |
| 26 | +from plain2code_console import console |
| 27 | + |
| 28 | +JOURNAL_SUBFOLDER = "conformance_test_journal" |
| 29 | + |
| 30 | +# Distinct failures whose text is retained. A fix loop that produces more distinct failures than this is |
| 31 | +# thrashing, and the oldest ones have stopped being informative. |
| 32 | +MAX_DISTINCT_ISSUES = 8 |
| 33 | + |
| 34 | +# Failure excerpts are held for the whole loop and several may be live at once, so they are capped tighter |
| 35 | +# than a single excerpt shown on its own would be. |
| 36 | +ISSUE_EXCERPT_MAX_LINES = 40 |
| 37 | +ISSUE_EXCERPT_MAX_CHARS = 2000 |
| 38 | + |
| 39 | +# Rounds whose diff is kept in full. Older changes are usually still visible in the current code; what the |
| 40 | +# recent diffs uniquely preserve are the intermediate and reverted states that are not. |
| 41 | +ROUNDS_WITH_FULL_DIFF = 5 |
| 42 | +DIFF_MAX_CHARS = 1500 |
| 43 | + |
| 44 | +TARGET_CONFORMANCE_TESTS = "conformance tests" |
| 45 | +TARGET_IMPLEMENTATION = "implementation" |
| 46 | + |
| 47 | +PROMPT_FILE_NAME = "conformance_test_journal.md" |
| 48 | + |
| 49 | + |
| 50 | +def _safe_name(value: str) -> str: |
| 51 | + return "".join(character if character.isalnum() or character in "._-" else "_" for character in value) |
| 52 | + |
| 53 | + |
| 54 | +class ConformanceTestJournal: |
| 55 | + """Append-only record of the fix rounds for one (module, functionality) pair.""" |
| 56 | + |
| 57 | + def __init__(self, module_name: str, frid: str, attempts: Optional[list] = None, issues: Optional[dict] = None): |
| 58 | + self.module_name = module_name |
| 59 | + self.frid = frid |
| 60 | + self.attempts: list[dict[str, Any]] = attempts or [] |
| 61 | + # signature (or a per-round stand-in) -> failure excerpt |
| 62 | + self.issues: dict[str, str] = issues or {} |
| 63 | + |
| 64 | + @staticmethod |
| 65 | + def journal_path(memory_folder: str, module_name: str, frid: str) -> str: |
| 66 | + return os.path.join(memory_folder, JOURNAL_SUBFOLDER, _safe_name(module_name), f"{_safe_name(frid)}.json") |
| 67 | + |
| 68 | + @classmethod |
| 69 | + def load(cls, memory_folder: str, module_name: str, frid: str) -> "ConformanceTestJournal": |
| 70 | + path = cls.journal_path(memory_folder, module_name, frid) |
| 71 | + if not os.path.exists(path): |
| 72 | + return cls(module_name, frid) |
| 73 | + |
| 74 | + try: |
| 75 | + with open(path, "r", encoding="utf-8") as journal_file: |
| 76 | + content = json.load(journal_file) |
| 77 | + return cls( |
| 78 | + module_name=content.get("module", module_name), |
| 79 | + frid=content.get("frid", frid), |
| 80 | + attempts=content.get("attempts", []), |
| 81 | + issues=content.get("issues", {}), |
| 82 | + ) |
| 83 | + except (json.JSONDecodeError, OSError, AttributeError) as exception: |
| 84 | + console.debug(f"Could not read the conformance test journal at {path}: {exception}. Starting a new one.") |
| 85 | + return cls(module_name, frid) |
| 86 | + |
| 87 | + def save(self, memory_folder: str) -> None: |
| 88 | + path = self.journal_path(memory_folder, self.module_name, self.frid) |
| 89 | + try: |
| 90 | + os.makedirs(os.path.dirname(path), exist_ok=True) |
| 91 | + with open(path, "w", encoding="utf-8") as journal_file: |
| 92 | + json.dump( |
| 93 | + {"module": self.module_name, "frid": self.frid, "issues": self.issues, "attempts": self.attempts}, |
| 94 | + journal_file, |
| 95 | + indent=2, |
| 96 | + ) |
| 97 | + except OSError as exception: |
| 98 | + console.debug(f"Could not write the conformance test journal to {path}: {exception}.") |
| 99 | + |
| 100 | + def delete(self, memory_folder: str) -> None: |
| 101 | + path = self.journal_path(memory_folder, self.module_name, self.frid) |
| 102 | + try: |
| 103 | + if os.path.exists(path): |
| 104 | + os.remove(path) |
| 105 | + except OSError as exception: |
| 106 | + console.debug(f"Could not delete the conformance test journal at {path}: {exception}.") |
| 107 | + |
| 108 | + def record_attempt( |
| 109 | + self, |
| 110 | + target: str, |
| 111 | + files_changed: list[str], |
| 112 | + diff_text: Optional[str] = None, |
| 113 | + issue_signature: Optional[str] = None, |
| 114 | + issue_excerpt: Optional[str] = None, |
| 115 | + ) -> None: |
| 116 | + """Record one fix round: the failure that prompted it and the change it made in response.""" |
| 117 | + round_number = len(self.attempts) + 1 |
| 118 | + |
| 119 | + # Without a signature the failure cannot be recognised as one seen before, but its text is still |
| 120 | + # worth keeping, so it is filed under a key that will not collide with another round's. |
| 121 | + issue_key = issue_signature or f"round-{round_number}" |
| 122 | + if issue_excerpt: |
| 123 | + self.issues.setdefault(issue_key, issue_excerpt) |
| 124 | + |
| 125 | + self.attempts.append( |
| 126 | + { |
| 127 | + "round": round_number, |
| 128 | + "target": target, |
| 129 | + "files_changed": sorted(files_changed), |
| 130 | + "diff": diff_text[:DIFF_MAX_CHARS] if diff_text else None, |
| 131 | + "issue": issue_key if issue_excerpt or issue_key in self.issues else None, |
| 132 | + "issue_identified": issue_signature is not None, |
| 133 | + } |
| 134 | + ) |
| 135 | + |
| 136 | + self._drop_diffs_from_older_rounds() |
| 137 | + self._evict_surplus_issues() |
| 138 | + |
| 139 | + def _drop_diffs_from_older_rounds(self) -> None: |
| 140 | + for attempt in self.attempts[:-ROUNDS_WITH_FULL_DIFF]: |
| 141 | + attempt["diff"] = None |
| 142 | + |
| 143 | + def _evict_surplus_issues(self) -> None: |
| 144 | + if len(self.issues) <= MAX_DISTINCT_ISSUES: |
| 145 | + return |
| 146 | + |
| 147 | + # Keep the failures referenced most recently; those are the ones still in play. |
| 148 | + most_recent_use: dict[str, int] = {} |
| 149 | + for attempt in self.attempts: |
| 150 | + if attempt.get("issue"): |
| 151 | + most_recent_use[attempt["issue"]] = attempt["round"] |
| 152 | + |
| 153 | + retained = sorted(self.issues, key=lambda key: most_recent_use.get(key, 0), reverse=True) |
| 154 | + self.issues = {key: self.issues[key] for key in retained[:MAX_DISTINCT_ISSUES]} |
| 155 | + |
| 156 | + def first_round_with_issue(self, issue_key: str, before_round: int) -> Optional[int]: |
| 157 | + """The earliest round that hit this same failure, if an earlier one did.""" |
| 158 | + for attempt in self.attempts: |
| 159 | + if attempt["round"] >= before_round: |
| 160 | + break |
| 161 | + if attempt.get("issue") == issue_key and attempt.get("issue_identified"): |
| 162 | + return int(attempt["round"]) |
| 163 | + return None |
| 164 | + |
| 165 | + def render_for_prompt(self) -> Optional[str]: |
| 166 | + """The journal as prose, or None when there is nothing worth saying yet.""" |
| 167 | + if not self.attempts: |
| 168 | + return None |
| 169 | + |
| 170 | + lines = [ |
| 171 | + f"# Previous attempts at fixing the conformance tests for functionality {self.frid} " |
| 172 | + f"in module {self.module_name}", |
| 173 | + "", |
| 174 | + "These attempts have already been made. Approaches recorded here as having failed should not be " |
| 175 | + "repeated unless there is a specific reason to believe the circumstances have changed.", |
| 176 | + "", |
| 177 | + ] |
| 178 | + |
| 179 | + for attempt in self.attempts: |
| 180 | + round_number = attempt["round"] |
| 181 | + lines.append(f"## Attempt {round_number}: changed the {attempt['target']}") |
| 182 | + |
| 183 | + if attempt["files_changed"]: |
| 184 | + lines.append(f"Files changed: {', '.join(attempt['files_changed'])}") |
| 185 | + else: |
| 186 | + lines.append("No files were changed.") |
| 187 | + |
| 188 | + issue_key = attempt.get("issue") |
| 189 | + if issue_key: |
| 190 | + repeated_from = self.first_round_with_issue(issue_key, round_number) |
| 191 | + if repeated_from is not None: |
| 192 | + lines.append( |
| 193 | + f"The failure that prompted this attempt was the same one already seen in attempt " |
| 194 | + f"{repeated_from}, so the changes made in between did not resolve it." |
| 195 | + ) |
| 196 | + excerpt = self.issues.get(issue_key) |
| 197 | + if excerpt and repeated_from is None: |
| 198 | + lines.append("") |
| 199 | + lines.append("The failure that prompted this attempt:") |
| 200 | + lines.append("```") |
| 201 | + lines.append(excerpt) |
| 202 | + lines.append("```") |
| 203 | + |
| 204 | + if attempt.get("diff"): |
| 205 | + lines.append("") |
| 206 | + lines.append("The change made:") |
| 207 | + lines.append("```") |
| 208 | + lines.append(attempt["diff"]) |
| 209 | + lines.append("```") |
| 210 | + |
| 211 | + lines.append("") |
| 212 | + |
| 213 | + return "\n".join(lines).rstrip() + "\n" |
| 214 | + |
| 215 | + |
| 216 | +def build_issue_excerpt(output: str) -> Optional[str]: |
| 217 | + return failure_signature.build_excerpt(output, max_lines=ISSUE_EXCERPT_MAX_LINES, max_chars=ISSUE_EXCERPT_MAX_CHARS) |
0 commit comments