|
| 1 | +"""Ranked retrieval over evidential memory records. |
| 2 | +
|
| 3 | +Two views are queried and unioned, following the multi-view indexing idea without the |
| 4 | +cost of embeddings: |
| 5 | +
|
| 6 | +* the **symbolic** view matches on recorded facts - same failure fingerprint, same test, |
| 7 | + overlapping files - and is the strongest signal for test failures, because an identical |
| 8 | + fingerprint is an exact match rather than a similarity guess |
| 9 | +* the **lexical** view scores normalized failure signatures with BM25, which is what |
| 10 | + surfaces near-miss failures the symbolic view cannot see |
| 11 | +
|
| 12 | +Retrieval depth adapts to how stuck the render is. Unlike a conversational agent we do |
| 13 | +not have to infer intent: the render already knows the current failure and how many fix |
| 14 | +attempts have been made, so no planner call is needed. |
| 15 | +""" |
| 16 | + |
| 17 | +from __future__ import annotations |
| 18 | + |
| 19 | +import math |
| 20 | +import re |
| 21 | +from collections import Counter |
| 22 | +from enum import Enum |
| 23 | +from typing import Iterable, Optional |
| 24 | + |
| 25 | +from memory_management.record import MemoryRecord, Status |
| 26 | + |
| 27 | +# Retrieval depth by number of fix attempts already made. A first pass retrieves nothing: |
| 28 | +# there is no failure to match against yet, and injecting memory speculatively is how |
| 29 | +# context gets wasted. |
| 30 | +_DEPTH_BY_ATTEMPTS = ((0, 0), (2, 3), (5, 6)) |
| 31 | +_MAX_DEPTH = 12 |
| 32 | + |
| 33 | +_BM25_K1 = 1.5 |
| 34 | +_BM25_B = 0.75 |
| 35 | +_TOKEN_PATTERN = re.compile(r"[a-z0-9_]+") |
| 36 | + |
| 37 | + |
| 38 | +class MemoryMode(str, Enum): |
| 39 | + """Which records may be retrieved. Exposed as ``--memory-mode`` for benchmarking.""" |
| 40 | + |
| 41 | + OFF = "off" |
| 42 | + VERIFIED = "verified" |
| 43 | + REFUTED = "refuted" |
| 44 | + ALL = "all" |
| 45 | + |
| 46 | + |
| 47 | +def retrieval_depth(fix_attempts: int) -> int: |
| 48 | + """How many records to retrieve, scaled to how stuck the current fix loop is.""" |
| 49 | + for threshold, depth in _DEPTH_BY_ATTEMPTS: |
| 50 | + if fix_attempts <= threshold: |
| 51 | + return depth |
| 52 | + return _MAX_DEPTH |
| 53 | + |
| 54 | + |
| 55 | +def _allowed_statuses(mode: MemoryMode) -> set[str]: |
| 56 | + if mode is MemoryMode.OFF: |
| 57 | + return set() |
| 58 | + if mode is MemoryMode.VERIFIED: |
| 59 | + return {Status.VERIFIED.value} |
| 60 | + if mode is MemoryMode.REFUTED: |
| 61 | + return {Status.REFUTED.value} |
| 62 | + return {Status.VERIFIED.value, Status.REFUTED.value} |
| 63 | + |
| 64 | + |
| 65 | +def _tokenize(text: str) -> list[str]: |
| 66 | + return _TOKEN_PATTERN.findall(text.lower()) |
| 67 | + |
| 68 | + |
| 69 | +def _bm25_scores(query: str, records: list[MemoryRecord]) -> dict[str, float]: |
| 70 | + """Score records against the query signature. Corpus is one render, so this is cheap.""" |
| 71 | + query_tokens = _tokenize(query) |
| 72 | + if not query_tokens or not records: |
| 73 | + return {} |
| 74 | + |
| 75 | + documents = {record.memory_id: _tokenize(record.failure.signature) for record in records} |
| 76 | + lengths = [len(tokens) for tokens in documents.values()] |
| 77 | + average_length = sum(lengths) / len(lengths) if lengths else 0.0 |
| 78 | + if average_length == 0.0: |
| 79 | + return {} |
| 80 | + |
| 81 | + document_frequency: Counter = Counter() |
| 82 | + for tokens in documents.values(): |
| 83 | + for token in set(tokens): |
| 84 | + document_frequency[token] += 1 |
| 85 | + |
| 86 | + total_documents = len(documents) |
| 87 | + scores: dict[str, float] = {} |
| 88 | + for memory_id, tokens in documents.items(): |
| 89 | + if not tokens: |
| 90 | + continue |
| 91 | + term_frequency = Counter(tokens) |
| 92 | + score = 0.0 |
| 93 | + for token in query_tokens: |
| 94 | + occurrences = term_frequency.get(token, 0) |
| 95 | + if occurrences == 0: |
| 96 | + continue |
| 97 | + matching_documents = document_frequency[token] |
| 98 | + idf = math.log(1 + (total_documents - matching_documents + 0.5) / (matching_documents + 0.5)) |
| 99 | + denominator = occurrences + _BM25_K1 * (1 - _BM25_B + _BM25_B * len(tokens) / average_length) |
| 100 | + score += idf * (occurrences * (_BM25_K1 + 1)) / denominator |
| 101 | + if score > 0.0: |
| 102 | + scores[memory_id] = score |
| 103 | + |
| 104 | + return scores |
| 105 | + |
| 106 | + |
| 107 | +def _symbolic_tier( |
| 108 | + record: MemoryRecord, |
| 109 | + fingerprint: Optional[str], |
| 110 | + test_name: Optional[str], |
| 111 | + files_changed: Optional[Iterable[str]], |
| 112 | +) -> Optional[int]: |
| 113 | + """Lower tiers are stronger matches. ``None`` means the symbolic view does not match.""" |
| 114 | + if fingerprint is not None and record.failure.fingerprint == fingerprint: |
| 115 | + # An exact fingerprint match on a verified intervention is the single most useful |
| 116 | + # record we can surface, so it outranks everything else. |
| 117 | + return 0 if record.status == Status.VERIFIED.value else 1 |
| 118 | + if test_name is not None and record.scope.test_name == test_name: |
| 119 | + return 2 |
| 120 | + if files_changed: |
| 121 | + if set(files_changed) & set(record.intervention.files_changed): |
| 122 | + return 3 |
| 123 | + return None |
| 124 | + |
| 125 | + |
| 126 | +def rank_records( |
| 127 | + records: list[MemoryRecord], |
| 128 | + fingerprint: Optional[str] = None, |
| 129 | + test_name: Optional[str] = None, |
| 130 | + files_changed: Optional[Iterable[str]] = None, |
| 131 | + signature: Optional[str] = None, |
| 132 | + mode: MemoryMode = MemoryMode.ALL, |
| 133 | +) -> list[MemoryRecord]: |
| 134 | + """Order records by relevance to the failure currently being worked on.""" |
| 135 | + allowed = _allowed_statuses(mode) |
| 136 | + candidates = [record for record in records if record.status in allowed] |
| 137 | + if not candidates: |
| 138 | + return [] |
| 139 | + |
| 140 | + lexical_scores = _bm25_scores(signature, candidates) if signature else {} |
| 141 | + |
| 142 | + scored: list[tuple[int, float, int, str, MemoryRecord]] = [] |
| 143 | + for record in candidates: |
| 144 | + tier = _symbolic_tier(record, fingerprint, test_name, files_changed) |
| 145 | + lexical_score = lexical_scores.get(record.memory_id, 0.0) |
| 146 | + if tier is None: |
| 147 | + if lexical_score <= 0.0: |
| 148 | + continue |
| 149 | + # Lexical-only matches rank behind every symbolic match. |
| 150 | + tier = 4 |
| 151 | + # More occurrences means the observation repeated within this render, which is |
| 152 | + # independent evidence; ties break toward it. |
| 153 | + scored.append((tier, -lexical_score, -record.occurrences, record.memory_id, record)) |
| 154 | + |
| 155 | + scored.sort(key=lambda item: item[:4]) |
| 156 | + return [item[4] for item in scored] |
| 157 | + |
| 158 | + |
| 159 | +def select_records( |
| 160 | + records: list[MemoryRecord], |
| 161 | + depth: int, |
| 162 | + fingerprint: Optional[str] = None, |
| 163 | + test_name: Optional[str] = None, |
| 164 | + files_changed: Optional[Iterable[str]] = None, |
| 165 | + signature: Optional[str] = None, |
| 166 | + mode: MemoryMode = MemoryMode.ALL, |
| 167 | +) -> list[MemoryRecord]: |
| 168 | + """Rank, then cut to the adaptive depth.""" |
| 169 | + if depth <= 0 or mode is MemoryMode.OFF: |
| 170 | + return [] |
| 171 | + |
| 172 | + ranked = rank_records( |
| 173 | + records, |
| 174 | + fingerprint=fingerprint, |
| 175 | + test_name=test_name, |
| 176 | + files_changed=files_changed, |
| 177 | + signature=signature, |
| 178 | + mode=mode, |
| 179 | + ) |
| 180 | + return ranked[:depth] |
0 commit comments