From f72c67ab24d80c24c60ecaf67fda51aa2d4bdd2e Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 20:11:20 +0530 Subject: [PATCH 001/109] eval: add clean failure diagnostics --- evals/run_clean_benchmarks.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/evals/run_clean_benchmarks.py b/evals/run_clean_benchmarks.py index 9317c97..5a1ff6c 100644 --- a/evals/run_clean_benchmarks.py +++ b/evals/run_clean_benchmarks.py @@ -156,6 +156,7 @@ def run( planner_correct = 0 planner_confusion: dict[str, dict[str, int]] = defaultdict(lambda: defaultdict(int)) asked = 0 + diagnostics: list[dict] = [] started = time.perf_counter() for sample in data: @@ -209,6 +210,19 @@ def run( planner_total += 1 planner_correct += int(planned_kind == expected_kind) planner_confusion[expected_kind][planned_kind] += 1 + diagnostics.append({ + "sample_id": sample_id, + "category": category, + "question": qa["question"], + "gold": golds, + "answer": result["answer"], + "token_f1": round(100 * score, 2), + "evidence_recall": round(100 * evidence_score, 2), + "gold_token_coverage": round(100 * coverage, 2), + "planned_kind": planned_kind, + "expected_kind": expected_kind, + "top_contexts": [str(x)[:320] for x in contexts_used[:4]], + }) asked += 1 if max_questions is not None and asked >= max_questions: @@ -258,6 +272,17 @@ def run( "by_category": by_category, "use_llm": use_llm, "use_reader": use_reader, + "worst_answer_failures": sorted( + diagnostics, + key=lambda item: (item["token_f1"], -item["gold_token_coverage"]), + )[:12], + "retrieval_failures": sorted( + diagnostics, + key=lambda item: (item["gold_token_coverage"], item["evidence_recall"]), + )[:12], + "planner_misses": [ + item for item in diagnostics if item["planned_kind"] != item["expected_kind"] + ][:12], } From bb6da2b01b49c9f99171353a4b8db2af0b28e9de Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 20:13:13 +0530 Subject: [PATCH 002/109] feat: infer distributed list and set retrieval generically --- src/ultimate_memory/planner.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/ultimate_memory/planner.py b/src/ultimate_memory/planner.py index 194b7af..70e6d8c 100644 --- a/src/ultimate_memory/planner.py +++ b/src/ultimate_memory/planner.py @@ -54,6 +54,14 @@ class QueryPlan(BaseModel): re.I, ) +_LIST_OR_SET_RE = re.compile( + r"\bwhich\s+(?:cities|places|countries|states|books|games|activities|items|things|ways|types|kinds)\b|" + r"\bwhat\s+(?:cities|places|countries|states|books|games|activities|items|things|ways|types|kinds)\b|" + r"\bwhat\s+does\s+.+?\s+(?:offer|provide|include)\b|" + r"\b(?:all|multiple|several)\s+(?:cities|places|books|activities|items|things|ways|types)\b", + re.I, +) + def _is_temporal_question(question: str) -> bool: return bool(_TEMPORAL_QUESTION_RE.search(question.lower())) @@ -61,7 +69,7 @@ def _is_temporal_question(question: str) -> bool: def _is_collective_multi_hop(question: str, entities: list[str]) -> bool: lower = question.lower() - if _COLLECTIVE_RE.search(lower): + if _COLLECTIVE_RE.search(lower) or _LIST_OR_SET_RE.search(lower): return True if len(entities) >= 2 and re.search(r"\b(?:and|versus|vs\.?|compared?\s+to)\b", lower): return True @@ -131,6 +139,10 @@ def _expansions(question: str) -> list[str]: expansions.append("procedure steps process") if re.search(r"\bbefore|previous|formerly|used to|prior\b", lower): expansions.append("previous formerly before historical") + if re.search(r"\bvisit|visited|trip|travel|cities|places\b", lower): + expansions.append("visited travel trip city place") + if re.search(r"\boffer|offers|offering|provide|provides|services\b", lower): + expansions.append("offer provides services classes workshops training") if _is_temporal_question(question): expansions.append("date year month day when duration time") return list(dict.fromkeys(expansions)) From d6b4a006aa43a9f2d33fdaeb4ca43fac58cf0570 Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 20:13:16 +0530 Subject: [PATCH 003/109] feat: add session-aware temporal answers and distributed list synthesis --- src/ultimate_memory/answer.py | 107 ++++++++++++++++++++++++++++++++-- 1 file changed, 101 insertions(+), 6 deletions(-) diff --git a/src/ultimate_memory/answer.py b/src/ultimate_memory/answer.py index 6a6d40a..8b914a1 100644 --- a/src/ultimate_memory/answer.py +++ b/src/ultimate_memory/answer.py @@ -112,7 +112,7 @@ r"October|November|December)\s+\d{4}\b", re.I, ), - re.compile(r"\b\d+\s+years?\s+ago\b", re.I), + re.compile(r"\b(?:a\s+few|several|\d+)\s+years?\s+ago\b", re.I), re.compile(r"\b(?:in|on|during)\s+(?:the\s+)?(?:year\s+)?((?:19|20)\d{2})\b", re.I), re.compile(r"\b(?:19|20)\d{2}\b"), re.compile(r"\b(?:last|next)\s+(?:Monday|Tuesday|Wednesday|Thursday|Friday|Saturday|Sunday)\b", re.I), @@ -122,7 +122,7 @@ r"^(?:last|next|this)\s+(?:monday|tuesday|wednesday|thursday|friday|saturday|sunday|" r"week|weekend|month|year)$|" r"^(?:yesterday|today|tomorrow|recently|earlier|later)$|" - r"^(?:a few days ago|two days ago|2 days ago|last night)$", + r"^(?:a few days ago|a few years ago|several years ago|two days ago|2 days ago|last night)$", re.I, ) @@ -432,6 +432,7 @@ class _ContextItem: memory_type: str = "note" provenance: dict[str, Any] | None = None score: float = 0.0 + session_date: str | None = None @dataclass(frozen=True) @@ -463,25 +464,100 @@ def _filter_context_text(text: str) -> str: return "\n".join(kept).strip() +def _context_session_date(text: str, provenance: dict[str, Any] | None = None) -> str | None: + provenance = provenance or {} + for key in ("session_date", "event_time", "created_at"): + value = provenance.get(key) + if value: + spans = _extract_date_spans(str(value)) + if spans: + return spans[0] + if str(value).strip(): + return str(value).strip() + for line in text.splitlines()[:8]: + lower = line.strip().lower() + if lower.startswith(("session_date:", "created_at:", "event_time:", "date:")): + value = line.split(":", 1)[1].strip() + spans = _extract_date_spans(value) + return spans[0] if spans else value + return None + + +def _is_list_question(question: str) -> bool: + lower = question.lower() + return bool( + re.search( + r"\bboth\b|\ball\b|" + r"\bwhich\s+(?:cities|places|countries|states|books|games|activities|items|things|ways|types|kinds)\b|" + r"\bwhat\s+(?:cities|places|countries|states|books|games|activities|items|things|ways|types|kinds)\b|" + r"\bwhat\s+does\s+.+?\s+(?:offer|provide|include)\b", + lower, + ) + ) + + +def _list_answer(question: str, normalized: list[_ContextItem], max_chars: int) -> str | None: + question_words = _content_words(question) + entities = _question_entities(question) + scored: list[tuple[float, str]] = [] + seen: set[str] = set() + for item in normalized: + for sentence in _split_sentences(item.text): + if len(sentence) < 8: + continue + overlap = _overlap_score(question_words, sentence, entities) + lower = sentence.lower() + relation_bonus = 0.0 + if re.search(r"\b(?:visit|visited|went to|trip to|travel(?:ed|led)? to)\b", lower): + relation_bonus += 0.45 + if re.search(r"\b(?:offer|offering|provide|provides|classes|workshops|training|services)\b", lower): + relation_bonus += 0.45 + if overlap < 0.12 and relation_bonus == 0.0: + continue + key = re.sub(r"\s+", " ", sentence.strip()).lower() + if key in seen: + continue + seen.add(key) + scored.append((overlap + relation_bonus + min(item.score, 1.0) * 0.12, sentence.strip())) + if not scored: + return None + scored.sort(key=lambda x: x[0], reverse=True) + chosen: list[str] = [] + used = 0 + for _, sentence in scored[:6]: + if used + len(sentence) > max_chars and chosen: + continue + chosen.append(sentence) + used += len(sentence) + 2 + if len(chosen) >= 3: + break + return _truncate(" ".join(chosen), max_chars) if chosen else None + + def _normalize_contexts( contexts: list[str] | list[dict[str, Any]], ) -> list[_ContextItem]: items: list[_ContextItem] = [] for raw in contexts: if isinstance(raw, str): + session_date = _context_session_date(raw) text = _filter_context_text(raw) if text: - items.append(_ContextItem(text=text)) + items.append(_ContextItem(text=text, session_date=session_date)) continue - text = _filter_context_text(str(raw.get("text") or "")) + raw_text = str(raw.get("text") or "") + provenance = raw.get("provenance") if isinstance(raw.get("provenance"), dict) else {} + session_date = _context_session_date(raw_text, provenance) + text = _filter_context_text(raw_text) if not text: continue items.append( _ContextItem( text=text, memory_type=str(raw.get("memory_type") or "note"), - provenance=raw.get("provenance") if isinstance(raw.get("provenance"), dict) else {}, + provenance=provenance, score=float(raw.get("score") or 0.0), + session_date=session_date, ) ) return items @@ -799,9 +875,17 @@ def synthesize_answer( occupation_question = _is_occupation_question(question) identity_question = _is_identity_question(question) + if _is_list_question(question): + list_answer = _list_answer(question, normalized, max_chars) + if list_answer: + return list_answer + # For "when" questions, prefer contexts that actually contain date spans. if kind == "when": - dated_only = [item for item in normalized if _extract_date_spans(item.text)] + dated_only = [ + item for item in normalized + if _extract_date_spans(item.text) or item.session_date + ] if dated_only: normalized = dated_only @@ -897,6 +981,17 @@ def synthesize_answer( date, ) ) + if item.session_date: + topical_sentences = [ + sentence for sentence in _split_sentences(item.text) + if _overlap_score(question_words, sentence, entities) >= 0.22 + ] + if topical_sentences: + best_overlap = max( + _overlap_score(question_words, sentence, entities) + for sentence in topical_sentences + ) + dated.append((meta + best_overlap + 0.55, item.session_date)) if dated: dated.sort(key=lambda x: (x[0], len(x[1])), reverse=True) # If the top hit is relative-only, skip down to an absolute one. From d20923338ee62c235d58b5a526fef23963dcf6ac Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 20:13:30 +0530 Subject: [PATCH 004/109] test: cover session dates relative dates and distributed lists --- tests/test_answer.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/test_answer.py b/tests/test_answer.py index 4bb6dd2..15f6f30 100644 --- a/tests/test_answer.py +++ b/tests/test_answer.py @@ -229,3 +229,30 @@ def test_answer_wraps_search_and_synthesis(self, tmp_path): assert result["search"]["results"] f1 = tokenize_f1(result["answer"], "May 2023") assert f1 >= 0.5 + + +def test_when_can_use_relevant_session_frontmatter_date(): + contexts = [ + { + "text": "---\nsession_date: 4 February, 2023\n---\n[D4:3] Jon: My group is performing at the festival this month.", + "memory_type": "log", + "score": 0.9, + } + ] + answer = synthesize_answer("When is Jon's group performing at a festival?", contexts) + assert "February" in answer and "2023" in answer + + +def test_relative_multi_year_date_is_extractable(): + contexts = [{"text": "Gina: I got my tattoo a few years ago.", "score": 1.0}] + assert synthesize_answer("When did Gina get her tattoo?", contexts).lower() == "a few years ago" + + +def test_list_question_combines_distributed_evidence(): + contexts = [ + {"text": "Jon: I visited Paris last winter.", "score": 0.9}, + {"text": "Jon: I took a trip to Rome this summer.", "score": 0.8}, + ] + answer = synthesize_answer("Which cities has Jon visited?", contexts) + assert "Paris" in answer + assert "Rome" in answer From ab9c3380a9f38f85ac3ec130ac8dd15fb1075f74 Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 20:13:33 +0530 Subject: [PATCH 005/109] test: cover generic list and offering query planning --- tests/test_planner.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/test_planner.py b/tests/test_planner.py index 567666f..c989521 100644 --- a/tests/test_planner.py +++ b/tests/test_planner.py @@ -45,3 +45,14 @@ def test_collective_both_question_triggers_multi_hop(): def test_two_entity_comparison_triggers_multi_hop(): plan = plan_query("What do Elena and Marcus have in common?") assert plan.kind == "multi_hop" + + +def test_plural_set_query_uses_multi_hop_plan(): + plan = plan_query("Which cities has Jon visited?") + assert plan.kind == "multi_hop" + assert plan.hop_depth >= 2 + + +def test_offer_query_uses_multi_evidence_plan(): + plan = plan_query("What does Jon's dance studio offer?") + assert plan.kind == "multi_hop" From 9fdaae00b1ad6adb90d711c4bca7b51c8d624dbf Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 20:14:19 +0530 Subject: [PATCH 006/109] feat: expose direct-turn provenance to retrieval ranking --- src/ultimate_memory/router.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/ultimate_memory/router.py b/src/ultimate_memory/router.py index 658946d..4b19877 100644 --- a/src/ultimate_memory/router.py +++ b/src/ultimate_memory/router.py @@ -1123,6 +1123,11 @@ def _atom_to_search_result(atom: AtomicMemory, *, score: float) -> SearchResult: "entities": atom.entities, "project_path": atom.project_path, "claim": atom.metadata.get("claim"), + "dia_id": atom.metadata.get("dia_id"), + "speaker": atom.metadata.get("speaker"), + "session_id": atom.metadata.get("session_id"), + "direct_turn": bool(atom.metadata.get("dia_id")), + "metadata": atom.metadata, }, ) From 91a2b041073d8b6c1d10da8483a152137a6dd76d Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 20:14:22 +0530 Subject: [PATCH 007/109] feat: prefer direct evidence over derived reflections --- src/ultimate_memory/ranking.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/ultimate_memory/ranking.py b/src/ultimate_memory/ranking.py index f8bc258..a719f94 100644 --- a/src/ultimate_memory/ranking.py +++ b/src/ultimate_memory/ranking.py @@ -43,6 +43,14 @@ def rerank_candidates( if isinstance(claim, dict): score += 0.04 * float(claim.get("confidence") or 0.0) + # Direct dialogue/tool observations are primary evidence. Derived reflections + # remain useful, but should not outrank an equally relevant original turn. + if provenance.get("direct_turn") or provenance.get("dia_id"): + direct_overlap = len(q_tokens & r_tokens) / len(q_tokens) if q_tokens else 0.0 + score += 0.12 + 0.18 * direct_overlap + if "auto-extracted from" in text_lower: + score -= 0.08 + if plan.temporal_mode == "current" and provenance.get("valid_until"): score -= 0.35 elif plan.temporal_mode in {"historical", "as_of"} and provenance.get("valid_until"): From 13ebcd72fa9b35c9892617543edf0c64d88a010a Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 20:15:38 +0530 Subject: [PATCH 008/109] fix: parse full session dates and gate list aggregation --- src/ultimate_memory/answer.py | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/src/ultimate_memory/answer.py b/src/ultimate_memory/answer.py index 8b914a1..1d996a8 100644 --- a/src/ultimate_memory/answer.py +++ b/src/ultimate_memory/answer.py @@ -80,7 +80,13 @@ _SENTENCE_SPLIT = re.compile(r"(?<=[.!?])\s+|\n+") _ENTITY_RE = re.compile(r"\b([A-Z][a-z]+(?:\s+[A-Z][a-z]+)?)\b") _DATE_PATTERNS: list[re.Pattern[str]] = [ - # Day Month Year: 7 May 2023 + # Day Month Year, with optional comma: 7 May 2023 / 7 May, 2023 + re.compile( + r"\b\d{1,2}\s+" + r"(?:January|February|March|April|May|June|July|August|September|" + r"October|November|December)(?:,\s*|\s+)\d{4}\b", + re.I, + ), re.compile( r"\b\d{1,2}\s+" r"(?:January|February|March|April|May|June|July|August|September|" @@ -875,10 +881,7 @@ def synthesize_answer( occupation_question = _is_occupation_question(question) identity_question = _is_identity_question(question) - if _is_list_question(question): - list_answer = _list_answer(question, normalized, max_chars) - if list_answer: - return list_answer + list_question = _is_list_question(question) # For "when" questions, prefer contexts that actually contain date spans. if kind == "when": @@ -1050,6 +1053,18 @@ def synthesize_answer( head = re.split(r"\s+for\s+|\s+as\s+", obj, maxsplit=1)[0].strip() return _truncate(head or obj, max_chars) + # Distributed/list questions may require evidence from multiple contexts. + # Only use aggregation when at least two distinct high-relevance sentences exist. + if list_question: + list_answer = _list_answer(question, normalized, max_chars) + if list_answer: + parts = _split_sentences(list_answer) + if len(parts) >= 2 or any( + cue in list_answer.lower() + for cue in ("visited", "trip to", "offer", "provid", "classes", "workshops", "training") + ): + return list_answer + # Prefer a tight span when it still overlaps the question. answer = _strip_supersession_tail(best.text if best.score >= 0.2 and len(best.text) <= max_chars else best.sentence) if answer: From ecdd6d0eb631ccc32fa0ef55893fe72a6813563e Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 20:16:34 +0530 Subject: [PATCH 009/109] feat: resolve conversational month week and weekday dates --- src/ultimate_memory/dates.py | 42 +++++++++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/src/ultimate_memory/dates.py b/src/ultimate_memory/dates.py index 79fae95..4a79cb5 100644 --- a/src/ultimate_memory/dates.py +++ b/src/ultimate_memory/dates.py @@ -71,6 +71,23 @@ def format_day_month_year(when: datetime) -> str: return f"{when.day} {when.strftime('%B')} {when.year}" +def format_month_year(when: datetime) -> str: + return f"{when.strftime('%B')} {when.year}" + + +def _shift_month(anchor: datetime, delta: int) -> datetime: + month0 = anchor.year * 12 + (anchor.month - 1) + delta + year, month_index = divmod(month0, 12) + return datetime(year, month_index + 1, 1) + + +def _previous_weekday(anchor: datetime, weekday: int) -> datetime: + days_back = (anchor.weekday() - weekday) % 7 + if days_back == 0: + days_back = 7 + return anchor - timedelta(days=days_back) + + def resolve_relative_dates(text: str, anchor: datetime | None) -> str: """Replace yesterday/today/last year/etc. with absolute dates using *anchor*.""" if anchor is None: @@ -84,11 +101,34 @@ def resolve_relative_dates(text: str, anchor: datetime | None) -> str: (r"\bthis\s+year\b", f"in {anchor.year}"), ( r"\blast\s+month\b", - f"in {format_day_month_year(anchor.replace(day=1) - timedelta(days=1))}", + f"in {format_month_year(_shift_month(anchor, -1))}", ), + (r"\bthis\s+month\b", f"in {format_month_year(anchor)}"), + (r"\bnext\s+month\b", f"in {format_month_year(_shift_month(anchor, 1))}"), + (r"\blast\s+week\b", f"around {format_day_month_year(anchor - timedelta(days=7))}"), + (r"\bthis\s+week\b", f"around {format_day_month_year(anchor)}"), + (r"\bnext\s+week\b", f"around {format_day_month_year(anchor + timedelta(days=7))}"), (r"\ba\s+year\s+ago\b", f"in {anchor.year - 1}"), (r"\btwo\s+years\s+ago\b", f"in {anchor.year - 2}"), ] for pattern, value in replacements: out = re.sub(pattern, value, out, flags=re.IGNORECASE) + + weekday_names = { + "monday": 0, + "tuesday": 1, + "wednesday": 2, + "thursday": 3, + "friday": 4, + "saturday": 5, + "sunday": 6, + } + for name, weekday in weekday_names.items(): + when = _previous_weekday(anchor, weekday) + out = re.sub( + rf"\blast\s+{name}\b", + f"on {format_day_month_year(when)}", + out, + flags=re.IGNORECASE, + ) return out From be2c5c2bff46ce5620e71682926ec30ce3e273c3 Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 20:16:55 +0530 Subject: [PATCH 010/109] test: cover relative month and weekday resolution --- tests/test_dates.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 tests/test_dates.py diff --git a/tests/test_dates.py b/tests/test_dates.py new file mode 100644 index 0000000..924073a --- /dev/null +++ b/tests/test_dates.py @@ -0,0 +1,15 @@ +from ultimate_memory.dates import parse_loose_date, resolve_relative_dates + + +def test_resolve_this_and_next_month(): + anchor = parse_loose_date("15 February 2023") + assert anchor is not None + assert "February 2023" in resolve_relative_dates("this month", anchor) + assert "March 2023" in resolve_relative_dates("next month", anchor) + + +def test_resolve_last_named_weekday(): + anchor = parse_loose_date("20 July 2023") + assert anchor is not None + result = resolve_relative_dates("last Saturday", anchor) + assert "15 July 2023" in result From 707408b8791e0567145c36368d69d5abda99899e Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 20:18:19 +0530 Subject: [PATCH 011/109] feat: add local semantic fallback retrieval --- src/ultimate_memory/local_semantic.py | 95 +++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100644 src/ultimate_memory/local_semantic.py diff --git a/src/ultimate_memory/local_semantic.py b/src/ultimate_memory/local_semantic.py new file mode 100644 index 0000000..e76f705 --- /dev/null +++ b/src/ultimate_memory/local_semantic.py @@ -0,0 +1,95 @@ +"""Optional in-process semantic retrieval for local/fallback mode. + +This reuses the configured FastEmbed model and keeps embeddings in process memory. +It is deliberately optional so normal unit tests and lightweight installs do not +need to download a model. Enable with ULTIMATE_MEMORY_LOCAL_SEMANTIC=1. +""" +from __future__ import annotations + +import math +import os + +from .models import AtomicMemory, SearchResult + + +def enabled() -> bool: + return os.environ.get("ULTIMATE_MEMORY_LOCAL_SEMANTIC", "0").strip().lower() in { + "1", "true", "yes", "on", + } + + +def cosine(left: list[float], right: list[float]) -> float: + if not left or not right or len(left) != len(right): + return 0.0 + dot = sum(a * b for a, b in zip(left, right, strict=True)) + ln = math.sqrt(sum(a * a for a in left)) + rn = math.sqrt(sum(b * b for b in right)) + if ln == 0.0 or rn == 0.0: + return 0.0 + return dot / (ln * rn) + + +class LocalSemanticIndex: + def __init__(self, embed) -> None: + self._embed = embed + self._vectors: dict[str, tuple[str, list[float]]] = {} + + def _ensure(self, atoms: list[AtomicMemory]) -> None: + missing = [ + atom for atom in atoms + if atom.id not in self._vectors or self._vectors[atom.id][0] != atom.content_hash + ] + if not missing: + return + vectors = self._embed([atom.text for atom in missing]) + for atom, vector in zip(missing, vectors, strict=True): + self._vectors[atom.id] = (atom.content_hash, vector) + + def search( + self, + query: str, + atoms: list[AtomicMemory], + *, + limit: int, + ) -> list[SearchResult]: + if not query.strip() or not atoms: + return [] + self._ensure(atoms) + query_vector = self._embed([query])[0] + + scored: list[tuple[float, AtomicMemory]] = [] + for atom in atoms: + cached = self._vectors.get(atom.id) + if cached is None: + continue + score = cosine(query_vector, cached[1]) + scored.append((score, atom)) + scored.sort(key=lambda item: item[0], reverse=True) + + out: list[SearchResult] = [] + for score, atom in scored[:limit]: + out.append( + SearchResult( + id=atom.id, + text=atom.text, + title=f"{atom.memory_type.value}: {atom.text[:72]}", + source_path=f"atom://{atom.id}", + memory_type=atom.memory_type.value, + score=max(0.0, min((score + 1.0) / 2.0, 1.0)), + provenance={ + "source": "local-semantic", + "semantic_score": score, + "salience": atom.salience, + "valid_from": atom.valid_from, + "valid_until": atom.valid_until, + "superseded_by": atom.superseded_by, + "entities": atom.entities, + "project_path": atom.project_path, + "dia_id": atom.metadata.get("dia_id"), + "speaker": atom.metadata.get("speaker"), + "direct_turn": bool(atom.metadata.get("dia_id")), + "claim": atom.metadata.get("claim"), + }, + ) + ) + return out From 2d2cf4925998f93bd47ba83fe460b520e729b791 Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 20:18:23 +0530 Subject: [PATCH 012/109] feat: use local semantic retrieval when Qdrant is unavailable --- src/ultimate_memory/router.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/src/ultimate_memory/router.py b/src/ultimate_memory/router.py index 4b19877..e146038 100644 --- a/src/ultimate_memory/router.py +++ b/src/ultimate_memory/router.py @@ -49,6 +49,7 @@ normalize_entity, ) from .llm_answer import use_llm_from_env +from .local_semantic import LocalSemanticIndex, enabled as local_semantic_enabled from .planner import plan_query from .ranking import rerank_candidates from .store import LocalStore @@ -88,6 +89,7 @@ def __init__(self, settings: Settings | None = None) -> None: self.graph = GraphAdapter(self.settings.neo4j) self._vector_ready_cache = False self._graph_ready_cache = False + self._local_semantic = LocalSemanticIndex(self.vector.embed) @property def _vector_ready(self) -> bool: @@ -377,6 +379,21 @@ def _atoms(): def _graph(): return self.graph.query(query, depth=1) if self._graph_ready else [] + def _local_semantic(): + if self._vector_ready or not local_semantic_enabled(): + return [] + atoms = self.store.list_active_atoms( + memory_types=memory_types, + project_path=project_path, + limit=max(actual_limit * 30, 240), + as_of=as_of, + ) + return self._local_semantic.search( + query, + atoms, + limit=max(actual_limit * 2, 20), + ) + with concurrent.futures.ThreadPoolExecutor(max_workers=3) as pool: fv = pool.submit(_vector) fb = pool.submit(_basic) @@ -387,9 +404,10 @@ def _graph(): vector_results: list[SearchResult] = fv.result() bm_results: list[SearchResult] = fb.result() graph_hits: list[dict] = fg.result() + semantic_results = _local_semantic() results = self._rrf_rank( - [vector_results, keyword_results, bm_results, atom_results], + [vector_results, keyword_results, bm_results, atom_results, semantic_results], memory_types=memory_types, limit=actual_limit * 2, ) @@ -415,6 +433,7 @@ def _graph(): "results": [result.model_dump() for result in results], "graph_hits": graph_hits[:5], "atoms_considered": len(atom_results), + "semantic_hits": len(semantic_results), "query_plan": query_plan.model_dump(), } From 2844a988c5196c8f767ff0133e054ce809472f56 Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 20:18:26 +0530 Subject: [PATCH 013/109] ci: benchmark local semantic fallback --- .github/workflows/full-clean-benchmark.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/full-clean-benchmark.yml b/.github/workflows/full-clean-benchmark.yml index 00a9d7d..0167b91 100644 --- a/.github/workflows/full-clean-benchmark.yml +++ b/.github/workflows/full-clean-benchmark.yml @@ -8,6 +8,8 @@ jobs: clean-benchmark: runs-on: ubuntu-latest timeout-minutes: 20 + env: + ULTIMATE_MEMORY_LOCAL_SEMANTIC: "1" steps: - uses: actions/checkout@v4 - name: Install uv From 6526eb9d3cad4cf8436cc3c9d692f568ceeb2387 Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 20:18:30 +0530 Subject: [PATCH 014/109] ci: benchmark local semantic fallback --- .github/workflows/reader-benchmark.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/reader-benchmark.yml b/.github/workflows/reader-benchmark.yml index d3213ab..032c993 100644 --- a/.github/workflows/reader-benchmark.yml +++ b/.github/workflows/reader-benchmark.yml @@ -10,6 +10,7 @@ jobs: timeout-minutes: 20 env: TOKENIZERS_PARALLELISM: "false" + ULTIMATE_MEMORY_LOCAL_SEMANTIC: "1" steps: - uses: actions/checkout@v4 - name: Install uv From b6d498b3d9c2ad5400441fd8187d172c0237137c Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 20:18:42 +0530 Subject: [PATCH 015/109] test: cover local semantic fallback without model downloads --- tests/test_local_semantic.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 tests/test_local_semantic.py diff --git a/tests/test_local_semantic.py b/tests/test_local_semantic.py new file mode 100644 index 0000000..aba1810 --- /dev/null +++ b/tests/test_local_semantic.py @@ -0,0 +1,28 @@ +from ultimate_memory.local_semantic import LocalSemanticIndex, cosine +from ultimate_memory.models import AtomicMemory, MemoryType + + +def fake_embed(texts): + vectors = [] + for text in texts: + lower = text.lower() + vectors.append([ + 1.0 if "dance" in lower or "studio" in lower else 0.0, + 1.0 if "database" in lower or "postgres" in lower else 0.0, + ]) + return vectors + + +def test_cosine_identity(): + assert cosine([1.0, 0.0], [1.0, 0.0]) == 1.0 + + +def test_semantic_index_ranks_related_atom(): + index = LocalSemanticIndex(fake_embed) + atoms = [ + AtomicMemory(text="Project uses PostgreSQL.", memory_type=MemoryType.FACT), + AtomicMemory(text="The dance studio offers classes.", memory_type=MemoryType.FACT), + ] + results = index.search("What does the studio offer?", atoms, limit=2) + assert "dance studio" in results[0].text.lower() + assert results[0].provenance["source"] == "local-semantic" From f812df282178b9208ddd73ecaa38d0ab8a6ee902 Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 20:20:38 +0530 Subject: [PATCH 016/109] feat: infer repeated-history questions as distributed retrieval --- src/ultimate_memory/planner.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/ultimate_memory/planner.py b/src/ultimate_memory/planner.py index 70e6d8c..13c3b8a 100644 --- a/src/ultimate_memory/planner.py +++ b/src/ultimate_memory/planner.py @@ -55,10 +55,12 @@ class QueryPlan(BaseModel): ) _LIST_OR_SET_RE = re.compile( - r"\bwhich\s+(?:cities|places|countries|states|books|games|activities|items|things|ways|types|kinds)\b|" - r"\bwhat\s+(?:cities|places|countries|states|books|games|activities|items|things|ways|types|kinds)\b|" - r"\bwhat\s+does\s+.+?\s+(?:offer|provide|include)\b|" - r"\b(?:all|multiple|several)\s+(?:cities|places|books|activities|items|things|ways|types)\b", + r"\bwhich\s+(?:[a-z]+s|cities|places|countries|states|books|games|activities|items|things|ways|types|kinds)\b|" + r"\bwhat\s+(?:[a-z]+s|cities|places|countries|states|books|games|activities|items|things|ways|types|kinds)\b|" + r"\bwhat\s+does\s+.+?\s+(?:offer|provide|include|do\s+to)\b|" + r"\b(?:all|multiple|several)\s+(?:[a-z]+s|cities|places|books|activities|items|things|ways|types)\b|" + r"\bwhere\s+has\s+.+?\s+(?:camped|traveled|travelled|visited|stayed|lived)\b|" + r"\bwhat\s+.+?\s+has\s+.+?\s+(?:done|read|visited|attended|participated|painted|tried|used)\b", re.I, ) From 38e545d6042bebc8af98644566071e7544e5f0d8 Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 20:20:42 +0530 Subject: [PATCH 017/109] feat: improve duration extraction and suppress greeting answers --- src/ultimate_memory/answer.py | 38 ++++++++++++++++++++++++++++------- 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/src/ultimate_memory/answer.py b/src/ultimate_memory/answer.py index 1d996a8..2f476b0 100644 --- a/src/ultimate_memory/answer.py +++ b/src/ultimate_memory/answer.py @@ -132,9 +132,19 @@ re.I, ) +_NUMBER_WORD = r"(?:one|two|three|four|five|six|seven|eight|nine|ten|eleven|twelve|a\s+few|several)" _DURATION_SPAN_RE = re.compile( - r"\b(\d+\s+years?(?:\s+ago)?|\d+\s+months?(?:\s+ago)?|\d+\s+weeks?(?:\s+ago)?|" - r"for\s+\d+\s+years?|\dover\s+\d+\s+years?)\b", + rf"\b((?:\d+|{_NUMBER_WORD})\s+years?(?:\s+ago)?|" + rf"(?:\d+|{_NUMBER_WORD})\s+months?(?:\s+ago)?|" + rf"(?:\d+|{_NUMBER_WORD})\s+weeks?(?:\s+ago)?|" + rf"(?:for|over)\s+(?:\d+|{_NUMBER_WORD})\s+(?:years?|months?|weeks?))\b", + re.I, +) + +_GREETING_ONLY_RE = re.compile( + r"^(?:\[D\d+:\d+\]\s*)?(?:[A-Z][a-z]+\s*:\s*)?" + r"(?:hey|hi|hello|thanks|thank\s+you|wow|great|awesome|nice|cool|sure|yep|yeah)" + r"(?:[\s,!.'-]+[A-Z][a-z]+)?[!?.]*$", re.I, ) @@ -736,6 +746,11 @@ def _score_candidate( score += 0.9 lower = span.lower() + if _GREETING_ONLY_RE.match(span.strip()): + score -= 1.4 + if _GREETING_ONLY_RE.match(sentence.strip()): + score -= 0.8 + if kind == "yes_no": if any(cue in lower for cue in _NEG_CUES): score += 0.2 @@ -885,12 +900,17 @@ def synthesize_answer( # For "when" questions, prefer contexts that actually contain date spans. if kind == "when": - dated_only = [ + duration_question = bool( + re.search(r"\bhow long\b|\bhow many\s+(?:years?|months?|weeks?)\b", question, re.I) + ) + temporal_only = [ item for item in normalized - if _extract_date_spans(item.text) or item.session_date + if _extract_date_spans(item.text) + or _extract_duration_spans(item.text) + or item.session_date ] - if dated_only: - normalized = dated_only + if temporal_only: + normalized = temporal_only if kind == "yes_no": sentences: list[tuple[float, str]] = [] @@ -945,7 +965,11 @@ def synthesize_answer( if kind == "when": duration_question = bool( - re.search(r"\bhow long\b|\bhow many years\b|\byears? ago\b", question, re.I) + re.search( + r"\bhow long\b|\bhow many\s+(?:years?|months?|weeks?)\b|\byears? ago\b", + question, + re.I, + ) ) # Prefer a date that co-occurs with question entities/content in the same sentence. dated: list[tuple[float, str]] = [] From 726b1fd682bed70eaeef595a62ca68cc713773cc Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 20:20:53 +0530 Subject: [PATCH 018/109] test: cover word durations and greeting suppression --- tests/test_answer.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/test_answer.py b/tests/test_answer.py index 15f6f30..a7be675 100644 --- a/tests/test_answer.py +++ b/tests/test_answer.py @@ -256,3 +256,21 @@ def test_list_question_combines_distributed_evidence(): answer = synthesize_answer("Which cities has Jon visited?", contexts) assert "Paris" in answer assert "Rome" in answer + + +def test_word_number_duration_is_preferred(): + contexts = [ + {"text": "Caroline: I've had this group of friends for four years now.", "score": 0.9}, + {"text": "---\nsession_date: 7 May 2023\n---\nCaroline: We met up recently.", "score": 0.5}, + ] + answer = synthesize_answer("How long has Caroline had this group of friends?", contexts) + assert "four years" in answer.lower() + + +def test_greeting_only_candidate_is_penalized(): + contexts = [ + {"text": "Gina: Wow!", "score": 1.0}, + {"text": "Gina: Dance feels magical to me.", "score": 0.8}, + ] + answer = synthesize_answer("How does Gina describe the feeling that dance brings?", contexts) + assert "magical" in answer.lower() From d732c932b5225235853dc22ce71536bc84706fd5 Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 20:20:57 +0530 Subject: [PATCH 019/109] test: cover generic repeated-history planning --- tests/test_planner.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_planner.py b/tests/test_planner.py index c989521..2d39b3d 100644 --- a/tests/test_planner.py +++ b/tests/test_planner.py @@ -56,3 +56,9 @@ def test_plural_set_query_uses_multi_hop_plan(): def test_offer_query_uses_multi_evidence_plan(): plan = plan_query("What does Jon's dance studio offer?") assert plan.kind == "multi_hop" + + +def test_generic_plural_history_query_uses_distributed_plan(): + assert plan_query("What martial arts has Alex done?").kind == "multi_hop" + assert plan_query("What books has Alex read?").kind == "multi_hop" + assert plan_query("Where has Alex camped?").kind == "multi_hop" From 83e49eb517d617e6ccaf1c2db6054e214c3004ae Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 20:21:43 +0530 Subject: [PATCH 020/109] fix: make local semantic retrieval supplemental instead of rank-disruptive --- src/ultimate_memory/router.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/ultimate_memory/router.py b/src/ultimate_memory/router.py index e146038..eb2b426 100644 --- a/src/ultimate_memory/router.py +++ b/src/ultimate_memory/router.py @@ -407,10 +407,24 @@ def _local_semantic(): semantic_results = _local_semantic() results = self._rrf_rank( - [vector_results, keyword_results, bm_results, atom_results, semantic_results], + [vector_results, keyword_results, bm_results, atom_results], memory_types=memory_types, limit=actual_limit * 2, ) + + # Local semantic retrieval is a recall supplement, not an equal RRF voter. + # This preserves strong lexical/direct evidence while still allowing + # paraphrased candidates to enter the final generic reranker. + existing_keys = {result.source_path or result.id for result in results} + for semantic in semantic_results: + key = semantic.source_path or semantic.id + if key in existing_keys: + continue + semantic.score *= 0.82 + semantic.provenance["supplemental_semantic"] = True + results.append(semantic) + existing_keys.add(key) + results = self._apply_salience_rerank(results) query_plan = plan_query(query, as_of=as_of) results = rerank_candidates(query, results, query_plan)[:actual_limit] From 1dbc512aec6bb35308f87dedfacbac8564ae44d1 Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 20:22:10 +0530 Subject: [PATCH 021/109] feat: hybrid reader routes temporal and distributed QA to structured reasoning --- src/ultimate_memory/reader.py | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/src/ultimate_memory/reader.py b/src/ultimate_memory/reader.py index bd7f72f..9c6a0c7 100644 --- a/src/ultimate_memory/reader.py +++ b/src/ultimate_memory/reader.py @@ -22,6 +22,18 @@ r"^(?:is|are|was|were|do|does|did|has|have|had|can|could|would|will|should|may|might)\b", re.I, ) +_TEMPORAL = re.compile( + r"\bwhen\b|\bwhat\s+(?:date|year|month|day)\b|" + r"\bhow\s+long\b|\bhow\s+many\s+(?:years?|months?|weeks?|days?)\b", + re.I, +) +_DISTRIBUTED = re.compile( + r"\bboth\b|\ball\b|" + r"\b(?:what|which)\s+[a-z]+s\b.*\b(?:has|have|did|does|are|were)\b|" + r"\bwhere\s+has\s+.+?\s+(?:camped|traveled|travelled|visited|stayed|lived)\b|" + r"\bwhat\s+does\s+.+?\s+(?:offer|provide|include|do\s+to)\b", + re.I, +) def reader_model_name() -> str: @@ -65,10 +77,17 @@ def answer(self, question: str, contexts: list[str] | list[dict[str, Any]]) -> s if not trimmed: return "" - # Extractive SQuAD readers cannot faithfully synthesize yes/no answers. - # Keep the deterministic fallback for that answer shape. - if _YES_NO.match(question.strip()): - return synthesize_answer(question, contexts) + # Span readers cannot faithfully synthesize yes/no, temporal metadata, + # or answers distributed across multiple memories. Route those shapes + # through the deterministic structured reasoner first. + if ( + _YES_NO.match(question.strip()) + or _TEMPORAL.search(question) + or _DISTRIBUTED.search(question) + ): + structured = synthesize_answer(question, contexts) + if structured: + return structured self._ensure_loaded() payloads = [{"question": question, "context": text} for text in trimmed] From c7ca14c388f8599a1ac08d318b1ee120facfeca4 Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 20:22:27 +0530 Subject: [PATCH 022/109] test: cover model-free temporal and distributed hybrid reader paths --- tests/test_reader.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/test_reader.py b/tests/test_reader.py index 71ea315..8822eb2 100644 --- a/tests/test_reader.py +++ b/tests/test_reader.py @@ -24,3 +24,26 @@ def test_reader_selects_highest_confidence_span_without_loading_transformers(): ], ) assert answer == "Portland" + + +def test_temporal_reader_uses_structured_reasoner_without_model(): + reader = ExtractiveReader() + contexts = [ + { + "text": "---\nsession_date: 4 February, 2023\n---\nJon: My group is performing at the festival this month.", + "memory_type": "log", + "score": 0.9, + } + ] + answer = reader.answer("When is Jon's group performing at a festival?", contexts) + assert "February" in answer and "2023" in answer + + +def test_distributed_reader_can_combine_multiple_memories_without_model(): + reader = ExtractiveReader() + contexts = [ + {"text": "Jon: I visited Paris last winter.", "score": 0.9}, + {"text": "Jon: I took a trip to Rome this summer.", "score": 0.8}, + ] + answer = reader.answer("Which cities has Jon visited?", contexts) + assert "Paris" in answer and "Rome" in answer From 004669d2906df775bc3a9321c67b2d999249fa27 Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 20:23:26 +0530 Subject: [PATCH 023/109] feat: retrieve adjacent conversation turns by session --- src/ultimate_memory/store.py | 63 ++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/src/ultimate_memory/store.py b/src/ultimate_memory/store.py index 556d5e6..b55bf1a 100644 --- a/src/ultimate_memory/store.py +++ b/src/ultimate_memory/store.py @@ -311,6 +311,69 @@ def get_atom(self, atom_id: str) -> AtomicMemory | None: ).fetchone() return self._row_to_atom(row) if row else None + def neighboring_turn_atoms( + self, + session_id: str, + dia_id: str, + *, + radius: int = 1, + project_path: str | None = None, + ) -> list[AtomicMemory]: + """Return nearby direct-turn atoms from the same conversation session.""" + match = re.fullmatch(r"D(\d+):(\d+)", dia_id.strip(), re.I) + if not session_id or not match or radius < 1: + return [] + dialogue_no = int(match.group(1)) + turn_no = int(match.group(2)) + + clauses = [ + "valid_until is null", + "superseded_by is null", + "id like 'atom:turn:%'", + ] + params: list[object] = [] + if project_path: + clauses.append("(project_path = ? or project_path is null or project_path = '')") + params.append(project_path) + + with self._connect() as conn: + try: + rows = conn.execute( + f""" + select * from memory_atoms + where {' and '.join(clauses)} + and json_extract(metadata_json, '$.session_id') = ? + order by created_at, id + limit 200 + """, + [*params, session_id], + ).fetchall() + except sqlite3.OperationalError: + rows = conn.execute( + f""" + select * from memory_atoms + where {' and '.join(clauses)} + and metadata_json like ? + order by created_at, id + limit 200 + """, + [*params, f'%"session_id": "{session_id}"%'], + ).fetchall() + + neighbors: list[tuple[int, AtomicMemory]] = [] + for row in rows: + atom = self._row_to_atom(row) + candidate_id = str(atom.metadata.get("dia_id") or "") + candidate = re.fullmatch(r"D(\d+):(\d+)", candidate_id, re.I) + if not candidate or int(candidate.group(1)) != dialogue_no: + continue + candidate_turn = int(candidate.group(2)) + distance = abs(candidate_turn - turn_no) + if 0 < distance <= radius: + neighbors.append((candidate_turn, atom)) + neighbors.sort(key=lambda item: item[0]) + return [atom for _, atom in neighbors] + def list_active_atoms( self, *, From 178fc039fb1c95a564505ccdefeb104f82573989 Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 20:23:30 +0530 Subject: [PATCH 024/109] feat: expand retrieved conversation turns with immediate neighbors --- src/ultimate_memory/router.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/src/ultimate_memory/router.py b/src/ultimate_memory/router.py index eb2b426..2e749ce 100644 --- a/src/ultimate_memory/router.py +++ b/src/ultimate_memory/router.py @@ -203,6 +203,37 @@ def answer( ) rich_contexts.extend(item for item in typed["results"] if item.get("text")) + # Conversation adjacency is evidence: a retrieved question/request turn + # is frequently answered by the immediately preceding or following turn. + neighbor_seen: set[str] = { + str(item.get("id") or item.get("source_path") or "") + for item in rich_contexts + } + for anchor in list(rich_contexts[: max(limit, 8)]): + provenance = anchor.get("provenance") or {} + session_id = str(provenance.get("session_id") or "") + dia_id = str(provenance.get("dia_id") or "") + if not session_id or not dia_id: + continue + for neighbor in self.store.neighboring_turn_atoms( + session_id, + dia_id, + radius=1, + project_path=project_path, + ): + if neighbor.id in neighbor_seen: + continue + neighbor_result = self._atom_to_search_result( + neighbor, + score=max(float(anchor.get("score") or 0.0) - 0.04, 0.45), + ).model_dump() + nprov = dict(neighbor_result.get("provenance") or {}) + nprov["conversation_neighbor"] = True + nprov["neighbor_of"] = dia_id + neighbor_result["provenance"] = nprov + rich_contexts.append(neighbor_result) + neighbor_seen.add(neighbor.id) + initial_results = search_result.get("results") or [] seen_entity_keys = {normalize_entity(entity) for entity in plan.entities} frontier = extract_hop_entities( From 0ead7ed127a2da7e93c7b3b90406f83948e96577 Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 20:23:55 +0530 Subject: [PATCH 025/109] test: cover adjacent conversation-turn evidence expansion --- tests/test_answer.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/test_answer.py b/tests/test_answer.py index a7be675..cd765c0 100644 --- a/tests/test_answer.py +++ b/tests/test_answer.py @@ -274,3 +274,26 @@ def test_greeting_only_candidate_is_penalized(): ] answer = synthesize_answer("How does Gina describe the feeling that dance brings?", contexts) assert "magical" in answer.lower() + + +def test_answer_expands_to_adjacent_conversation_turn(tmp_path): + router = MemoryRouter(make_settings(tmp_path)) + transcript = """--- +session_date: 10 May 2023 +--- +[D1:1] Melanie: What pet do you have? +[D1:2] Caroline: I have a guinea pig named Clover. +[D1:3] Melanie: That sounds adorable. +""" + router.ingest_log( + client="test", + session_id="adjacency", + transcript_or_path=transcript, + project_path="/project", + ) + result = router.answer("What pet does Caroline have?", project_path="/project", limit=6) + assert any( + (item.get("provenance") or {}).get("conversation_neighbor") + for item in result["search"].get("results", []) + ) is False # neighbors are answer-context expansion, not base search output + assert "guinea pig" in " ".join(result["contexts_used"]).lower() From ec44322f086852c29bac07ac8ebcce7065a095ab Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 20:24:38 +0530 Subject: [PATCH 026/109] feat: compose adjacent Q&A turns into reader evidence windows --- src/ultimate_memory/router.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/src/ultimate_memory/router.py b/src/ultimate_memory/router.py index 2e749ce..3e9f379 100644 --- a/src/ultimate_memory/router.py +++ b/src/ultimate_memory/router.py @@ -234,6 +234,33 @@ def answer( rich_contexts.append(neighbor_result) neighbor_seen.add(neighbor.id) + anchor_text = str(anchor.get("text") or "").strip() + if "?" in anchor_text or "?" in neighbor.text: + window_id = f"window:{session_id}:{dia_id}:{neighbor.metadata.get('dia_id', '')}" + if window_id not in neighbor_seen: + rich_contexts.append( + { + "id": window_id, + "title": "conversation evidence window", + "text": f"{anchor_text}\n{neighbor.text}", + "source_path": window_id, + "memory_type": "fact", + "score": min( + float(anchor.get("score") or 0.0) + 0.03, + 1.25, + ), + "provenance": { + "source": "conversation-window", + "conversation_neighbor": True, + "session_id": session_id, + "anchor_dia_id": dia_id, + "neighbor_dia_id": neighbor.metadata.get("dia_id"), + "project_path": project_path, + }, + } + ) + neighbor_seen.add(window_id) + initial_results = search_result.get("results") or [] seen_entity_keys = {normalize_entity(entity) for entity in plan.entities} frontier = extract_hop_entities( From 371ade57cc471f9eafbd813fa5ef6e206439f2c4 Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 20:24:49 +0530 Subject: [PATCH 027/109] feat: prioritize conversational neighbor evidence in synthesis --- src/ultimate_memory/answer.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/ultimate_memory/answer.py b/src/ultimate_memory/answer.py index 2f476b0..635f5d9 100644 --- a/src/ultimate_memory/answer.py +++ b/src/ultimate_memory/answer.py @@ -605,6 +605,11 @@ def _context_metadata_bonus( if provenance.get("hop"): bonus += 0.45 + if provenance.get("conversation_neighbor"): + bonus += 0.28 + if provenance.get("source") == "conversation-window": + bonus += 0.22 + if provenance.get("chain_reachable"): bonus += 0.14 bonus += min(0.28, 0.08 * int(provenance.get("chain_target_hits") or 0)) From 4a7dd31c554a7efe80372850b847cad8ee73e4fb Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 20:25:31 +0530 Subject: [PATCH 028/109] feat: synthesize compact values for distributed list answers --- src/ultimate_memory/answer.py | 143 +++++++++++++++++++++++++++++++--- 1 file changed, 132 insertions(+), 11 deletions(-) diff --git a/src/ultimate_memory/answer.py b/src/ultimate_memory/answer.py index 635f5d9..0894a89 100644 --- a/src/ultimate_memory/answer.py +++ b/src/ultimate_memory/answer.py @@ -512,11 +512,99 @@ def _is_list_question(question: str) -> bool: ) +def _compact_list_values(question: str, sentence: str) -> list[str]: + """Extract compact candidate values from a relevant sentence. + + This is deliberately schema/generic: it recognizes relation shapes rather + than benchmark entities or known answers. + """ + lower_q = question.lower() + values: list[str] = [] + + # Quoted works/titles are high-precision list values. + if re.search(r"\bbooks?|titles?|movies?|films?|songs?|works?\b", lower_q): + values.extend( + match.group(1).strip() + for match in re.finditer(r'["“]([^"”]{2,90})["”]', sentence) + ) + + # Travel / location histories. + if re.search(r"\b(?:cities|places|states|countries|where)\b", lower_q): + for pattern in ( + re.compile( + r"\b(?:visited|went|traveled|travelled|vacationed|camped|stayed|lived)" + r"\s+(?:in|at|to)?\s*([A-Z][a-zA-Z]+(?:\s+[A-Z][a-zA-Z]+){0,2})", + re.I, + ), + re.compile( + r"\b(?:trip|vacation|camping)\s+(?:in|at|to)\s+" + r"([A-Z][a-zA-Z]+(?:\s+[A-Z][a-zA-Z]+){0,2})", + re.I, + ), + ): + values.extend(match.group(1).strip() for match in pattern.finditer(sentence)) + + # Product/service/class offerings: capture coordinated objects after the verb. + if re.search(r"\b(?:offer|provide|include|services?|classes?|training|workshops?)\b", lower_q): + for match in re.finditer( + r"\b(?:offers?|provides?|includes?|has)\s+([^.!?]{3,140})", + sentence, + re.I, + ): + phrase = match.group(1) + phrase = re.split(r"\b(?:because|so that|which|where|when)\b", phrase, maxsplit=1, flags=re.I)[0] + values.extend( + part.strip(" ,.;:-") + for part in re.split(r",|\band\b|\bor\b", phrase, flags=re.I) + if 2 <= len(part.strip()) <= 80 + ) + + # Generic "I do/read/paint/attend X and Y" histories. Use only when the + # question itself asks for a plural/set answer. + if re.search(r"\b(?:what|which)\s+[a-z]+s\b|\bwhat\s+.+?\s+has\b", lower_q): + for match in re.finditer( + r"\b(?:do|does|did|done|read|reads|painted|paints|attended|attends|" + r"participated\s+in|practiced|practises|practices|tried|uses?|enjoys?|likes?)\s+" + r"([^.!?]{2,120})", + sentence, + re.I, + ): + phrase = re.split( + r"\b(?:because|since|when|while|which|that|to\s+help|to\s+make)\b", + match.group(1), + maxsplit=1, + flags=re.I, + )[0] + values.extend( + part.strip(" ,.;:-") + for part in re.split(r",|\band\b|\bor\b", phrase, flags=re.I) + if 2 <= len(part.strip()) <= 70 + ) + + # De-duplicate and discard obvious dialogue scaffolding. + cleaned: list[str] = [] + seen: set[str] = set() + for value in values: + value = re.sub(r"^(?:a|an|the)\s+", "", value.strip(), flags=re.I) + value = re.sub(r"^(?:my|our|his|her|their)\s+", "", value, flags=re.I) + if not value or _GREETING_ONLY_RE.match(value): + continue + if value.lower() in {"it", "them", "this", "that", "things", "stuff"}: + continue + key = value.casefold() + if key in seen: + continue + seen.add(key) + cleaned.append(value) + return cleaned + + def _list_answer(question: str, normalized: list[_ContextItem], max_chars: int) -> str | None: question_words = _content_words(question) entities = _question_entities(question) - scored: list[tuple[float, str]] = [] - seen: set[str] = set() + scored_sentences: list[tuple[float, str]] = [] + seen_sentences: set[str] = set() + for item in normalized: for sentence in _split_sentences(item.text): if len(sentence) < 8: @@ -524,28 +612,61 @@ def _list_answer(question: str, normalized: list[_ContextItem], max_chars: int) overlap = _overlap_score(question_words, sentence, entities) lower = sentence.lower() relation_bonus = 0.0 - if re.search(r"\b(?:visit|visited|went to|trip to|travel(?:ed|led)? to)\b", lower): + if re.search( + r"\b(?:visit|visited|went to|trip to|travel(?:ed|led)? to|" + r"read|painted|attended|participated|camped|vacationed)\b", + lower, + ): relation_bonus += 0.45 - if re.search(r"\b(?:offer|offering|provide|provides|classes|workshops|training|services)\b", lower): + if re.search( + r"\b(?:offer|offering|provide|provides|classes|workshops|training|services)\b", + lower, + ): relation_bonus += 0.45 if overlap < 0.12 and relation_bonus == 0.0: continue key = re.sub(r"\s+", " ", sentence.strip()).lower() - if key in seen: + if key in seen_sentences: continue - seen.add(key) - scored.append((overlap + relation_bonus + min(item.score, 1.0) * 0.12, sentence.strip())) - if not scored: + seen_sentences.add(key) + scored_sentences.append( + ( + overlap + relation_bonus + min(item.score, 1.0) * 0.12, + sentence.strip(), + ) + ) + + if not scored_sentences: return None - scored.sort(key=lambda x: x[0], reverse=True) + scored_sentences.sort(key=lambda item: item[0], reverse=True) + + values: list[str] = [] + seen_values: set[str] = set() + for _, sentence in scored_sentences[:10]: + for value in _compact_list_values(question, sentence): + key = value.casefold() + if key in seen_values: + continue + seen_values.add(key) + values.append(value) + if len(values) >= 8: + break + if len(values) >= 8: + break + + if values: + compact = ", ".join(values) + return _truncate(compact, max_chars) + + # Last-resort evidence aggregation when no structured values were extractable. chosen: list[str] = [] used = 0 - for _, sentence in scored[:6]: + for _, sentence in scored_sentences[:6]: if used + len(sentence) > max_chars and chosen: continue chosen.append(sentence) used += len(sentence) + 2 - if len(chosen) >= 3: + if len(chosen) >= 2: break return _truncate(" ".join(chosen), max_chars) if chosen else None From eaa47477fc008786afc87b4777c438690efa5f2b Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 20:25:41 +0530 Subject: [PATCH 029/109] test: require compact distributed-list synthesis --- tests/test_answer.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/test_answer.py b/tests/test_answer.py index cd765c0..74fb40c 100644 --- a/tests/test_answer.py +++ b/tests/test_answer.py @@ -297,3 +297,23 @@ def test_answer_expands_to_adjacent_conversation_turn(tmp_path): for item in result["search"].get("results", []) ) is False # neighbors are answer-context expansion, not base search output assert "guinea pig" in " ".join(result["contexts_used"]).lower() + + +def test_list_synthesis_returns_compact_locations(): + contexts = [ + {"text": "Jon: I visited Paris last winter.", "score": 0.9}, + {"text": "Jon: I traveled to Rome this summer.", "score": 0.8}, + ] + answer = synthesize_answer("Which cities has Jon visited?", contexts) + assert "Paris" in answer and "Rome" in answer + assert len(answer) < 80 + + +def test_list_synthesis_extracts_quoted_titles(): + contexts = [ + {"text": 'Alex: I read "Dune" last month.', "score": 0.9}, + {"text": 'Alex: I also read "The Hobbit" this year.', "score": 0.8}, + ] + answer = synthesize_answer("What books has Alex read?", contexts) + assert "Dune" in answer and "The Hobbit" in answer + assert len(answer) < 80 From 1def955ded394ee68c63d90ed3ee8ffed53c2509 Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 20:26:12 +0530 Subject: [PATCH 030/109] feat: index adjacent conversational question-response pairs --- src/ultimate_memory/router.py | 51 +++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/src/ultimate_memory/router.py b/src/ultimate_memory/router.py index 3e9f379..2312c7f 100644 --- a/src/ultimate_memory/router.py +++ b/src/ultimate_memory/router.py @@ -998,9 +998,11 @@ def _dialogue_turn_memory( stamp = event_time or now_iso() chunks: list[MemoryChunk] = [] atoms: list[AtomicMemory] = [] + resolved_turns: list[tuple[object, str]] = [] for turn in turns: resolved = resolve_relative_dates(turn.utterance, anchor) + resolved_turns.append((turn, resolved)) line_text = f"[{turn.dia_id}] {turn.speaker}: {resolved}" informative = len(resolved.strip()) >= 20 chunks.append( @@ -1040,6 +1042,55 @@ def _dialogue_turn_memory( ) ) + # Index adjacent conversational pairs so question/request terms and + # their response live in the same searchable evidence unit. This improves + # discourse-aware recall without inventing or summarizing any facts. + for index in range(len(resolved_turns) - 1): + first, first_text = resolved_turns[index] + second, second_text = resolved_turns[index + 1] + first_dia = getattr(first, "dia_id", "") + second_dia = getattr(second, "dia_id", "") + first_speaker = getattr(first, "speaker", "") + second_speaker = getattr(second, "speaker", "") + is_query_like = ( + "?" in first_text + or bool( + re.search( + r"^(?:can|could|would|will|please|tell|show|explain|describe|" + r"what|when|where|who|why|how)\b", + first_text.strip(), + re.I, + ) + ) + ) + if not is_query_like: + continue + pair_text = ( + f"[{first_dia}] {first_speaker}: {first_text}\n" + f"[{second_dia}] {second_speaker}: {second_text}" + ) + chunks.append( + MemoryChunk( + id=f"pair:{safe_session}:{first_dia}:{second_dia}", + text=pair_text, + source_path=str(log_path), + title=f"{first_speaker} → {second_speaker} [{first_dia}/{second_dia}]", + memory_type=MemoryType.FACT, + project_path=project_path, + tags=[*tags, "conversation-pair"], + metadata={ + "client": client, + "session_id": session_id, + "pair": True, + "question_dia_id": first_dia, + "answer_dia_id": second_dia, + "question_speaker": first_speaker, + "answer_speaker": second_speaker, + "event_time": event_time, + }, + ) + ) + return chunks, atoms def _commit_atoms_from_reflection( From c619c09356a0b8ed77a02ea821566ad4201e6706 Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 20:26:24 +0530 Subject: [PATCH 031/109] test: require searchable question-response pair chunks --- tests/test_answer.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/test_answer.py b/tests/test_answer.py index 74fb40c..01a1ee8 100644 --- a/tests/test_answer.py +++ b/tests/test_answer.py @@ -317,3 +317,25 @@ def test_list_synthesis_extracts_quoted_titles(): answer = synthesize_answer("What books has Alex read?", contexts) assert "Dune" in answer and "The Hobbit" in answer assert len(answer) < 80 + + +def test_ingestion_indexes_question_response_pair(tmp_path): + router = MemoryRouter(make_settings(tmp_path)) + transcript = """--- +session_date: 10 May 2023 +--- +[D1:1] Melanie: What pet do you have? +[D1:2] Caroline: I have a guinea pig named Clover. +""" + router.ingest_log( + client="test", + session_id="pair-index", + transcript_or_path=transcript, + project_path="/project", + ) + result = router.search("pet Caroline", project_path="/project", limit=10) + pair_hits = [ + item for item in result["results"] + if "What pet do you have?" in item["text"] and "guinea pig" in item["text"] + ] + assert pair_hits From 32ba7134a4cd7969a247c4a276e6ee97170409ca Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 20:27:11 +0530 Subject: [PATCH 032/109] feat: bound local semantic candidates by query entities --- src/ultimate_memory/store.py | 53 ++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/src/ultimate_memory/store.py b/src/ultimate_memory/store.py index b55bf1a..f0de1f5 100644 --- a/src/ultimate_memory/store.py +++ b/src/ultimate_memory/store.py @@ -406,6 +406,59 @@ def list_active_atoms( rows = conn.execute(sql, params).fetchall() return [self._row_to_atom(row) for row in rows] + def list_active_atoms_for_entities( + self, + entities: list[str], + *, + project_path: str | None = None, + limit: int = 240, + as_of: str | None = None, + ) -> list[AtomicMemory]: + """Return active atoms mentioning any requested entity. + + This is used to bound local semantic search without requiring a graph or + external vector database. + """ + cleaned = [entity.strip().casefold() for entity in entities if entity.strip()] + if not cleaned: + return self.list_active_atoms( + project_path=project_path, + limit=limit, + as_of=as_of, + ) + + if as_of: + clauses = ["valid_from <= ?", "(valid_until is null or valid_until > ?)"] + params: list[object] = [as_of, as_of] + else: + clauses = ["valid_until is null", "superseded_by is null"] + params = [] + if project_path: + clauses.append("(project_path = ? or project_path is null or project_path = '')") + params.append(project_path) + + entity_clauses: list[str] = [] + for entity in cleaned: + entity_clauses.append( + "(lower(entities_json) like ? or lower(text) like ?)" + ) + pattern = f"%{entity}%" + params.extend([pattern, pattern]) + clauses.append("(" + " or ".join(entity_clauses) + ")") + params.append(limit) + + with self._connect() as conn: + rows = conn.execute( + f""" + select * from memory_atoms + where {' and '.join(clauses)} + order by salience desc, created_at desc + limit ? + """, + params, + ).fetchall() + return [self._row_to_atom(row) for row in rows] + def search_atoms( self, query: str, From 2c365bb6498e7788e8d06097cab94ae2c01873f5 Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 20:27:15 +0530 Subject: [PATCH 033/109] perf: scope local semantic retrieval to query entities --- src/ultimate_memory/router.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/ultimate_memory/router.py b/src/ultimate_memory/router.py index 2312c7f..6d64465 100644 --- a/src/ultimate_memory/router.py +++ b/src/ultimate_memory/router.py @@ -389,6 +389,7 @@ def search( ) -> dict: actual_limit = limit or self.settings.retrieval.default_limit self.store.refresh_salience(limit=200) + query_plan = plan_query(query, as_of=as_of) temporal_query = as_of is None and is_temporal_query(query) if temporal_query: @@ -440,12 +441,17 @@ def _graph(): def _local_semantic(): if self._vector_ready or not local_semantic_enabled(): return [] - atoms = self.store.list_active_atoms( - memory_types=memory_types, + atoms = self.store.list_active_atoms_for_entities( + query_plan.entities, project_path=project_path, - limit=max(actual_limit * 30, 240), + limit=max(actual_limit * 12, 120) + if query_plan.entities + else max(actual_limit * 20, 160), as_of=as_of, ) + if memory_types: + allowed = set(memory_types) + atoms = [atom for atom in atoms if atom.memory_type.value in allowed] return self._local_semantic.search( query, atoms, @@ -484,7 +490,6 @@ def _local_semantic(): existing_keys.add(key) results = self._apply_salience_rerank(results) - query_plan = plan_query(query, as_of=as_of) results = rerank_candidates(query, results, query_plan)[:actual_limit] touched = [ From f231be84405c66c69addd6478cb0f491e1b9a622 Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 20:27:31 +0530 Subject: [PATCH 034/109] test: cover entity-scoped local semantic candidates --- tests/test_atoms.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/test_atoms.py b/tests/test_atoms.py index f8e426e..8865125 100644 --- a/tests/test_atoms.py +++ b/tests/test_atoms.py @@ -289,3 +289,30 @@ def test_group_near_duplicates_helper(self): groups = group_near_duplicates(atoms, threshold=0.5) assert len(groups) == 1 assert {a.id for a in groups[0]} == {"1", "2"} + + +def test_entity_scoped_atom_candidates(tmp_path): + router = MemoryRouter(make_settings(tmp_path)) + router.store.upsert_atom( + AtomicMemory( + text="Caroline keeps a guinea pig named Clover.", + memory_type=MemoryType.FACT, + entities=["Caroline", "Clover"], + project_path="/project", + ) + ) + router.store.upsert_atom( + AtomicMemory( + text="Jon is opening a dance studio.", + memory_type=MemoryType.FACT, + entities=["Jon"], + project_path="/project", + ) + ) + candidates = router.store.list_active_atoms_for_entities( + ["Caroline"], + project_path="/project", + limit=10, + ) + assert any("guinea pig" in atom.text for atom in candidates) + assert all("Jon" not in atom.text for atom in candidates) From 44be2dc24792dcfe28e43bd8ae0c69cb09cc0612 Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 20:30:26 +0530 Subject: [PATCH 035/109] feat: add independent Prosus MemEval system adapter --- integrations/memeval/ultimate_memory.py | 239 ++++++++++++++++++++++++ 1 file changed, 239 insertions(+) create mode 100644 integrations/memeval/ultimate_memory.py diff --git a/integrations/memeval/ultimate_memory.py b/integrations/memeval/ultimate_memory.py new file mode 100644 index 0000000..377bb3e --- /dev/null +++ b/integrations/memeval/ultimate_memory.py @@ -0,0 +1,239 @@ +"""ProsusAI/MemEval adapter for Ultimate Memory. + +Copy this file into MemEval's ``src/agents_memory/systems/ultimate_memory.py`` +and install Ultimate Memory in the same environment. MemEval will discover it +automatically through its system registry. + +The adapter intentionally uses the benchmark-provided answer model and +text-embedding-3-small so comparisons with PropMem use the same reader and +embedding family rather than Ultimate Memory's local extractive reader. +""" +from __future__ import annotations + +import json +import os +from pathlib import Path +from tempfile import TemporaryDirectory + +from openai import OpenAI + +from agents_memory.systems._helpers import _qa_results +from ultimate_memory.config import ( + BasicMemoryConfig, + DashboardConfig, + Neo4jConfig, + PathsConfig, + QdrantConfig, + RetrievalConfig, + Settings, +) +from ultimate_memory.local_semantic import LocalSemanticIndex +from ultimate_memory.router import MemoryRouter + + +SYSTEM_INFO = { + "architecture": ( + "atomic bi-temporal memory + entity-scoped hybrid retrieval + " + "conversation windows + adaptive multi-hop planning" + ), + "infrastructure": "SQLite FTS + in-process OpenAI semantic retrieval; no external DB required", +} + + +ANSWER_PROMPT = """Answer the question using ONLY the memory evidence below. + +Known conversation participants: {participants} + +Evidence: +{evidence} + +Question: {question} + +Rules: +1. Reason from the evidence, including across multiple evidence items when needed. +2. Keep facts attached to the correct person; do not transfer another person's facts. +3. For temporal questions, use the dates/timestamps in the evidence and resolve relative dates. +4. For lists or multi-part questions, include every supported item and deduplicate them. +5. For questions asking what would/could/likely happen, make the minimal inference supported by evidence. +6. Give a direct compact answer, not an explanation. Prefer exact words from the evidence. +7. If the evidence does not support an answer, return "None". +8. Answer in the same language as the question. + +Return JSON exactly as: +{{"answer": "direct compact answer"}} +""" + + +def _settings(root: Path) -> Settings: + vault = root / "vault" + private = root / "private" + vault.mkdir(parents=True, exist_ok=True) + return Settings( + paths=PathsConfig( + repo_root=root, + basic_memory_vault=vault, + private_store=private, + ), + retrieval=RetrievalConfig( + collection_name="memeval", + embedding_model="BAAI/bge-small-en-v1.5", + chunk_chars=900, + chunk_overlap=120, + default_limit=12, + bootstrap_token_budget_chars=6000, + ), + qdrant=QdrantConfig(url="http://127.0.0.1:1"), + neo4j=Neo4jConfig( + uri="bolt://127.0.0.1:1", + user="neo4j", + password="disabled", + ), + basic_memory=BasicMemoryConfig(project="memeval", cli="basic-memory-missing"), + dashboard=DashboardConfig(host="127.0.0.1", port=8787), + ) + + +def _session_text(conversation: dict, session_key: str) -> str: + turns = conversation.get(session_key) or [] + when = conversation.get(f"{session_key}_date_time") or "" + lines = ["---"] + if when: + lines.extend([f"created_at: {when}", f"session_date: {when}"]) + lines.extend(["---", ""]) + for turn in turns: + speaker = str(turn.get("speaker") or "Speaker") + dia_id = str(turn.get("dia_id") or "") + text = str(turn.get("text") or "").strip() + prefix = f"[{dia_id}] " if dia_id else "" + lines.append(f"{prefix}{speaker}: {text}") + return "\n".join(lines) + + +def _participants(conversation: dict) -> list[str]: + names: list[str] = [] + for key in ("speaker_a", "speaker_b"): + value = str(conversation.get(key) or "").strip() + if value and value not in names: + names.append(value) + if names: + return names + for key, turns in conversation.items(): + if not key.startswith("session_") or key.endswith("_date_time"): + continue + if not isinstance(turns, list): + continue + for turn in turns: + speaker = str(turn.get("speaker") or "").strip() + if speaker and speaker not in names: + names.append(speaker) + return names[:8] + + +def run( + conv: dict, + llm_model: str, + run_judge: bool, + category_names: dict | None = None, + judge_fn: str | None = None, +) -> list[dict]: + client = OpenAI(api_key=os.environ["OPENAI_API_KEY"]) + conversation = conv["conversation"] + participants = _participants(conversation) + + with TemporaryDirectory(prefix="ultimate-memory-memeval-") as temp_dir: + root = Path(temp_dir) + project_path = str(root / "conversation") + router = MemoryRouter(_settings(root)) + + # Use the exact embedding family from the MemEval comparison rig. + def openai_embed(texts: list[str]) -> list[list[float]]: + if not texts: + return [] + response = client.embeddings.create( + model=os.environ.get("EMBEDDING_MODEL", "text-embedding-3-small"), + input=texts, + ) + ordered = sorted(response.data, key=lambda item: item.index) + return [list(item.embedding) for item in ordered] + + router._local_semantic = LocalSemanticIndex(openai_embed) + + previous_semantic = os.environ.get("ULTIMATE_MEMORY_LOCAL_SEMANTIC") + os.environ["ULTIMATE_MEMORY_LOCAL_SEMANTIC"] = "1" + try: + session_keys = sorted( + ( + key + for key in conversation + if key.startswith("session_") and not key.endswith("_date_time") + ), + key=lambda key: int(key.split("_")[1]), + ) + for session_key in session_keys: + router.ingest_log( + client="memeval", + session_id=session_key, + transcript_or_path=_session_text(conversation, session_key), + project_path=project_path, + tags=["memeval"], + ) + + def answer_fn(question: str) -> str: + result = router.answer( + question, + project_path=project_path, + limit=16, + use_llm=False, + ) + contexts: list[str] = [] + seen: set[str] = set() + for raw in result.get("contexts_used") or []: + text = str(raw).strip() + key = text.casefold() + if not text or key in seen: + continue + seen.add(key) + contexts.append(text) + if len(contexts) >= 20: + break + + evidence = "\n\n".join( + f"[Memory {index}] {text}" + for index, text in enumerate(contexts, start=1) + ) + response = client.chat.completions.create( + model=llm_model, + messages=[ + { + "role": "user", + "content": ANSWER_PROMPT.format( + participants=", ".join(participants) or "unknown", + evidence=evidence or "(no evidence retrieved)", + question=question, + ), + } + ], + response_format={"type": "json_object"}, + temperature=0, + max_tokens=512, + ) + content = response.choices[0].message.content or "{}" + try: + payload = json.loads(content) + answer = str(payload.get("answer") or "").strip() + except json.JSONDecodeError: + answer = content.strip() + return answer or "None" + + return _qa_results( + conv, + answer_fn, + run_judge, + category_names=category_names, + judge_fn=judge_fn, + ) + finally: + if previous_semantic is None: + os.environ.pop("ULTIMATE_MEMORY_LOCAL_SEMANTIC", None) + else: + os.environ["ULTIMATE_MEMORY_LOCAL_SEMANTIC"] = previous_semantic From fbf86c551ee844330c90ba1bac7a69a9a1f0126d Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 20:30:31 +0530 Subject: [PATCH 036/109] docs: document independent MemEval comparison workflow --- integrations/memeval/README.md | 54 ++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 integrations/memeval/README.md diff --git a/integrations/memeval/README.md b/integrations/memeval/README.md new file mode 100644 index 0000000..f1ff85c --- /dev/null +++ b/integrations/memeval/README.md @@ -0,0 +1,54 @@ +# Ultimate Memory on ProsusAI MemEval + +ProsusAI/MemEval is the primary external fair-comparison gate for Ultimate Memory. +It standardizes the answer LLM, embedding model, scoring pipeline, and token accounting +across memory systems. + +At the time this integration was added, MemEval's published LoCoMo leader is PropMem +at 0.605 token-F1 and 0.823 judge score. + +## Run head-to-head + +Clone MemEval next to Ultimate Memory, install both projects, and copy the adapter: + +```bash +git clone https://github.com/ProsusAI/MemEval.git +cd MemEval +uv sync --all-extras +uv pip install -e ../ultimate-memory +cp ../ultimate-memory/integrations/memeval/ultimate_memory.py \ + src/agents_memory/systems/ultimate_memory.py +``` + +Set `OPENAI_API_KEY`, then run one conversation first: + +```bash +uv run python scripts/run_full_benchmark.py \ + --systems ultimate_memory,propmem \ + --num-samples 1 \ + --llm-model gpt-4.1-mini \ + --skip-judge +``` + +For the full externally comparable LoCoMo run: + +```bash +uv run python scripts/run_full_benchmark.py \ + --systems ultimate_memory,propmem \ + --num-samples 10 \ + --llm-model gpt-4.1-mini +``` + +Do not claim a leaderboard position from Ultimate Memory's internal clean harness. +Use the MemEval result for cross-system claims. + +## Fairness + +The adapter: + +- ingests only raw conversation sessions; +- never receives the gold question category; +- uses MemEval's supplied answer model; +- uses `text-embedding-3-small` for semantic fallback; +- leaves MemEval's scoring and judge code untouched; +- counts answer-LLM tokens through MemEval's normal OpenAI instrumentation. From ad58bab3b91ea00946c188870ac9b1e3c176692b Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 20:31:36 +0530 Subject: [PATCH 037/109] feat: add evidence-linked compiled memory schema --- src/ultimate_memory/models.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/ultimate_memory/models.py b/src/ultimate_memory/models.py index 8c0a0df..846709f 100644 --- a/src/ultimate_memory/models.py +++ b/src/ultimate_memory/models.py @@ -61,6 +61,18 @@ class ReflectionPayload(BaseModel): open_questions: list[str] = Field(default_factory=list) +class CompiledMemory(BaseModel): + """Evidence-linked atomic memory produced by an optional external compiler.""" + + text: str + memory_type: MemoryType = MemoryType.FACT + entities: list[str] = Field(default_factory=list) + source_dia_ids: list[str] = Field(default_factory=list) + confidence: float = Field(default=0.8, ge=0.0, le=1.0) + valid_from: str | None = None + metadata: dict[str, Any] = Field(default_factory=dict) + + class AtomicMemory(BaseModel): """Typed, bi-temporal, salience-tracked memory unit.""" From 55f24955f6977cb3aa16c407792fcc3a3d3dadf1 Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 20:31:42 +0530 Subject: [PATCH 038/109] feat: ingest externally compiled propositions with provenance --- src/ultimate_memory/router.py | 81 +++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/src/ultimate_memory/router.py b/src/ultimate_memory/router.py index 6d64465..1d6614c 100644 --- a/src/ultimate_memory/router.py +++ b/src/ultimate_memory/router.py @@ -33,6 +33,7 @@ from .models import ( AtomicMemory, AuditEvent, + CompiledMemory, MemoryChunk, MemoryType, ReflectionPayload, @@ -703,6 +704,86 @@ def ingest_log( "reflection": reflection_result, } + def ingest_compiled_memories( + self, + memories: list[CompiledMemory], + *, + session_id: str, + project_path: str | None = None, + event_time: str | None = None, + compiler: str = "external", + ) -> dict: + """Commit externally compiled atomic memories while preserving evidence links. + + The compiler is optional; raw logs/turns remain the source of truth. This + method exists so local or hosted models can improve proposition quality + without coupling the core memory engine to any one LLM provider. + """ + created = 0 + duplicates = 0 + superseded = 0 + atom_ids: list[str] = [] + + for index, item in enumerate(memories): + text = item.text.strip() + if not text: + continue + source_refs = [ + f"session:{session_id}:dia:{dia_id}" + for dia_id in item.source_dia_ids + if dia_id + ] or [f"session:{session_id}"] + metadata = { + **item.metadata, + "compiled": True, + "compiler": compiler, + "compiler_confidence": item.confidence, + "session_id": session_id, + "source_dia_ids": item.source_dia_ids, + } + atom = AtomicMemory( + id=f"atom:compiled:{safe_slug(session_id, 'session')}:{index}", + text=text, + memory_type=item.memory_type, + project_path=project_path, + entities=list(dict.fromkeys(item.entities)), + source_refs=source_refs, + valid_from=item.valid_from or event_time or now_iso(), + event_time=event_time, + importance=min(1.0, max(0.45, item.confidence)), + metadata=metadata, + ) + outcome = self._ingest_atom(atom) + status = outcome.get("status") + if status == "created": + created += 1 + atom_ids.append(atom.id) + elif status == "duplicate": + duplicates += 1 + superseded += len(outcome.get("superseded") or []) + + self.store.write_audit( + AuditEvent( + action="ingest_compiled_memories", + payload={ + "session_id": session_id, + "compiler": compiler, + "received": len(memories), + "created": created, + "duplicates": duplicates, + "superseded": superseded, + }, + source_refs=[f"session:{session_id}"], + ) + ) + return { + "received": len(memories), + "created": created, + "duplicates": duplicates, + "superseded": superseded, + "atom_ids": atom_ids, + } + def reflect( self, payload: ReflectionPayload, From d27cf619e6e33ea040b4037464c57fec1f71c206 Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 20:32:14 +0530 Subject: [PATCH 039/109] feat: compile evidence-linked propositions in MemEval adapter --- integrations/memeval/ultimate_memory.py | 117 +++++++++++++++++++++++- 1 file changed, 116 insertions(+), 1 deletion(-) diff --git a/integrations/memeval/ultimate_memory.py b/integrations/memeval/ultimate_memory.py index 377bb3e..9d04ba9 100644 --- a/integrations/memeval/ultimate_memory.py +++ b/integrations/memeval/ultimate_memory.py @@ -27,7 +27,9 @@ RetrievalConfig, Settings, ) +from ultimate_memory.dates import parse_loose_date from ultimate_memory.local_semantic import LocalSemanticIndex +from ultimate_memory.models import CompiledMemory from ultimate_memory.router import MemoryRouter @@ -40,6 +42,40 @@ } +COMPILE_PROMPT = """Compile durable atomic memories from this raw conversation session. + +Conversation participants: {participants} +Session date: {session_date} + +Raw turns: +{turns} + +Rules: +1. Extract only information supported by the turns. Never invent or infer unstated facts. +2. Each memory must be atomic: one durable fact, preference, decision, procedure, event, + relationship, status, ownership fact, activity, or plan. +3. Name the subject explicitly. Replace first-person pronouns with the speaker name. +4. Preserve exact names, places, titles, dates, quantities, and temporal qualifiers. +5. Keep separate facts separate; do not merge unrelated information. +6. Include the dialogue IDs that directly support each memory. +7. entities must contain the people/organizations/places directly involved. +8. memory_type must be one of: fact, preference, decision, procedure. +9. Do not extract greetings, compliments, filler, or questions that contain no answer. +10. Confidence is 0-1 and should reflect how explicitly the memory is stated. + +Return JSON: +{{"memories": [ + {{ + "text": "explicit atomic memory", + "memory_type": "fact", + "entities": ["Entity"], + "source_dia_ids": ["D1:2"], + "confidence": 0.95 + }} +]}} +""" + + ANSWER_PROMPT = """Answer the question using ONLY the memory evidence below. Known conversation participants: {participants} @@ -109,6 +145,57 @@ def _session_text(conversation: dict, session_key: str) -> str: return "\n".join(lines) +def _compile_session( + client: OpenAI, + model: str, + *, + participants: list[str], + session_date: str, + session_text: str, +) -> list[CompiledMemory]: + turn_lines = [ + line + for line in session_text.splitlines() + if line.strip().startswith("[D") + ] + if not turn_lines: + return [] + response = client.chat.completions.create( + model=model, + messages=[ + { + "role": "user", + "content": COMPILE_PROMPT.format( + participants=", ".join(participants) or "unknown", + session_date=session_date or "unknown", + turns="\n".join(turn_lines), + ), + } + ], + response_format={"type": "json_object"}, + temperature=0, + max_tokens=4096, + ) + content = response.choices[0].message.content or "{}" + try: + payload = json.loads(content) + except json.JSONDecodeError: + return [] + raw_memories = payload.get("memories") or payload.get("facts") or [] + if not isinstance(raw_memories, list): + return [] + + compiled: list[CompiledMemory] = [] + for raw in raw_memories[:60]: + if not isinstance(raw, dict): + continue + try: + compiled.append(CompiledMemory.model_validate(raw)) + except Exception: + continue + return compiled + + def _participants(conversation: dict) -> list[str]: names: list[str] = [] for key in ("speaker_a", "speaker_b"): @@ -169,15 +256,43 @@ def openai_embed(texts: list[str]) -> list[list[float]]: ), key=lambda key: int(key.split("_")[1]), ) + compiled_total = 0 for session_key in session_keys: + raw_session = _session_text(conversation, session_key) router.ingest_log( client="memeval", session_id=session_key, - transcript_or_path=_session_text(conversation, session_key), + transcript_or_path=raw_session, project_path=project_path, tags=["memeval"], ) + session_date = str( + conversation.get(f"{session_key}_date_time") or "" + ) + compiled = _compile_session( + client, + llm_model, + participants=participants, + session_date=session_date, + session_text=raw_session, + ) + parsed_date = parse_loose_date(session_date) + event_time = parsed_date.isoformat() if parsed_date else None + outcome = router.ingest_compiled_memories( + compiled, + session_id=session_key, + project_path=project_path, + event_time=event_time, + compiler=f"memeval:{llm_model}", + ) + compiled_total += int(outcome.get("created") or 0) + + print( + f" Ingested: sessions={len(session_keys)}, " + f"compiled_memories={compiled_total}" + ) + def answer_fn(question: str) -> str: result = router.answer( question, From 27cb889d87a05cfaec5974c97076561e14e87861 Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 20:32:27 +0530 Subject: [PATCH 040/109] test: cover evidence-linked compiled memory ingestion --- tests/test_atoms.py | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/tests/test_atoms.py b/tests/test_atoms.py index 8865125..ff030d2 100644 --- a/tests/test_atoms.py +++ b/tests/test_atoms.py @@ -24,7 +24,7 @@ RetrievalConfig, Settings, ) -from ultimate_memory.models import AtomicMemory, MemoryType, ReflectionPayload +from ultimate_memory.models import AtomicMemory, CompiledMemory, MemoryType, ReflectionPayload from ultimate_memory.router import MemoryRouter @@ -316,3 +316,29 @@ def test_entity_scoped_atom_candidates(tmp_path): ) assert any("guinea pig" in atom.text for atom in candidates) assert all("Jon" not in atom.text for atom in candidates) + + +def test_ingest_compiled_memories_preserves_evidence_links(tmp_path): + router = MemoryRouter(make_settings(tmp_path)) + result = router.ingest_compiled_memories( + [ + CompiledMemory( + text="Caroline has a guinea pig named Clover.", + memory_type=MemoryType.FACT, + entities=["Caroline", "Clover"], + source_dia_ids=["D1:2"], + confidence=0.97, + ) + ], + session_id="session_1", + project_path="/project", + event_time="2023-05-10T00:00:00", + compiler="test-compiler", + ) + assert result["created"] == 1 + atoms = router.list_atoms(query="guinea pig", project_path="/project", limit=5) + atom = atoms["atoms"][0] + assert atom["metadata"]["compiled"] is True + assert atom["metadata"]["compiler"] == "test-compiler" + assert atom["metadata"]["source_dia_ids"] == ["D1:2"] + assert atom["source_refs"] == ["session:session_1:dia:D1:2"] From 08c116b0d49d839da95a6923d249b5052216cf76 Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 20:32:54 +0530 Subject: [PATCH 041/109] ci: cancel stale benchmark runs on newer commits --- .github/workflows/tests.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 8d784da..7f79443 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -6,6 +6,10 @@ on: pull_request: branches: [master] +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: test: runs-on: ubuntu-latest From 5c657accbf248780125a2b6baad3688062c33fe3 Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 20:32:58 +0530 Subject: [PATCH 042/109] ci: cancel stale benchmark runs on newer commits --- .github/workflows/full-clean-benchmark.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/full-clean-benchmark.yml b/.github/workflows/full-clean-benchmark.yml index 0167b91..5e4b860 100644 --- a/.github/workflows/full-clean-benchmark.yml +++ b/.github/workflows/full-clean-benchmark.yml @@ -4,6 +4,10 @@ on: pull_request: workflow_dispatch: +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: clean-benchmark: runs-on: ubuntu-latest From c5d623d172e816895352908830e68140214b008f Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 20:33:04 +0530 Subject: [PATCH 043/109] ci: cancel stale benchmark runs on newer commits --- .github/workflows/reader-benchmark.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/reader-benchmark.yml b/.github/workflows/reader-benchmark.yml index 032c993..ccefdfa 100644 --- a/.github/workflows/reader-benchmark.yml +++ b/.github/workflows/reader-benchmark.yml @@ -4,6 +4,10 @@ on: pull_request: workflow_dispatch: +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: reader-benchmark: runs-on: ubuntu-latest From e44590710a24a3ddcda8c89176022ae457f47cea Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 20:33:50 +0530 Subject: [PATCH 044/109] feat: add diversity-aware context packet compiler --- src/ultimate_memory/context_compiler.py | 93 +++++++++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 src/ultimate_memory/context_compiler.py diff --git a/src/ultimate_memory/context_compiler.py b/src/ultimate_memory/context_compiler.py new file mode 100644 index 0000000..df2ba18 --- /dev/null +++ b/src/ultimate_memory/context_compiler.py @@ -0,0 +1,93 @@ +"""Compact retrieved evidence into a diverse, token-efficient context packet.""" +from __future__ import annotations + +import re +from collections import Counter + +_WORD_RE = re.compile(r"[A-Za-z0-9_.-]{2,}") + + +def _tokens(text: str) -> set[str]: + return {match.group(0).casefold() for match in _WORD_RE.finditer(text)} + + +def _similarity(left: str, right: str) -> float: + a = _tokens(left) + b = _tokens(right) + if not a or not b: + return 0.0 + return len(a & b) / len(a | b) + + +def _source_bucket(item: dict) -> str: + provenance = item.get("provenance") or {} + session_id = provenance.get("session_id") + if session_id: + return f"session:{session_id}" + source = str(provenance.get("source") or "") + source_path = str(item.get("source_path") or "") + return source_path or source or str(item.get("id") or "unknown") + + +def compile_context_packet( + contexts: list[dict], + *, + max_chars: int = 12000, + max_items: int = 24, + near_duplicate_threshold: float = 0.84, + max_per_source: int = 6, +) -> list[dict]: + """Select high-ranked, diverse evidence under a deterministic size budget. + + The input order is treated as ranking order. Near-duplicates and repeated + copies of the same session are suppressed, but small direct-turn evidence + can coexist with a richer conversation window when they are materially + different. + """ + if not contexts or max_chars <= 0 or max_items <= 0: + return [] + + selected: list[dict] = [] + selected_texts: list[str] = [] + source_counts: Counter[str] = Counter() + used_chars = 0 + + for item in contexts: + text = str(item.get("text") or "").strip() + if not text: + continue + + bucket = _source_bucket(item) + if source_counts[bucket] >= max_per_source: + continue + + if any( + _similarity(text, existing) >= near_duplicate_threshold + for existing in selected_texts + ): + continue + + remaining = max_chars - used_chars + if remaining <= 0: + break + + # Keep evidence boundaries intact whenever possible. Only truncate one + # oversized first item rather than filling the packet with fragments. + if len(text) > remaining: + if selected: + continue + clipped = text[:remaining].rsplit(" ", 1)[0].rstrip() + if not clipped: + continue + item = dict(item) + item["text"] = clipped + text = clipped + + selected.append(item) + selected_texts.append(text) + source_counts[bucket] += 1 + used_chars += len(text) + 2 + if len(selected) >= max_items: + break + + return selected From f47674d7889104386f2805642a0d5db74005c364 Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 20:34:16 +0530 Subject: [PATCH 045/109] test: cover context dedupe diversity and budgets --- tests/test_context_compiler.py | 38 ++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 tests/test_context_compiler.py diff --git a/tests/test_context_compiler.py b/tests/test_context_compiler.py new file mode 100644 index 0000000..dfd7366 --- /dev/null +++ b/tests/test_context_compiler.py @@ -0,0 +1,38 @@ +from ultimate_memory.context_compiler import compile_context_packet + + +def item(text, score=1.0, session="s1"): + return { + "id": text[:8], + "text": text, + "score": score, + "provenance": {"session_id": session}, + } + + +def test_near_duplicates_are_removed(): + contexts = [ + item("Caroline has a guinea pig named Clover."), + item("Caroline has a guinea pig named Clover!"), + item("Jon runs a dance studio.", session="s2"), + ] + packet = compile_context_packet(contexts) + assert len(packet) == 2 + + +def test_budget_is_respected(): + contexts = [ + item("alpha " * 20, session="a"), + item("beta " * 20, session="b"), + ] + packet = compile_context_packet(contexts, max_chars=130) + assert sum(len(x["text"]) + 2 for x in packet) <= 132 + + +def test_source_diversity_cap(): + contexts = [ + item(f"distinct fact number {i} with unique token x{i}", session="same") + for i in range(10) + ] + packet = compile_context_packet(contexts, max_per_source=3) + assert len(packet) == 3 From ca3211e12ead494961351397c1e207cf952a90b7 Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 20:34:49 +0530 Subject: [PATCH 046/109] feat: compile diverse evidence packets under context budgets --- src/ultimate_memory/router.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/ultimate_memory/router.py b/src/ultimate_memory/router.py index 1d6614c..7a30e07 100644 --- a/src/ultimate_memory/router.py +++ b/src/ultimate_memory/router.py @@ -23,6 +23,7 @@ from .chain import rank_evidence_chain from .claims import ensure_claim_metadata, structured_conflict_score from .config import Settings, load_settings +from .context_compiler import compile_context_packet from .dates import parse_loose_date, resolve_relative_dates from .extraction import ( extract_from_transcript, @@ -336,7 +337,14 @@ def answer( rich_contexts = rank_evidence_chain(question, rich_contexts) else: rich_contexts.sort(key=lambda item: float(item.get("score") or 0.0), reverse=True) - rich_contexts = rich_contexts[: max(limit * 3, 18)] + + context_budget = 18000 if plan.kind == "multi_hop" else 12000 + rich_contexts = compile_context_packet( + rich_contexts, + max_chars=context_budget, + max_items=max(limit * 3, 18), + max_per_source=8 if plan.kind == "multi_hop" else 5, + ) use_local_llm = use_llm_from_env() if use_llm is None else use_llm context_texts = [str(item.get("text") or "") for item in rich_contexts if item.get("text")] @@ -370,6 +378,8 @@ def answer( "answer": answer_text, "f1_text": f1_ready_text(answer_text), "contexts_used": context_texts, + "context_chars": sum(len(text) for text in context_texts), + "context_items": len(context_texts), "search": search_result, "query_plan": plan.model_dump(), "hop_entities": list(dict.fromkeys(hop_entities)), From af413e9314bca5c9db22e6890f24d741e9ed91d8 Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 20:36:34 +0530 Subject: [PATCH 047/109] fix: exclude auxiliaries from plural distributed-query detection --- src/ultimate_memory/planner.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ultimate_memory/planner.py b/src/ultimate_memory/planner.py index 13c3b8a..6b9989d 100644 --- a/src/ultimate_memory/planner.py +++ b/src/ultimate_memory/planner.py @@ -55,8 +55,8 @@ class QueryPlan(BaseModel): ) _LIST_OR_SET_RE = re.compile( - r"\bwhich\s+(?:[a-z]+s|cities|places|countries|states|books|games|activities|items|things|ways|types|kinds)\b|" - r"\bwhat\s+(?:[a-z]+s|cities|places|countries|states|books|games|activities|items|things|ways|types|kinds)\b|" + r"\bwhich\s+(?!(?:does|has|is|was|this)\b)(?:[a-z]+s|cities|places|countries|states|books|games|activities|items|things|ways|types|kinds)\b|" + r"\bwhat\s+(?!(?:does|has|is|was|this)\b)(?:[a-z]+s|cities|places|countries|states|books|games|activities|items|things|ways|types|kinds)\b|" r"\bwhat\s+does\s+.+?\s+(?:offer|provide|include|do\s+to)\b|" r"\b(?:all|multiple|several)\s+(?:[a-z]+s|cities|places|books|activities|items|things|ways|types)\b|" r"\bwhere\s+has\s+.+?\s+(?:camped|traveled|travelled|visited|stayed|lived)\b|" From 2fe67330326853a0befdbd3f085c976fa7c65146 Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 20:36:37 +0530 Subject: [PATCH 048/109] fix: normalize punctuation in near-duplicate context detection --- src/ultimate_memory/context_compiler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ultimate_memory/context_compiler.py b/src/ultimate_memory/context_compiler.py index df2ba18..9eb64ca 100644 --- a/src/ultimate_memory/context_compiler.py +++ b/src/ultimate_memory/context_compiler.py @@ -4,7 +4,7 @@ import re from collections import Counter -_WORD_RE = re.compile(r"[A-Za-z0-9_.-]{2,}") +_WORD_RE = re.compile(r"[A-Za-z0-9_-]{2,}") def _tokens(text: str) -> set[str]: From c33d105cb39194c2d04962f2d5adc6ea991b4c74 Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 20:36:43 +0530 Subject: [PATCH 049/109] fix: keep compact list answers and normalize temporal date output --- src/ultimate_memory/answer.py | 41 +++++++++++++++++++++++++++++------ 1 file changed, 34 insertions(+), 7 deletions(-) diff --git a/src/ultimate_memory/answer.py b/src/ultimate_memory/answer.py index 0894a89..94aaa66 100644 --- a/src/ultimate_memory/answer.py +++ b/src/ultimate_memory/answer.py @@ -143,7 +143,7 @@ _GREETING_ONLY_RE = re.compile( r"^(?:\[D\d+:\d+\]\s*)?(?:[A-Z][a-z]+\s*:\s*)?" - r"(?:hey|hi|hello|thanks|thank\s+you|wow|great|awesome|nice|cool|sure|yep|yeah)" + r"(?:hey|hi|hello|thanks(?:\s+a\s+bunch)?|thank\s+you|wow|great|awesome|nice|cool|sure|yep|yeah)" r"(?:[\s,!.'-]+[A-Z][a-z]+)?[!?.]*$", re.I, ) @@ -391,6 +391,30 @@ def _extract_date_spans(sentence: str) -> list[str]: return spans +def _display_date(span: str) -> str: + """Render ISO-style dates as compact human-readable dates.""" + value = span.strip() + match = re.fullmatch(r"(\d{4})-(\d{2})-(\d{2})(?:[T ][^\s]+)?", value) + if match: + year, month, day = map(int, match.groups()) + months = ( + "January", "February", "March", "April", "May", "June", + "July", "August", "September", "October", "November", "December", + ) + if 1 <= month <= 12 and 1 <= day <= 31: + return f"{day} {months[month - 1]} {year}" + match = re.fullmatch(r"(\d{4})-(\d{2})", value) + if match: + year, month = map(int, match.groups()) + months = ( + "January", "February", "March", "April", "May", "June", + "July", "August", "September", "October", "November", "December", + ) + if 1 <= month <= 12: + return f"{months[month - 1]} {year}" + return value + + def _is_relative_only_date(span: str) -> bool: return bool(_RELATIVE_ONLY_DATE_RE.match(span.strip())) @@ -1152,8 +1176,8 @@ def synthesize_answer( if not _is_relative_only_date(date) or all( _is_relative_only_date(d) for _, d in dated ): - return _truncate(date, max_chars) - return _truncate(dated[0][1], max_chars) + return _truncate(_display_date(date), max_chars) + return _truncate(_display_date(dated[0][1]), max_chars) dates = _prefer_absolute_date_spans( _extract_duration_spans(best.text) or _extract_date_spans(best.text) @@ -1161,7 +1185,7 @@ def synthesize_answer( or _extract_date_spans(best.sentence) ) if dates: - return _truncate(dates[0], max_chars) + return _truncate(_display_date(dates[0]), max_chars) # Avoid vague relative answers when no absolute date is available. if re.search( r"\b(?:last|next|this)\s+(?:monday|tuesday|wednesday|thursday|friday|saturday|sunday|week|weekend|month)\b", @@ -1173,7 +1197,7 @@ def synthesize_answer( _extract_date_spans(cand.text) or _extract_date_spans(cand.sentence) ) if alt and not _is_relative_only_date(alt[0]): - return _truncate(alt[0], max_chars) + return _truncate(_display_date(alt[0]), max_chars) if kind == "where": locs = _extract_location_spans(best.text) or _extract_location_spans(best.sentence) @@ -1209,9 +1233,12 @@ def synthesize_answer( list_answer = _list_answer(question, normalized, max_chars) if list_answer: parts = _split_sentences(list_answer) - if len(parts) >= 2 or any( + if "," in list_answer or len(parts) >= 2 or any( cue in list_answer.lower() - for cue in ("visited", "trip to", "offer", "provid", "classes", "workshops", "training") + for cue in ( + "visited", "trip to", "offer", "provid", + "classes", "workshops", "training", + ) ): return list_answer From 27adb37449a48324f10c6679e9b06cc4b8b20da7 Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 20:36:56 +0530 Subject: [PATCH 050/109] fix: avoid auxiliary false positives in hybrid reader routing --- src/ultimate_memory/reader.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ultimate_memory/reader.py b/src/ultimate_memory/reader.py index 9c6a0c7..90dc902 100644 --- a/src/ultimate_memory/reader.py +++ b/src/ultimate_memory/reader.py @@ -29,7 +29,8 @@ ) _DISTRIBUTED = re.compile( r"\bboth\b|\ball\b|" - r"\b(?:what|which)\s+[a-z]+s\b.*\b(?:has|have|did|does|are|were)\b|" + r"\b(?:what|which)\s+(?!(?:does|has|is|was|this)\b)[a-z]+s\b" + r".*\b(?:has|have|did|does|are|were)\b|" r"\bwhere\s+has\s+.+?\s+(?:camped|traveled|travelled|visited|stayed|lived)\b|" r"\bwhat\s+does\s+.+?\s+(?:offer|provide|include|do\s+to)\b", re.I, From 42c749213b888be1d7117ab2cffe35540645b8c6 Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 20:39:39 +0530 Subject: [PATCH 051/109] feat: intersect multi-person evidence for shared-answer questions --- src/ultimate_memory/answer.py | 108 ++++++++++++++++++++++++++++++++++ 1 file changed, 108 insertions(+) diff --git a/src/ultimate_memory/answer.py b/src/ultimate_memory/answer.py index 94aaa66..d592cc9 100644 --- a/src/ultimate_memory/answer.py +++ b/src/ultimate_memory/answer.py @@ -623,6 +623,110 @@ def _compact_list_values(question: str, sentence: str) -> list[str]: return cleaned +def _stem_shared_token(token: str) -> str: + lower = token.casefold().strip(".,;:!?'\"") + for suffix in ("ing", "ed", "es", "s"): + if len(lower) > len(suffix) + 3 and lower.endswith(suffix): + lower = lower[: -len(suffix)] + break + return lower + + +def _shared_entity_answer( + question: str, + normalized: list[_ContextItem], + max_chars: int, +) -> str | None: + """Find compact values independently supported for every named entity.""" + if not re.search(r"\bboth\b|\bin common\b|\bshared\b|\beach\b", question, re.I): + return None + + entities = sorted(_question_entities(question)) + if len(entities) < 2: + return None + + per_entity_values: dict[str, list[str]] = {entity: [] for entity in entities} + per_entity_tokens: dict[str, Counter[str]] = {entity: Counter() for entity in entities} + q_words = _content_words(question) + + for item in normalized: + for sentence in _split_sentences(item.text): + lower = sentence.casefold() + matched_entities = [entity for entity in entities if entity in lower] + if not matched_entities: + continue + + compact = _compact_list_values(question, sentence) + for entity in matched_entities: + per_entity_values[entity].extend(compact) + + # Generic relation/action fallback for non-list commonality questions. + # Keep only content words not already present in the question/entity. + for token in _content_words(sentence): + stem = _stem_shared_token(token) + if ( + stem + and stem not in {_stem_shared_token(word) for word in q_words} + and stem not in {_stem_shared_token(e) for e in entities} + and len(stem) >= 4 + ): + per_entity_tokens[entity][stem] += 1 + + # Exact compact-value intersection first (cities, titles, services, etc.). + value_sets: list[dict[str, str]] = [] + for entity in entities: + mapping: dict[str, str] = {} + for value in per_entity_values[entity]: + key = " ".join(_stem_shared_token(tok) for tok in _tokens(value)) + if key: + mapping.setdefault(key, value) + value_sets.append(mapping) + + if value_sets and all(value_sets): + common = set(value_sets[0]) + for mapping in value_sets[1:]: + common &= set(mapping) + if common: + values = [value_sets[0][key] for key in sorted(common)] + return _truncate(", ".join(values[:8]), max_chars) + + # Fallback: intersect salient content stems across each person's evidence. + token_sets = [set(counter) for counter in per_entity_tokens.values()] + if not token_sets or not all(token_sets): + return None + shared = set.intersection(*token_sets) + if not shared: + return None + + generic = { + "have", "with", "from", "that", "this", "they", "their", "your", "just", + "really", "about", "been", "want", "like", "love", "great", "good", "also", + "make", "help", "thing", "time", "need", "when", "what", "both", + } + ranked = [ + token + for token in shared + if token not in generic and token not in {_stem_shared_token(x) for x in q_words} + ] + if not ranked: + return None + + # Prefer tokens repeatedly attested across people. + ranked.sort( + key=lambda token: sum(per_entity_tokens[e][token] for e in entities), + reverse=True, + ) + best = ranked[0] + # Recover a readable surface form from evidence. + variants: Counter[str] = Counter() + for item in normalized: + for token in re.findall(r"[A-Za-z][A-Za-z'-]{2,}", item.text): + if _stem_shared_token(token) == best: + variants[token] += 1 + surface = variants.most_common(1)[0][0] if variants else best + return _truncate(surface, max_chars) + + def _list_answer(question: str, normalized: list[_ContextItem], max_chars: int) -> str | None: question_words = _content_words(question) entities = _question_entities(question) @@ -1048,6 +1152,10 @@ def synthesize_answer( list_question = _is_list_question(question) + shared_answer = _shared_entity_answer(question, normalized, max_chars) + if shared_answer: + return shared_answer + # For "when" questions, prefer contexts that actually contain date spans. if kind == "when": duration_question = bool( From 108cc937ff404d22494e9380686a25a89bf30850 Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 20:39:51 +0530 Subject: [PATCH 052/109] test: cover shared-value intersection across multiple people --- tests/test_answer.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/test_answer.py b/tests/test_answer.py index 01a1ee8..b67f695 100644 --- a/tests/test_answer.py +++ b/tests/test_answer.py @@ -339,3 +339,25 @@ def test_ingestion_indexes_question_response_pair(tmp_path): if "What pet do you have?" in item["text"] and "guinea pig" in item["text"] ] assert pair_hits + + +def test_shared_city_intersection_across_people(): + contexts = [ + {"text": "Jean: I visited Rome last spring.", "score": 0.9}, + {"text": "Jean: I also visited Paris.", "score": 0.8}, + {"text": "John: I traveled to Rome last year.", "score": 0.9}, + {"text": "John: I visited Berlin too.", "score": 0.8}, + ] + answer = synthesize_answer("Which city have both Jean and John visited?", contexts) + assert "Rome" in answer + assert "Paris" not in answer + assert "Berlin" not in answer + + +def test_shared_activity_intersection_uses_entity_evidence(): + contexts = [ + {"text": "Jon: I dance whenever I need to destress.", "score": 0.9}, + {"text": "Gina: Dancing helps me relax after stressful days.", "score": 0.9}, + ] + answer = synthesize_answer("How do Jon and Gina both like to destress?", contexts) + assert "danc" in answer.lower() From 6005ee5e28885403e3175d6e59328bc6fed4c88b Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 20:40:26 +0530 Subject: [PATCH 053/109] feat: plan duration-between-events questions as multi-evidence retrieval --- src/ultimate_memory/planner.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/ultimate_memory/planner.py b/src/ultimate_memory/planner.py index 6b9989d..d39df98 100644 --- a/src/ultimate_memory/planner.py +++ b/src/ultimate_memory/planner.py @@ -73,6 +73,12 @@ def _is_collective_multi_hop(question: str, entities: list[str]) -> bool: lower = question.lower() if _COLLECTIVE_RE.search(lower) or _LIST_OR_SET_RE.search(lower): return True + if re.search( + r"\bhow\s+long\b.*\b(?:take|took|until|from|between|before|after)\b|" + r"\b(?:duration|elapsed|time\s+between)\b", + lower, + ): + return True if len(entities) >= 2 and re.search(r"\b(?:and|versus|vs\.?|compared?\s+to)\b", lower): return True return False @@ -147,6 +153,8 @@ def _expansions(question: str) -> list[str]: expansions.append("offer provides services classes workshops training") if _is_temporal_question(question): expansions.append("date year month day when duration time") + if re.search(r"\bhow\s+long\b|\bduration\b|\belapsed\b", lower): + expansions.append("started began finished completed opened duration elapsed") return list(dict.fromkeys(expansions)) From 1c906e5ce57d74b933ea157168dfcf259bc7d482 Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 20:40:37 +0530 Subject: [PATCH 054/109] test: cover duration-between-events retrieval planning --- tests/test_planner.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/test_planner.py b/tests/test_planner.py index 2d39b3d..7953bf2 100644 --- a/tests/test_planner.py +++ b/tests/test_planner.py @@ -62,3 +62,10 @@ def test_generic_plural_history_query_uses_distributed_plan(): assert plan_query("What martial arts has Alex done?").kind == "multi_hop" assert plan_query("What books has Alex read?").kind == "multi_hop" assert plan_query("Where has Alex camped?").kind == "multi_hop" + + +def test_duration_between_events_uses_multi_evidence_plan(): + plan = plan_query("How long did it take Alex to open the studio?") + assert plan.kind == "multi_hop" + assert plan.hop_depth >= 2 + assert any("started" in expansion for expansion in plan.expansions) From e5126e2a474d01add9b9bcf3a12248717da0d1d5 Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 20:41:20 +0530 Subject: [PATCH 055/109] ci: add independent MemEval head-to-head workflow --- .github/workflows/memeval.yml | 73 +++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 .github/workflows/memeval.yml diff --git a/.github/workflows/memeval.yml b/.github/workflows/memeval.yml new file mode 100644 index 0000000..77ef7ff --- /dev/null +++ b/.github/workflows/memeval.yml @@ -0,0 +1,73 @@ +name: Independent MemEval + +on: + workflow_dispatch: + inputs: + samples: + description: "Number of LoCoMo conversations" + required: true + default: "1" + with_judge: + description: "Run GPT judge in addition to token F1" + required: true + type: boolean + default: false + +concurrency: + group: independent-memeval + cancel-in-progress: true + +jobs: + memeval: + runs-on: ubuntu-latest + timeout-minutes: 60 + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + LLM_MODEL: gpt-4.1-mini + EMBEDDING_MODEL: text-embedding-3-small + steps: + - name: Require API key + run: | + if [ -z "$OPENAI_API_KEY" ]; then + echo "::error::OPENAI_API_KEY Actions secret is not configured." + exit 1 + fi + + - name: Checkout Ultimate Memory + uses: actions/checkout@v4 + with: + path: ultimate-memory + + - name: Checkout Prosus MemEval + uses: actions/checkout@v4 + with: + repository: ProsusAI/MemEval + path: MemEval + + - name: Install uv + uses: astral-sh/setup-uv@v3 + + - name: Install MemEval and Ultimate Memory + working-directory: MemEval + run: | + uv sync --all-extras + uv pip install -e ../ultimate-memory + cp ../ultimate-memory/integrations/memeval/ultimate_memory.py src/agents_memory/systems/ultimate_memory.py + + - name: Run fair head-to-head + working-directory: MemEval + shell: bash + run: | + EXTRA="" + if [ "${{ inputs.with_judge }}" != "true" ]; then + EXTRA="--skip-judge" + fi + uv run python scripts/run_full_benchmark.py --systems ultimate_memory,propmem --num-samples "${{ inputs.samples }}" --llm-model "$LLM_MODEL" $EXTRA --output-dir benchmark-output + + - name: Upload benchmark results + if: always() + uses: actions/upload-artifact@v4 + with: + name: independent-memeval-results + path: MemEval/benchmark-output/ + if-no-files-found: warn From 45839b366e9c6ab07ccec6b173bd512da2e2d790 Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 20:41:56 +0530 Subject: [PATCH 056/109] feat: extract explicit identity and favorite scalar values first --- src/ultimate_memory/answer.py | 51 +++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/src/ultimate_memory/answer.py b/src/ultimate_memory/answer.py index d592cc9..c28e06c 100644 --- a/src/ultimate_memory/answer.py +++ b/src/ultimate_memory/answer.py @@ -181,6 +181,17 @@ r"\btransgender\b|\b(?:trans\s+)?woman\b|\b(?:trans\s+)?man\b|\bidentity\b", re.I, ) +_EXPLICIT_IDENTITY_RE = re.compile( + r"\b(transgender\s+(?:woman|man)|trans\s+(?:woman|man)|nonbinary|non-binary)\b", + re.I, +) +_FAVORITE_VALUE_RE = re.compile( + r"\b(?:my|his|her|their|[A-Z][a-z]+'s)\s+" + r"(?:favorite|favourite)\s+[^.!?,:]{0,50}?\s+(?:is|are)\s+([^.!?,;]{1,70})|" + r"\b([^.!?,;]{1,50})\s+(?:is|are)\s+" + r"(?:my|his|her|their|[A-Z][a-z]+'s)\s+(?:favorite|favourite)\b", + re.I, +) _JUNK_LINE_PATTERNS: tuple[re.Pattern[str], ...] = ( re.compile(r"^#"), @@ -315,6 +326,42 @@ def _is_identity_question(question: str) -> bool: return bool(_IDENTITY_QUESTION_RE.search(question)) +def _favorite_subject(question: str) -> bool: + return bool(re.search(r"\bfavou?rite\b", question, re.I)) + + +def _precise_scalar_answer(question: str, normalized: list["_ContextItem"]) -> str | None: + """Return high-precision scalar values before generic span ranking.""" + identity = _is_identity_question(question) + favorite = _favorite_subject(question) + + if identity: + for item in normalized: + match = _EXPLICIT_IDENTITY_RE.search(item.text) + if match: + return match.group(1).strip() + + if favorite: + q_words = _content_words(question) + entities = _question_entities(question) + candidates: list[tuple[float, str]] = [] + for item in normalized: + for sentence in _split_sentences(item.text): + match = _FAVORITE_VALUE_RE.search(sentence) + if not match: + continue + value = (match.group(1) or match.group(2) or "").strip(" .,:;-") + if not value: + continue + score = _overlap_score(q_words, sentence, entities) + candidates.append((score, value)) + if candidates: + candidates.sort(key=lambda pair: (pair[0], -len(pair[1])), reverse=True) + return candidates[0][1] + + return None + + def _dialogue_turn_bonus( text: str, *, @@ -1143,6 +1190,10 @@ def synthesize_answer( if not normalized: return "" + precise = _precise_scalar_answer(question, normalized) + if precise: + return _truncate(precise, max_chars) + temporal_bias = _question_temporal_bias(question) kind = _question_kind(question) question_words = _content_words(question) From b12c1ba82e90607d90d3f147fc4ce0dabb2d837c Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 20:42:08 +0530 Subject: [PATCH 057/109] test: cover precise identity and favorite-value extraction --- tests/test_answer.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/test_answer.py b/tests/test_answer.py index b67f695..ba4bba3 100644 --- a/tests/test_answer.py +++ b/tests/test_answer.py @@ -361,3 +361,21 @@ def test_shared_activity_intersection_uses_entity_evidence(): ] answer = synthesize_answer("How do Jon and Gina both like to destress?", contexts) assert "danc" in answer.lower() + + +def test_explicit_identity_label_beats_related_identity_sentence(): + contexts = [ + {"text": "Caroline: Painting helps me explore my identity and be true to myself.", "score": 1.0}, + {"text": "Caroline: I'm a transgender woman and coming out changed my life.", "score": 0.7}, + ] + answer = synthesize_answer("What is Caroline's identity?", contexts) + assert answer.lower() == "transgender woman" + + +def test_favorite_value_is_extracted_compactly(): + contexts = [ + {"text": "Gina: My favorite style of dance is Contemporary.", "score": 0.8}, + {"text": "Gina: Dance is a huge part of my life.", "score": 1.0}, + ] + answer = synthesize_answer("What is Gina's favorite style of dance?", contexts) + assert answer.lower() == "contemporary" From 4954f6924b969c3b70b8611722e17a3f907818df Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 20:42:25 +0530 Subject: [PATCH 058/109] feat: expose compiled-memory confidence to retrieval ranking --- src/ultimate_memory/router.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/ultimate_memory/router.py b/src/ultimate_memory/router.py index 7a30e07..85bd60d 100644 --- a/src/ultimate_memory/router.py +++ b/src/ultimate_memory/router.py @@ -1361,6 +1361,9 @@ def _atom_to_search_result(atom: AtomicMemory, *, score: float) -> SearchResult: "entities": atom.entities, "project_path": atom.project_path, "claim": atom.metadata.get("claim"), + "compiled": bool(atom.metadata.get("compiled")), + "compiler_confidence": atom.metadata.get("compiler_confidence"), + "source_dia_ids": atom.metadata.get("source_dia_ids"), "dia_id": atom.metadata.get("dia_id"), "speaker": atom.metadata.get("speaker"), "session_id": atom.metadata.get("session_id"), From 3a4bf541d10fb8b08c8f70b7add9fed70c7878a7 Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 20:42:28 +0530 Subject: [PATCH 059/109] feat: use compiler confidence as a small generic ranking signal --- src/ultimate_memory/ranking.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/ultimate_memory/ranking.py b/src/ultimate_memory/ranking.py index a719f94..ce4f48d 100644 --- a/src/ultimate_memory/ranking.py +++ b/src/ultimate_memory/ranking.py @@ -48,6 +48,11 @@ def rerank_candidates( if provenance.get("direct_turn") or provenance.get("dia_id"): direct_overlap = len(q_tokens & r_tokens) / len(q_tokens) if q_tokens else 0.0 score += 0.12 + 0.18 * direct_overlap + + if provenance.get("compiled"): + confidence = float(provenance.get("compiler_confidence") or 0.0) + score += min(0.10, max(0.0, confidence) * 0.10) + if "auto-extracted from" in text_lower: score -= 0.08 From 143d1e9d248b2db1e7af2e8e6890b3ca0b4181ae Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 20:43:12 +0530 Subject: [PATCH 060/109] feat: add hard entity-scoped evidence filter --- src/ultimate_memory/ranking.py | 39 ++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/src/ultimate_memory/ranking.py b/src/ultimate_memory/ranking.py index ce4f48d..90969db 100644 --- a/src/ultimate_memory/ranking.py +++ b/src/ultimate_memory/ranking.py @@ -15,6 +15,45 @@ def _tokens(text: str) -> set[str]: } +def filter_entity_scoped_results( + results: list[dict], + entities: list[str], + *, + min_matches: int = 2, +) -> list[dict]: + """Prefer evidence explicitly attached to the entities named in the query. + + This is intentionally a post-retrieval gate rather than a storage filter: + global memories remain available, and multi-hop callers can simply skip this + helper when bridge-entity traversal is required. + """ + entity_keys = [entity.strip().casefold() for entity in entities if entity.strip()] + if not entity_keys or not results: + return list(results) + + matched: list[dict] = [] + for item in results: + text = str(item.get("text") or "").casefold() + provenance = item.get("provenance") or {} + prov_entities = { + str(entity).strip().casefold() + for entity in provenance.get("entities") or [] + if str(entity).strip() + } + speaker = str(provenance.get("speaker") or "").strip().casefold() + + if any( + key in text + or key == speaker + or key in prov_entities + for key in entity_keys + ): + matched.append(item) + + # Never collapse a query to an unusably tiny evidence pool. + return matched if len(matched) >= min_matches else list(results) + + def rerank_candidates( query: str, results: list[SearchResult], From e153d82379dbebe3b4d011fa27fcae85aeff06da Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 20:43:25 +0530 Subject: [PATCH 061/109] feat: hard-filter direct evidence to named query entities --- src/ultimate_memory/router.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/ultimate_memory/router.py b/src/ultimate_memory/router.py index 85bd60d..421440f 100644 --- a/src/ultimate_memory/router.py +++ b/src/ultimate_memory/router.py @@ -53,7 +53,7 @@ from .llm_answer import use_llm_from_env from .local_semantic import LocalSemanticIndex, enabled as local_semantic_enabled from .planner import plan_query -from .ranking import rerank_candidates +from .ranking import filter_entity_scoped_results, rerank_candidates from .store import LocalStore logger = logging.getLogger(__name__) @@ -205,6 +205,16 @@ def answer( ) rich_contexts.extend(item for item in typed["results"] if item.get("text")) + # Direct and temporal questions should stay attached to the named + # subject. Multi-hop questions deliberately skip this hard gate because + # they must traverse bridge entities. + if plan.kind != "multi_hop" and plan.entities: + rich_contexts = filter_entity_scoped_results( + rich_contexts, + plan.entities, + min_matches=2, + ) + # Conversation adjacency is evidence: a retrieved question/request turn # is frequently answered by the immediately preceding or following turn. neighbor_seen: set[str] = { From d671444506ade0aa513514099c19ef3c0f62e064 Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 20:43:37 +0530 Subject: [PATCH 062/109] test: cover entity-scoped evidence filtering --- tests/test_entity_scope.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 tests/test_entity_scope.py diff --git a/tests/test_entity_scope.py b/tests/test_entity_scope.py new file mode 100644 index 0000000..731be20 --- /dev/null +++ b/tests/test_entity_scope.py @@ -0,0 +1,31 @@ +from ultimate_memory.ranking import filter_entity_scoped_results + + +def result(text, entities=None, speaker=None): + return { + "text": text, + "provenance": { + "entities": entities or [], + "speaker": speaker, + }, + } + + +def test_entity_scope_removes_other_people_when_enough_matches_exist(): + items = [ + result("Caroline lives in Boston.", ["Caroline"], "Caroline"), + result("Caroline likes painting.", ["Caroline"], "Caroline"), + result("Melanie likes pottery.", ["Melanie"], "Melanie"), + ] + scoped = filter_entity_scoped_results(items, ["Caroline"]) + assert len(scoped) == 2 + assert all("Melanie" not in item["text"] for item in scoped) + + +def test_entity_scope_falls_back_when_pool_is_too_small(): + items = [ + result("Caroline likes painting.", ["Caroline"], "Caroline"), + result("Melanie likes pottery.", ["Melanie"], "Melanie"), + ] + scoped = filter_entity_scoped_results(items, ["Caroline"], min_matches=2) + assert scoped == items From cfafec12b8914bbb06cf5148d1e649c4d269c67f Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 20:44:30 +0530 Subject: [PATCH 063/109] fix: project-scope SQLite chunk and FTS retrieval --- src/ultimate_memory/store.py | 58 +++++++++++++++++++++++++++++------- 1 file changed, 47 insertions(+), 11 deletions(-) diff --git a/src/ultimate_memory/store.py b/src/ultimate_memory/store.py index f0de1f5..31157d5 100644 --- a/src/ultimate_memory/store.py +++ b/src/ultimate_memory/store.py @@ -89,10 +89,18 @@ def _init_db(self) -> None: memory_type text not null, text text not null, metadata_json text not null, + project_path text, created_at text not null ) """ ) + columns = { + row["name"] + for row in conn.execute("pragma table_info(memory_index)").fetchall() + } + if "project_path" not in columns: + conn.execute("alter table memory_index add column project_path text") + conn.execute( """ create virtual table if not exists memory_fts using fts5( @@ -170,20 +178,22 @@ def upsert_chunk( text: str, metadata: dict, created_at: str, + project_path: str | None = None, ) -> None: conn = self._connect() with conn: conn.execute( """ insert into memory_index - (id, source_path, title, memory_type, text, metadata_json, created_at) - values (?, ?, ?, ?, ?, ?, ?) + (id, source_path, title, memory_type, text, metadata_json, project_path, created_at) + values (?, ?, ?, ?, ?, ?, ?, ?) on conflict(id) do update set source_path=excluded.source_path, title=excluded.title, memory_type=excluded.memory_type, text=excluded.text, metadata_json=excluded.metadata_json, + project_path=excluded.project_path, created_at=excluded.created_at """, ( @@ -193,6 +203,7 @@ def upsert_chunk( memory_type, text, json.dumps(metadata, ensure_ascii=True), + project_path, created_at, ), ) @@ -205,31 +216,56 @@ def upsert_chunk( (chunk_id, title, text, source_path, memory_type), ) - def keyword_search(self, query: str, limit: int = 8) -> list[dict]: + def keyword_search( + self, + query: str, + limit: int = 8, + *, + project_path: str | None = None, + ) -> list[dict]: if not query.strip(): return [] + + project_clause = "" + params: list[object] = [query] + if project_path: + project_clause = "and (i.project_path = ? or i.project_path is null or i.project_path = '')" + params.append(project_path) + params.append(limit) + with self._connect() as conn: try: rows = conn.execute( - """ - select id, title, text, source_path, memory_type, + f""" + select f.id, f.title, f.text, f.source_path, f.memory_type, bm25(memory_fts) as score - from memory_fts + from memory_fts f + join memory_index i on i.id = f.id where memory_fts match ? + {project_clause} order by score limit ? """, - (query, limit), + params, ).fetchall() except sqlite3.OperationalError: + like_params: list[object] = [f"%{query}%", f"%{query}%"] + fallback_clause = "" + if project_path: + fallback_clause = ( + "and (project_path = ? or project_path is null or project_path = '')" + ) + like_params.append(project_path) + like_params.append(limit) rows = conn.execute( - """ + f""" select id, title, text, source_path, memory_type, 0.0 as score - from memory_fts - where title like ? or text like ? + from memory_index + where (title like ? or text like ?) + {fallback_clause} limit ? """, - (f"%{query}%", f"%{query}%", limit), + like_params, ).fetchall() return [dict(row) for row in rows] From a50c228e0f9f53f51ee2e92c240ddf72fdc87e9a Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 21:41:57 +0530 Subject: [PATCH 064/109] fix: keep temporal modifiers out of shared location values --- src/ultimate_memory/answer.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/ultimate_memory/answer.py b/src/ultimate_memory/answer.py index c28e06c..95b1668 100644 --- a/src/ultimate_memory/answer.py +++ b/src/ultimate_memory/answer.py @@ -604,13 +604,11 @@ def _compact_list_values(question: str, sentence: str) -> list[str]: for pattern in ( re.compile( r"\b(?:visited|went|traveled|travelled|vacationed|camped|stayed|lived)" - r"\s+(?:in|at|to)?\s*([A-Z][a-zA-Z]+(?:\s+[A-Z][a-zA-Z]+){0,2})", - re.I, + r"\s+(?:in|at|to)?\s*([A-Z][a-zA-Z]+(?:\s+[A-Z][a-zA-Z]+){0,2})" ), re.compile( r"\b(?:trip|vacation|camping)\s+(?:in|at|to)\s+" - r"([A-Z][a-zA-Z]+(?:\s+[A-Z][a-zA-Z]+){0,2})", - re.I, + r"([A-Z][a-zA-Z]+(?:\s+[A-Z][a-zA-Z]+){0,2})" ), ): values.extend(match.group(1).strip() for match in pattern.finditer(sentence)) From 0fd954bae22f14ab15b389693b52379168d57334 Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 21:44:39 +0530 Subject: [PATCH 065/109] ci: run one-sample independent MemEval on pull requests --- .github/workflows/memeval.yml | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/.github/workflows/memeval.yml b/.github/workflows/memeval.yml index 77ef7ff..d82c0c4 100644 --- a/.github/workflows/memeval.yml +++ b/.github/workflows/memeval.yml @@ -1,6 +1,7 @@ name: Independent MemEval on: + pull_request: workflow_dispatch: inputs: samples: @@ -58,11 +59,20 @@ jobs: working-directory: MemEval shell: bash run: | - EXTRA="" - if [ "${{ inputs.with_judge }}" != "true" ]; then - EXTRA="--skip-judge" + SAMPLES="1" + EXTRA="--skip-judge" + if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then + SAMPLES="${{ inputs.samples }}" + if [ "${{ inputs.with_judge }}" = "true" ]; then + EXTRA="" + fi fi - uv run python scripts/run_full_benchmark.py --systems ultimate_memory,propmem --num-samples "${{ inputs.samples }}" --llm-model "$LLM_MODEL" $EXTRA --output-dir benchmark-output + uv run python scripts/run_full_benchmark.py \ + --systems ultimate_memory,propmem \ + --num-samples "$SAMPLES" \ + --llm-model "$LLM_MODEL" \ + $EXTRA \ + --output-dir benchmark-output - name: Upload benchmark results if: always() From 6d429adb7def5aeac331be3f20010b4378a9b5d6 Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 21:46:06 +0530 Subject: [PATCH 066/109] fix: handle singular shared-location questions --- src/ultimate_memory/answer.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/ultimate_memory/answer.py b/src/ultimate_memory/answer.py index 95b1668..7eea5dd 100644 --- a/src/ultimate_memory/answer.py +++ b/src/ultimate_memory/answer.py @@ -600,7 +600,7 @@ def _compact_list_values(question: str, sentence: str) -> list[str]: ) # Travel / location histories. - if re.search(r"\b(?:cities|places|states|countries|where)\b", lower_q): + if re.search(r"\b(?:city|cities|place|places|state|states|country|countries|where)\b", lower_q): for pattern in ( re.compile( r"\b(?:visited|went|traveled|travelled|vacationed|camped|stayed|lived)" @@ -747,6 +747,8 @@ def _shared_entity_answer( "have", "with", "from", "that", "this", "they", "their", "your", "just", "really", "about", "been", "want", "like", "love", "great", "good", "also", "make", "help", "thing", "time", "need", "when", "what", "both", + "last", "next", "this", "year", "month", "week", "spring", "summer", + "autumn", "fall", "winter", "ago", "recent", "recently", } ranked = [ token From fc4882a0ada44884176087cafc8f2a05f8c2c614 Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 21:46:53 +0530 Subject: [PATCH 067/109] eval: strengthen fair MemEval extraction and reasoning --- integrations/memeval/ultimate_memory.py | 78 +++++++++++++++++++------ 1 file changed, 59 insertions(+), 19 deletions(-) diff --git a/integrations/memeval/ultimate_memory.py b/integrations/memeval/ultimate_memory.py index 9d04ba9..7904728 100644 --- a/integrations/memeval/ultimate_memory.py +++ b/integrations/memeval/ultimate_memory.py @@ -55,13 +55,15 @@ 2. Each memory must be atomic: one durable fact, preference, decision, procedure, event, relationship, status, ownership fact, activity, or plan. 3. Name the subject explicitly. Replace first-person pronouns with the speaker name. -4. Preserve exact names, places, titles, dates, quantities, and temporal qualifiers. -5. Keep separate facts separate; do not merge unrelated information. -6. Include the dialogue IDs that directly support each memory. -7. entities must contain the people/organizations/places directly involved. -8. memory_type must be one of: fact, preference, decision, procedure. -9. Do not extract greetings, compliments, filler, or questions that contain no answer. -10. Confidence is 0-1 and should reflect how explicitly the memory is stated. +4. Preserve exact names, places, titles, dates, quantities, and technical values verbatim. +5. Convert relative dates to absolute dates using the session date whenever possible. For example, resolve yesterday, last week, this month, three months ago, and next Tuesday into an explicit calendar date or month/year. +6. Keep separate facts separate; do not merge unrelated information. +7. Include the dialogue IDs that directly support each memory. +8. entities must contain the people/organizations/places directly involved. +9. memory_type must be one of: fact, preference, decision, procedure. +10. Extract EVERY durable fact, even minor ones; do not stop after the most salient facts. +11. Do not extract greetings, compliments, filler, or questions that contain no answer. +12. Confidence is 0-1 and should reflect how explicitly the memory is stated. Return JSON: {{"memories": [ @@ -96,7 +98,7 @@ 8. Answer in the same language as the question. Return JSON exactly as: -{{"answer": "direct compact answer"}} +{{"reasoning": "brief evidence-grounded reasoning", "answer": "direct compact answer"}} """ @@ -174,7 +176,7 @@ def _compile_session( ], response_format={"type": "json_object"}, temperature=0, - max_tokens=4096, + max_tokens=8192, ) content = response.choices[0].message.content or "{}" try: @@ -186,7 +188,7 @@ def _compile_session( return [] compiled: list[CompiledMemory] = [] - for raw in raw_memories[:60]: + for raw in raw_memories[:160]: if not isinstance(raw, dict): continue try: @@ -216,6 +218,19 @@ def _participants(conversation: dict) -> list[str]: return names[:8] + +def _is_inferential(question: str) -> bool: + lower = question.lower().strip() + return bool( + lower.startswith(("would ", "could ", "should ")) + or re.match(r"^what\s+(?:would|might|could)\b", lower) + or re.match(r"^(?:do you think|how would|is it likely)\b", lower) + or " likely " in lower + or " might " in lower + or (" prefer " in lower and "?" in question) + ) + + def run( conv: dict, llm_model: str, @@ -257,6 +272,7 @@ def openai_embed(texts: list[str]) -> list[list[float]]: key=lambda key: int(key.split("_")[1]), ) compiled_total = 0 + session_payloads: list[tuple[str, str, str]] = [] for session_key in session_keys: raw_session = _session_text(conversation, session_key) router.ingest_log( @@ -266,17 +282,34 @@ def openai_embed(texts: list[str]) -> list[list[float]]: project_path=project_path, tags=["memeval"], ) - session_date = str( conversation.get(f"{session_key}_date_time") or "" ) - compiled = _compile_session( - client, - llm_model, - participants=participants, - session_date=session_date, - session_text=raw_session, - ) + session_payloads.append((session_key, session_date, raw_session)) + + compiled_by_session: dict[str, list[CompiledMemory]] = {} + with ThreadPoolExecutor(max_workers=min(10, max(1, len(session_payloads)))) as pool: + futures = { + pool.submit( + _compile_session, + client, + llm_model, + participants=participants, + session_date=session_date, + session_text=raw_session, + ): session_key + for session_key, session_date, raw_session in session_payloads + } + for future in as_completed(futures): + session_key = futures[future] + try: + compiled_by_session[session_key] = future.result() + except Exception as exc: + print(f" Compile error ({session_key}): {exc}") + compiled_by_session[session_key] = [] + + for session_key, session_date, _ in session_payloads: + compiled = compiled_by_session.get(session_key, []) parsed_date = parse_loose_date(session_date) event_time = parsed_date.isoformat() if parsed_date else None outcome = router.ingest_compiled_memories( @@ -324,7 +357,14 @@ def answer_fn(question: str) -> str: "content": ANSWER_PROMPT.format( participants=", ".join(participants) or "unknown", evidence=evidence or "(no evidence retrieved)", - question=question, + question=( + question + + ( + "\n\nThis is an inference question. Draw the minimal logical conclusion supported by the named entity's own facts; do not return None merely because the conclusion is not stated verbatim." + if _is_inferential(question) + else "" + ) + ), ), } ], From a1c693804cc6ab561b88e4439fba380f9a56ee50 Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 21:47:51 +0530 Subject: [PATCH 068/109] fix: import MemEval adapter concurrency helpers --- integrations/memeval/ultimate_memory.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/integrations/memeval/ultimate_memory.py b/integrations/memeval/ultimate_memory.py index 7904728..bfca2c7 100644 --- a/integrations/memeval/ultimate_memory.py +++ b/integrations/memeval/ultimate_memory.py @@ -12,6 +12,8 @@ import json import os +import re +from concurrent.futures import ThreadPoolExecutor, as_completed from pathlib import Path from tempfile import TemporaryDirectory From c09693fefabfe6831be9b34281a83105d072a86e Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 21:48:30 +0530 Subject: [PATCH 069/109] perf: keep independent MemEval setup minimal --- .github/workflows/memeval.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/memeval.yml b/.github/workflows/memeval.yml index d82c0c4..181628a 100644 --- a/.github/workflows/memeval.yml +++ b/.github/workflows/memeval.yml @@ -51,7 +51,8 @@ jobs: - name: Install MemEval and Ultimate Memory working-directory: MemEval run: | - uv sync --all-extras + uv sync + uv pip install httpx uv pip install -e ../ultimate-memory cp ../ultimate-memory/integrations/memeval/ultimate_memory.py src/agents_memory/systems/ultimate_memory.py From 738efeeffd3597eaa1e0ff9df64a5cf89a6e3207 Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 21:49:39 +0530 Subject: [PATCH 070/109] feat: distinguish multi-evidence queries from true bridge reasoning --- src/ultimate_memory/planner.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/ultimate_memory/planner.py b/src/ultimate_memory/planner.py index d39df98..2c7097c 100644 --- a/src/ultimate_memory/planner.py +++ b/src/ultimate_memory/planner.py @@ -22,6 +22,8 @@ class QueryPlan(BaseModel): memory_types: list[str] = Field(default_factory=list) expansions: list[str] = Field(default_factory=list) signals: list[str] = Field(default_factory=list) + multi_evidence: bool = False + requires_bridge: bool = False _CAPITALIZED = re.compile(r"\b([A-Z][a-zA-Z0-9_.-]*(?:\s+[A-Z][a-zA-Z0-9_.-]*){0,3})\b") @@ -101,6 +103,20 @@ def _entities(question: str) -> list[str]: return found[:8] +def _requires_bridge(question: str) -> bool: + """Whether answering requires traversing an unnamed relationship chain.""" + lower = question.lower() + possessives = len(re.findall(r"\b[\w.-]+'s\b", question)) + relation_hits = sum( + 1 for word in _RELATION_WORDS + if re.search(rf"\b{re.escape(word)}\b", lower) + ) + chained_of = len( + re.findall(r"\bof\s+(?:the\s+)?(?:\w+\s+){0,2}(?:of|for|at)\b", lower) + ) + return possessives >= 2 or chained_of > 0 or (possessives >= 1 and relation_hits >= 1) + + def _hop_depth(question: str, entities: list[str] | None = None) -> int: lower = question.lower() entities = entities or _entities(question) @@ -181,6 +197,8 @@ def plan_query(question: str, *, as_of: str | None = None) -> QueryPlan: include_superseded = False entities = _entities(question) + multi_evidence = _is_collective_multi_hop(question, entities) + requires_bridge = _requires_bridge(question) depth = _hop_depth(question, entities) temporal_question = _is_temporal_question(question) if depth > 1: @@ -200,4 +218,6 @@ def plan_query(question: str, *, as_of: str | None = None) -> QueryPlan: memory_types=_memory_types(question), expansions=_expansions(question), signals=signals, + multi_evidence=multi_evidence, + requires_bridge=requires_bridge, ) From adebbc6835ada56b8c811fb4e643ae987dd54b0a Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 21:49:43 +0530 Subject: [PATCH 071/109] refactor: keep distributed retrieval entity-scoped and anchor expansions --- src/ultimate_memory/router.py | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/src/ultimate_memory/router.py b/src/ultimate_memory/router.py index 421440f..2300cc7 100644 --- a/src/ultimate_memory/router.py +++ b/src/ultimate_memory/router.py @@ -178,7 +178,13 @@ def answer( if token.lower() not in stop and len(token) > 2 ] dense_query = " ".join(dense_terms[:10]) or question - search_queries = list(dict.fromkeys([dense_query, question, *plan.expansions])) + expanded_queries = [ + f"{dense_query} {expansion}".strip() + for expansion in plan.expansions + ] + search_queries = list( + dict.fromkeys([dense_query, question, *expanded_queries]) + ) rich_contexts: list[dict] = [] search_result: dict = {"results": []} @@ -208,7 +214,7 @@ def answer( # Direct and temporal questions should stay attached to the named # subject. Multi-hop questions deliberately skip this hard gate because # they must traverse bridge entities. - if plan.kind != "multi_hop" and plan.entities: + if not plan.requires_bridge and plan.entities: rich_contexts = filter_entity_scoped_results( rich_contexts, plan.entities, @@ -275,11 +281,15 @@ def answer( initial_results = search_result.get("results") or [] seen_entity_keys = {normalize_entity(entity) for entity in plan.entities} - frontier = extract_hop_entities( - question, - initial_results, - exclude=None, - strict=plan.hop_depth > 1, + frontier = ( + extract_hop_entities( + question, + initial_results, + exclude=None, + strict=True, + ) + if plan.requires_bridge + else [] ) hop_entities: list[str] = list(frontier) hop_searches: list[dict] = [] @@ -343,7 +353,7 @@ def answer( depth += 1 rich_contexts = merge_contexts(rich_contexts, []) - if plan.kind == "multi_hop": + if plan.requires_bridge: rich_contexts = rank_evidence_chain(question, rich_contexts) else: rich_contexts.sort(key=lambda item: float(item.get("score") or 0.0), reverse=True) From 1ed3b23da7c1ac8ad44a237dacc163f4491ce3c9 Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 21:49:52 +0530 Subject: [PATCH 072/109] test: cover multi-evidence versus bridge planning --- tests/test_planner.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/test_planner.py b/tests/test_planner.py index 7953bf2..ac5d1b0 100644 --- a/tests/test_planner.py +++ b/tests/test_planner.py @@ -69,3 +69,22 @@ def test_duration_between_events_uses_multi_evidence_plan(): assert plan.kind == "multi_hop" assert plan.hop_depth >= 2 assert any("started" in expansion for expansion in plan.expansions) + + +def test_distributed_list_query_is_not_a_bridge_walk(): + plan = plan_query("What activities does Melanie partake in?") + assert plan.kind == "multi_hop" + assert plan.multi_evidence is True + assert plan.requires_bridge is False + + +def test_shared_named_entities_are_multi_evidence_not_unknown_bridge(): + plan = plan_query("Which city have both Jean and John visited?") + assert plan.multi_evidence is True + assert plan.requires_bridge is False + + +def test_possessive_relation_chain_requires_bridge(): + plan = plan_query("Where does Elena's sister's mentor work?") + assert plan.kind == "multi_hop" + assert plan.requires_bridge is True From 3fae5f8f397e64f0e08b579f12ed9bba3552c22e Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 21:50:14 +0530 Subject: [PATCH 073/109] feat: generalize distributed activity and event synthesis --- src/ultimate_memory/answer.py | 36 +++++++++++++++++++++++++++++++---- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/src/ultimate_memory/answer.py b/src/ultimate_memory/answer.py index 7eea5dd..a74ffce 100644 --- a/src/ultimate_memory/answer.py +++ b/src/ultimate_memory/answer.py @@ -574,10 +574,11 @@ def _is_list_question(question: str) -> bool: lower = question.lower() return bool( re.search( - r"\bboth\b|\ball\b|" - r"\bwhich\s+(?:cities|places|countries|states|books|games|activities|items|things|ways|types|kinds)\b|" - r"\bwhat\s+(?:cities|places|countries|states|books|games|activities|items|things|ways|types|kinds)\b|" - r"\bwhat\s+does\s+.+?\s+(?:offer|provide|include)\b", + r"\bboth\b|\ball\b|\bin\s+what\s+ways\b|" + r"\bwhich\s+(?!(?:is|was|does|did|has|have)\b)[a-z]+s\b|" + r"\bwhat\s+(?!(?:is|was|does|did|has|have)\b)[a-z]+s\b|" + r"\bwhat\s+does\s+.+?\s+(?:offer|provide|include|do\s+to)\b|" + r"\bwhere\s+has\s+.+?\s+(?:camped|traveled|travelled|visited|stayed|lived)\b", lower, ) ) @@ -650,6 +651,33 @@ def _compact_list_values(question: str, sentence: str) -> list[str]: if 2 <= len(part.strip()) <= 70 ) + # Generic activity / participation values. + if re.search( + r"\b(?:activities?|hobbies?|partake|destress|de-stress|in what ways|participat|events?)\b", + lower_q, + ): + for pattern in ( + re.compile( + r"\b(?:been|started|kept|enjoys?|likes?|loves?|go|goes|went)\s+" + r"(?:to\s+)?([a-z][a-z-]+ing)\b", + re.I, + ), + re.compile( + r"\bsigned\s+up\s+for\s+(?:a\s+|an\s+)?" + r"([a-z][a-z-]+)(?:\s+class|\s+course|\s+workshop)\b", + re.I, + ), + re.compile( + r"\b(?:attended|joined|participated\s+in|went\s+to)\s+" + r"(?:a\s+|an\s+|the\s+)?([^,.!?]{2,70})", + re.I, + ), + ): + for match in pattern.finditer(sentence): + value = match.group(1).strip(" ,.;:-") + if 2 <= len(value) <= 70: + values.append(value) + # De-duplicate and discard obvious dialogue scaffolding. cleaned: list[str] = [] seen: set[str] = set() From 7b37d80d7ac027a1f4688ddfbacdff786d286033 Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 21:50:26 +0530 Subject: [PATCH 074/109] test: cover generic activity and event aggregation --- tests/test_answer.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/test_answer.py b/tests/test_answer.py index ba4bba3..479d445 100644 --- a/tests/test_answer.py +++ b/tests/test_answer.py @@ -379,3 +379,23 @@ def test_favorite_value_is_extracted_compactly(): ] answer = synthesize_answer("What is Gina's favorite style of dance?", contexts) assert answer.lower() == "contemporary" + + +def test_generic_activity_list_synthesis(): + contexts = [ + {"text": "Melanie: I've been running farther to de-stress.", "score": 0.9}, + {"text": "Melanie: I signed up for a pottery class because it feels therapeutic.", "score": 0.8}, + ] + answer = synthesize_answer("What does Melanie do to destress?", contexts) + assert "running" in answer.lower() + assert "pottery" in answer.lower() + + +def test_generic_event_participation_synthesis(): + contexts = [ + {"text": "Caroline: I attended a pride parade downtown.", "score": 0.9}, + {"text": "Caroline: I went to a support group last week.", "score": 0.8}, + ] + answer = synthesize_answer("What events has Caroline participated in?", contexts) + assert "pride parade" in answer.lower() + assert "support group" in answer.lower() From dcbb73cb8b3b6da7945490c9bc50b0d78addae50 Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 21:52:00 +0530 Subject: [PATCH 075/109] feat: route favorite questions through preference memory --- src/ultimate_memory/planner.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/ultimate_memory/planner.py b/src/ultimate_memory/planner.py index 2c7097c..a7d13dd 100644 --- a/src/ultimate_memory/planner.py +++ b/src/ultimate_memory/planner.py @@ -136,7 +136,7 @@ def _hop_depth(question: str, entities: list[str] | None = None) -> int: def _memory_types(question: str) -> list[str]: lower = question.lower() types: list[str] = [] - if re.search(r"\bprefer|preference|always|never|style|likes?\b", lower): + if re.search(r"\bprefer|preference|favorite|favourite|always|never|style|likes?\b", lower): types.append("preference") if re.search(r"\bdecision|decide|decided|chose|chosen|why did we|why was\b", lower): types.append("decision") @@ -155,8 +155,8 @@ def _expansions(question: str) -> list[str]: expansions.append("work job employer role") if re.search(r"\blive|lives|location|based|where\b", lower): expansions.append("location lives based moved") - if re.search(r"\bprefer|preference|likes?\b", lower): - expansions.append("preference prefer likes") + if re.search(r"\bprefer|preference|favorite|favourite|likes?\b", lower): + expansions.append("preference prefer favorite favourite likes") if re.search(r"\bdecision|decide|chose|chosen|why\b", lower): expansions.append("decision chose reason rationale") if re.search(r"\bhow do|how to|steps?|procedure|process\b", lower): From b31b7b2e337506562fbe07edc0ccda809d1bcdaa Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 21:52:09 +0530 Subject: [PATCH 076/109] test: cover favorite preference routing --- tests/test_planner.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_planner.py b/tests/test_planner.py index ac5d1b0..f75cef1 100644 --- a/tests/test_planner.py +++ b/tests/test_planner.py @@ -88,3 +88,9 @@ def test_possessive_relation_chain_requires_bridge(): plan = plan_query("Where does Elena's sister's mentor work?") assert plan.kind == "multi_hop" assert plan.requires_bridge is True + + +def test_favorite_question_prefers_preference_memory(): + plan = plan_query("What is Gina's favorite style of dance?") + assert "preference" in plan.memory_types + assert any("favorite" in expansion for expansion in plan.expansions) From 86a1bd3894842e4f739af5cfe7238f776caf2a51 Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 21:53:01 +0530 Subject: [PATCH 077/109] fix: narrow identity intent and prioritize duration answers --- src/ultimate_memory/answer.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/ultimate_memory/answer.py b/src/ultimate_memory/answer.py index a74ffce..46c16fb 100644 --- a/src/ultimate_memory/answer.py +++ b/src/ultimate_memory/answer.py @@ -174,7 +174,10 @@ ) _DIA_TURN_RE = re.compile(r"^\[D\d+:\d+\]") _IDENTITY_QUESTION_RE = re.compile( - r"\bidentity\b|\b(?:gender|transgender)\b|what is .+'s (?:identity|gender)", + r"\bwhat\s+is\s+.+?'s\s+(?:identity|gender)\b|" + r"\bwhat\s+(?:identity|gender)\s+(?:does|is|was)\b|" + r"\b(?:identify|identifies)\s+as\b|" + r"\bis\s+.+?\s+(?:transgender|nonbinary|non-binary)\b", re.I, ) _IDENTITY_PHRASE_RE = re.compile( @@ -1331,6 +1334,13 @@ def synthesize_answer( spans = spans or _extract_date_spans(sentence) spans = _prefer_absolute_date_spans(spans) for date in spans: + # Duration questions should answer with a duration, not the + # session timestamp that happens to anchor the evidence. + duration_bonus = ( + 1.25 + if duration_question and _extract_duration_spans(date) + else 0.0 + ) # Penalize dates that are just session stamps without topical words. topical = overlap + ( 0.4 if any(w in sentence.lower() for w in question_words) else 0.0 @@ -1341,11 +1351,12 @@ def synthesize_answer( # when overlap is otherwise similar. dated.append( ( - meta + topical + abs_bonus + min(len(date), 40) * 0.015, + meta + topical + abs_bonus + duration_bonus + + min(len(date), 40) * 0.015, date, ) ) - if item.session_date: + if item.session_date and not duration_question: topical_sentences = [ sentence for sentence in _split_sentences(item.text) if _overlap_score(question_words, sentence, entities) >= 0.22 From 3e43f31a90c2bdf0b221a773475fd94d998e0800 Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 21:53:05 +0530 Subject: [PATCH 078/109] test: protect identity intent and duration precedence --- tests/test_answer.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/test_answer.py b/tests/test_answer.py index 479d445..45ae9ec 100644 --- a/tests/test_answer.py +++ b/tests/test_answer.py @@ -399,3 +399,23 @@ def test_generic_event_participation_synthesis(): answer = synthesize_answer("What events has Caroline participated in?", contexts) assert "pride parade" in answer.lower() assert "support group" in answer.lower() + + +def test_transgender_topic_does_not_imply_identity_question(): + contexts = [ + {"text": "Caroline is a transgender woman.", "score": 0.5}, + {"text": "Caroline: I'm going to a transgender conference in July 2023.", "score": 0.9}, + ] + answer = synthesize_answer("When is Caroline going to the transgender conference?", contexts) + assert "July" in answer and "2023" in answer + + +def test_duration_beats_session_date_for_how_long_question(): + contexts = [ + { + "text": "---\nsession_date: 13 September 2023\n---\nCaroline: I've had this group of friends for 4 years.", + "score": 0.9, + } + ] + answer = synthesize_answer("How long has Caroline had this group of friends for?", contexts) + assert answer.lower() == "4 years" From 7455df2b1d8e6a59986a07dd568a6f5cc26a3ec9 Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 21:53:20 +0530 Subject: [PATCH 079/109] feat: enforce duration quantity consistency in reranking --- src/ultimate_memory/ranking.py | 36 ++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/src/ultimate_memory/ranking.py b/src/ultimate_memory/ranking.py index 90969db..6b7d180 100644 --- a/src/ultimate_memory/ranking.py +++ b/src/ultimate_memory/ranking.py @@ -54,6 +54,29 @@ def filter_entity_scoped_results( return matched if len(matched) >= min_matches else list(results) + +_NUMBER_WORDS = { + "one": "1", "two": "2", "three": "3", "four": "4", "five": "5", + "six": "6", "seven": "7", "eight": "8", "nine": "9", "ten": "10", + "eleven": "11", "twelve": "12", +} + + +def _duration_quantities(text: str) -> set[tuple[str, str]]: + out: set[tuple[str, str]] = set() + pattern = re.compile( + r"\b(\d+|one|two|three|four|five|six|seven|eight|nine|ten|eleven|twelve)" + r"\s+(years?|months?|weeks?|days?|hours?)\b", + re.I, + ) + for match in pattern.finditer(text): + raw = match.group(1).lower() + number = _NUMBER_WORDS.get(raw, raw) + unit = match.group(2).lower().rstrip("s") + out.add((number, unit)) + return out + + def rerank_candidates( query: str, results: list[SearchResult], @@ -61,6 +84,7 @@ def rerank_candidates( ) -> list[SearchResult]: q_tokens = _tokens(query) wanted_types = set(plan.memory_types) + query_quantities = _duration_quantities(query) for result in results: score = float(result.score) @@ -74,6 +98,18 @@ def rerank_candidates( entity_hits = sum(1 for entity in plan.entities if entity.lower() in text_lower) score += min(0.18, 0.07 * entity_hits) + if query_quantities: + result_quantities = _duration_quantities(result.text) + if result_quantities: + if query_quantities & result_quantities: + score += 0.18 + elif any( + q_unit == r_unit + for _, q_unit in query_quantities + for _, r_unit in result_quantities + ): + score -= 0.18 + if result.memory_type in wanted_types: score += 0.08 From 5564e3258d98be271b3d9db1b9a68d1210994432 Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 21:53:32 +0530 Subject: [PATCH 080/109] test: cover quantity-consistent reranking --- tests/test_ranking.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 tests/test_ranking.py diff --git a/tests/test_ranking.py b/tests/test_ranking.py new file mode 100644 index 0000000..09b77ef --- /dev/null +++ b/tests/test_ranking.py @@ -0,0 +1,28 @@ +from ultimate_memory.models import SearchResult +from ultimate_memory.planner import plan_query +from ultimate_memory.ranking import rerank_candidates + + +def result(text: str, score: float = 0.5) -> SearchResult: + return SearchResult( + id=text, + text=text, + title=text, + source_path=text, + memory_type="fact", + score=score, + provenance={}, + ) + + +def test_matching_duration_quantity_beats_conflicting_duration(): + query = "Where did Caroline move from 4 years ago?" + ranked = rerank_candidates( + query, + [ + result("Caroline moved from Sweden four years ago."), + result("Caroline started guitar five years ago."), + ], + plan_query(query), + ) + assert "Sweden" in ranked[0].text From c771831deb4584a2d2bc4b55e1fff96c932e75b5 Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 21:54:03 +0530 Subject: [PATCH 081/109] ci: skip independent MemEval cleanly when API secret is absent --- .github/workflows/memeval.yml | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/.github/workflows/memeval.yml b/.github/workflows/memeval.yml index 181628a..1e52d04 100644 --- a/.github/workflows/memeval.yml +++ b/.github/workflows/memeval.yml @@ -27,28 +27,35 @@ jobs: LLM_MODEL: gpt-4.1-mini EMBEDDING_MODEL: text-embedding-3-small steps: - - name: Require API key + - name: Check API key + id: api_key run: | if [ -z "$OPENAI_API_KEY" ]; then - echo "::error::OPENAI_API_KEY Actions secret is not configured." - exit 1 + echo "::notice::OPENAI_API_KEY Actions secret is not configured; independent MemEval is skipped." + echo "available=false" >> "$GITHUB_OUTPUT" + else + echo "available=true" >> "$GITHUB_OUTPUT" fi - name: Checkout Ultimate Memory + if: steps.api_key.outputs.available == 'true' uses: actions/checkout@v4 with: path: ultimate-memory - name: Checkout Prosus MemEval + if: steps.api_key.outputs.available == 'true' uses: actions/checkout@v4 with: repository: ProsusAI/MemEval path: MemEval - name: Install uv + if: steps.api_key.outputs.available == 'true' uses: astral-sh/setup-uv@v3 - name: Install MemEval and Ultimate Memory + if: steps.api_key.outputs.available == 'true' working-directory: MemEval run: | uv sync @@ -57,6 +64,7 @@ jobs: cp ../ultimate-memory/integrations/memeval/ultimate_memory.py src/agents_memory/systems/ultimate_memory.py - name: Run fair head-to-head + if: steps.api_key.outputs.available == 'true' working-directory: MemEval shell: bash run: | @@ -76,7 +84,7 @@ jobs: --output-dir benchmark-output - name: Upload benchmark results - if: always() + if: always() && steps.api_key.outputs.available == 'true' uses: actions/upload-artifact@v4 with: name: independent-memeval-results From 7aa1b30154a0112bb3c3f0a04685e7291b863571 Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 21:54:39 +0530 Subject: [PATCH 082/109] feat: model recency queries as current temporal intent --- src/ultimate_memory/planner.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/ultimate_memory/planner.py b/src/ultimate_memory/planner.py index a7d13dd..c16b139 100644 --- a/src/ultimate_memory/planner.py +++ b/src/ultimate_memory/planner.py @@ -39,7 +39,8 @@ class QueryPlan(BaseModel): r"\bhow\s+many\s+(?:years?|months?|weeks?|days?|hours?)\b|" r"\b(?:years?|months?|weeks?|days?|hours?)\s+ago\b|" r"\b(?:since|until|during)\b|" - r"\b(?:before|after|earlier|later|previously|formerly|prior)\b" + r"\b(?:before|after|earlier|later|previously|formerly|prior)\b|" + r"\b(?:recent|recently|latest|newest|most\s+recent)\b" ) _COLLECTIVE_RE = re.compile( @@ -188,7 +189,10 @@ def plan_query(question: str, *, as_of: str | None = None) -> QueryPlan: temporal_mode = "historical" include_superseded = True signals.append("historical_language") - elif re.search(r"\b(now|current|currently|today|latest)\b", lower): + elif re.search( + r"\b(now|current|currently|today|latest|recent|recently|newest|most\s+recent)\b", + lower, + ): temporal_mode = "current" include_superseded = False signals.append("current_language") @@ -204,7 +208,7 @@ def plan_query(question: str, *, as_of: str | None = None) -> QueryPlan: if depth > 1: kind = "multi_hop" signals.append(f"relation_chain_depth_{depth}") - elif temporal_mode in {"historical", "as_of"} or temporal_question: + elif temporal_mode != "unspecified" or temporal_question: kind = "temporal" else: kind = "single_hop" From 08400acae2c1ea0d3e90a108d49dcbea0862a299 Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 21:54:43 +0530 Subject: [PATCH 083/109] feat: rerank recent queries by event time --- src/ultimate_memory/ranking.py | 35 ++++++++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/src/ultimate_memory/ranking.py b/src/ultimate_memory/ranking.py index 6b7d180..152be51 100644 --- a/src/ultimate_memory/ranking.py +++ b/src/ultimate_memory/ranking.py @@ -2,6 +2,7 @@ from __future__ import annotations import re +from datetime import datetime from .models import SearchResult from .planner import QueryPlan @@ -77,6 +78,22 @@ def _duration_quantities(text: str) -> set[tuple[str, str]]: return out + +def _event_timestamp(result: SearchResult) -> float | None: + provenance = result.provenance or {} + raw = provenance.get("event_time") or provenance.get("created_at") + if not raw: + metadata = provenance.get("metadata") + if isinstance(metadata, dict): + raw = metadata.get("event_time") or metadata.get("created_at") + if not raw: + return None + try: + return datetime.fromisoformat(str(raw).replace("Z", "+00:00")).timestamp() + except (ValueError, TypeError): + return None + + def rerank_candidates( query: str, results: list[SearchResult], @@ -85,6 +102,9 @@ def rerank_candidates( q_tokens = _tokens(query) wanted_types = set(plan.memory_types) query_quantities = _duration_quantities(query) + timestamps = [ts for result in results if (ts := _event_timestamp(result)) is not None] + newest_timestamp = max(timestamps) if timestamps else None + oldest_timestamp = min(timestamps) if timestamps else None for result in results: score = float(result.score) @@ -131,8 +151,19 @@ def rerank_candidates( if "auto-extracted from" in text_lower: score -= 0.08 - if plan.temporal_mode == "current" and provenance.get("valid_until"): - score -= 0.35 + if plan.temporal_mode == "current": + if provenance.get("valid_until"): + score -= 0.35 + event_ts = _event_timestamp(result) + if ( + event_ts is not None + and newest_timestamp is not None + and oldest_timestamp is not None + and newest_timestamp > oldest_timestamp + ): + recency = (event_ts - oldest_timestamp) / (newest_timestamp - oldest_timestamp) + score += 0.22 * recency + provenance["recency_score"] = round(recency, 6) elif plan.temporal_mode in {"historical", "as_of"} and provenance.get("valid_until"): score += 0.08 From 8c274a3021f6c65a2a017ba3439b324db77119bc Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 21:54:53 +0530 Subject: [PATCH 084/109] test: cover recency temporal planning --- tests/test_planner.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_planner.py b/tests/test_planner.py index f75cef1..7a899c8 100644 --- a/tests/test_planner.py +++ b/tests/test_planner.py @@ -94,3 +94,9 @@ def test_favorite_question_prefers_preference_memory(): plan = plan_query("What is Gina's favorite style of dance?") assert "preference" in plan.memory_types assert any("favorite" in expansion for expansion in plan.expansions) + + +def test_recent_question_is_current_temporal(): + plan = plan_query("What did Melanie paint recently?") + assert plan.kind == "temporal" + assert plan.temporal_mode == "current" From 8468af353814f586cbcbead2d8adcd846100b015 Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 21:54:57 +0530 Subject: [PATCH 085/109] test: cover recency event-time reranking --- tests/test_ranking.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/test_ranking.py b/tests/test_ranking.py index 09b77ef..bd87d26 100644 --- a/tests/test_ranking.py +++ b/tests/test_ranking.py @@ -26,3 +26,14 @@ def test_matching_duration_quantity_beats_conflicting_duration(): plan_query(query), ) assert "Sweden" in ranked[0].text + + +def test_recent_query_prefers_newer_event_time(): + query = "What did Melanie paint recently?" + plan = plan_query(query) + old = result("Melanie painted horses.") + old.provenance["event_time"] = "2023-01-01T00:00:00+00:00" + new = result("Melanie painted a sunset.") + new.provenance["event_time"] = "2023-10-01T00:00:00+00:00" + ranked = rerank_candidates(query, [old, new], plan) + assert "sunset" in ranked[0].text From 1d03427f89ced3accbef31636a4ba1e64cbd93e0 Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 21:55:20 +0530 Subject: [PATCH 086/109] feat: add generic semantic-domain query expansions --- src/ultimate_memory/planner.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/ultimate_memory/planner.py b/src/ultimate_memory/planner.py index c16b139..8c04fe5 100644 --- a/src/ultimate_memory/planner.py +++ b/src/ultimate_memory/planner.py @@ -168,6 +168,18 @@ def _expansions(question: str) -> list[str]: expansions.append("visited travel trip city place") if re.search(r"\boffer|offers|offering|provide|provides|services\b", lower): expansions.append("offer provides services classes workshops training") + if re.search(r"\brelationship|dating|married|single|partner\b", lower): + expansions.append("relationship status dating married single partner") + if re.search(r"\bidentity|gender|transgender|nonbinary|non-binary\b", lower): + expansions.append("identity gender transgender nonbinary") + if re.search(r"\bcareer|profession|education|educaton|field|study|degree\b", lower): + expansions.append("career education study degree certification profession") + if re.search(r"\bactivities?|hobbies?|destress|de-stress|relax|leisure\b", lower): + expansions.append("activity hobby recreation leisure destress relax") + if re.search(r"\bevents?|participat|community|involvement|attend|joined?\b", lower): + expansions.append("event attended participated joined involvement community") + if re.search(r"\bpaint|painting|artwork|art\b", lower): + expansions.append("painting artwork canvas subject landscape") if _is_temporal_question(question): expansions.append("date year month day when duration time") if re.search(r"\bhow\s+long\b|\bduration\b|\belapsed\b", lower): From a03e8cb977034372a6dda612ec2e88aa62519a43 Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 21:56:09 +0530 Subject: [PATCH 087/109] fix: extract lowercase locations without temporal suffix leakage --- src/ultimate_memory/answer.py | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/src/ultimate_memory/answer.py b/src/ultimate_memory/answer.py index 46c16fb..3074c94 100644 --- a/src/ultimate_memory/answer.py +++ b/src/ultimate_memory/answer.py @@ -587,6 +587,21 @@ def _is_list_question(question: str) -> bool: ) +def _clean_place_value(value: str) -> str: + value = value.strip(" ,.;:-") + value = re.sub(r"^(?:the)\s+", "", value, flags=re.I) + value = re.sub( + r"\s+(?:last|next|this)\s+" + r"(?:spring|summer|autumn|fall|winter|year|month|week|weekend|" + r"monday|tuesday|wednesday|thursday|friday|saturday|sunday)\b.*$", + "", + value, + flags=re.I, + ) + value = re.sub(r"\s+(?:a\s+few|several|\d+)\s+(?:days?|weeks?|months?|years?)\s+ago\b.*$", "", value, flags=re.I) + return value.strip(" ,.;:-") + + def _compact_list_values(question: str, sentence: str) -> list[str]: """Extract compact candidate values from a relevant sentence. @@ -608,14 +623,21 @@ def _compact_list_values(question: str, sentence: str) -> list[str]: for pattern in ( re.compile( r"\b(?:visited|went|traveled|travelled|vacationed|camped|stayed|lived)" - r"\s+(?:in|at|to)?\s*([A-Z][a-zA-Z]+(?:\s+[A-Z][a-zA-Z]+){0,2})" + r"\s+(?:in|at|to|near|on)?\s*(?:the\s+)?" + r"([a-z][a-z-]+(?:\s+[a-z][a-z-]+){0,2})", + re.I, ), re.compile( - r"\b(?:trip|vacation|camping)\s+(?:in|at|to)\s+" - r"([A-Z][a-zA-Z]+(?:\s+[A-Z][a-zA-Z]+){0,2})" + r"\b(?:trip|vacation|camping)\s+(?:in|at|to|near|on)\s+" + r"(?:the\s+)?([a-z][a-z-]+(?:\s+[a-z][a-z-]+){0,2})", + re.I, ), ): - values.extend(match.group(1).strip() for match in pattern.finditer(sentence)) + values.extend( + place + for match in pattern.finditer(sentence) + if (place := _clean_place_value(match.group(1))) + ) # Product/service/class offerings: capture coordinated objects after the verb. if re.search(r"\b(?:offer|provide|include|services?|classes?|training|workshops?)\b", lower_q): From b509ab9a776dde7b7155fb362b1f7f02b6903284 Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 21:56:18 +0530 Subject: [PATCH 088/109] test: cover lowercase place extraction and time cleanup --- tests/test_answer.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/test_answer.py b/tests/test_answer.py index 45ae9ec..f0962ae 100644 --- a/tests/test_answer.py +++ b/tests/test_answer.py @@ -419,3 +419,24 @@ def test_duration_beats_session_date_for_how_long_question(): ] answer = synthesize_answer("How long has Caroline had this group of friends for?", contexts) assert answer.lower() == "4 years" + + +def test_location_list_handles_lowercase_places_and_strips_time_modifiers(): + contexts = [ + {"text": "Melanie: We camped at the beach last summer.", "score": 0.9}, + {"text": "Melanie: We camped in the forest this spring.", "score": 0.8}, + ] + answer = synthesize_answer("Where has Melanie camped?", contexts) + assert "beach" in answer.lower() + assert "forest" in answer.lower() + assert "last summer" not in answer.lower() + assert "this spring" not in answer.lower() + + +def test_shared_city_cleanup_keeps_city_not_time_modifier(): + contexts = [ + {"text": "Jean: I visited Rome last spring.", "score": 0.9}, + {"text": "John: I traveled to Rome last year.", "score": 0.9}, + ] + answer = synthesize_answer("Which city have both Jean and John visited?", contexts) + assert answer.lower() == "rome" From 0d0a93f31f348662b1aac7ffb76d52307f6210b3 Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 21:56:55 +0530 Subject: [PATCH 089/109] feat: detect participation and group-preference queries as multi-evidence --- src/ultimate_memory/planner.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/ultimate_memory/planner.py b/src/ultimate_memory/planner.py index 8c04fe5..d49b9d2 100644 --- a/src/ultimate_memory/planner.py +++ b/src/ultimate_memory/planner.py @@ -63,6 +63,9 @@ class QueryPlan(BaseModel): r"\bwhat\s+does\s+.+?\s+(?:offer|provide|include|do\s+to)\b|" r"\b(?:all|multiple|several)\s+(?:[a-z]+s|cities|places|books|activities|items|things|ways|types)\b|" r"\bwhere\s+has\s+.+?\s+(?:camped|traveled|travelled|visited|stayed|lived)\b|" + r"\bin\s+what\s+ways\b|" + r"\bhow\s+(?:does|is|has)\s+.+?\s+(?:participat|involv|contribut)\w*\b|" + r"\bwhat\s+do\s+.+?'s\s+[a-z]+s\s+(?:like|enjoy|prefer|do)\b|" r"\bwhat\s+.+?\s+has\s+.+?\s+(?:done|read|visited|attended|participated|painted|tried|used)\b", re.I, ) From b8558c71d7ba397f430dcdd33d6cd0c121feae20 Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 21:56:59 +0530 Subject: [PATCH 090/109] feat: aggregate participation and group-preference answers --- src/ultimate_memory/answer.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/ultimate_memory/answer.py b/src/ultimate_memory/answer.py index 3074c94..776e2e4 100644 --- a/src/ultimate_memory/answer.py +++ b/src/ultimate_memory/answer.py @@ -581,6 +581,8 @@ def _is_list_question(question: str) -> bool: r"\bwhich\s+(?!(?:is|was|does|did|has|have)\b)[a-z]+s\b|" r"\bwhat\s+(?!(?:is|was|does|did|has|have)\b)[a-z]+s\b|" r"\bwhat\s+does\s+.+?\s+(?:offer|provide|include|do\s+to)\b|" + r"\bin\s+what\s+ways\b|" + r"\bwhat\s+do\s+.+?'s\s+[a-z]+s\s+(?:like|enjoy|prefer|do)\b|" r"\bwhere\s+has\s+.+?\s+(?:camped|traveled|travelled|visited|stayed|lived)\b", lower, ) From bf35d7a09c610622a085f987a157e5813ce839c5 Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 21:57:19 +0530 Subject: [PATCH 091/109] test: cover participation and group preference planning --- tests/test_planner.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/test_planner.py b/tests/test_planner.py index 7a899c8..69b8ca0 100644 --- a/tests/test_planner.py +++ b/tests/test_planner.py @@ -100,3 +100,15 @@ def test_recent_question_is_current_temporal(): plan = plan_query("What did Melanie paint recently?") assert plan.kind == "temporal" assert plan.temporal_mode == "current" + + +def test_in_what_ways_query_is_multi_evidence_not_bridge(): + plan = plan_query("In what ways is Caroline participating in the LGBTQ community?") + assert plan.multi_evidence is True + assert plan.requires_bridge is False + + +def test_group_preference_query_is_multi_evidence(): + plan = plan_query("What do Melanie's kids like?") + assert plan.multi_evidence is True + assert plan.requires_bridge is False From 4d711cfde3ae981fcf3849d4fdc73a6145182a26 Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 21:57:34 +0530 Subject: [PATCH 092/109] fix: align hybrid reader with recency and multi-evidence planning --- src/ultimate_memory/reader.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ultimate_memory/reader.py b/src/ultimate_memory/reader.py index 90dc902..952973b 100644 --- a/src/ultimate_memory/reader.py +++ b/src/ultimate_memory/reader.py @@ -24,7 +24,8 @@ ) _TEMPORAL = re.compile( r"\bwhen\b|\bwhat\s+(?:date|year|month|day)\b|" - r"\bhow\s+long\b|\bhow\s+many\s+(?:years?|months?|weeks?|days?)\b", + r"\bhow\s+long\b|\bhow\s+many\s+(?:years?|months?|weeks?|days?)\b|" + r"\b(?:recent|recently|latest|newest|most\s+recent)\b", re.I, ) _DISTRIBUTED = re.compile( From e9990f95731d4651b67942c624c8f7fa2a647100 Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 21:58:38 +0530 Subject: [PATCH 093/109] feat: balance evidence across multiple named entities --- src/ultimate_memory/ranking.py | 57 +++++++++++++++++++++++++++------- 1 file changed, 45 insertions(+), 12 deletions(-) diff --git a/src/ultimate_memory/ranking.py b/src/ultimate_memory/ranking.py index 152be51..0d96ec2 100644 --- a/src/ultimate_memory/ranking.py +++ b/src/ultimate_memory/ranking.py @@ -22,17 +22,19 @@ def filter_entity_scoped_results( *, min_matches: int = 2, ) -> list[dict]: - """Prefer evidence explicitly attached to the entities named in the query. + """Prefer and balance evidence attached to entities named in the query. - This is intentionally a post-retrieval gate rather than a storage filter: - global memories remain available, and multi-hop callers can simply skip this - helper when bridge-entity traversal is required. + For one entity this is a hard topical gate when enough evidence exists. + For multiple named entities, matched evidence is interleaved so one person's + larger memory history cannot crowd the other person out of the context packet. """ entity_keys = [entity.strip().casefold() for entity in entities if entity.strip()] if not entity_keys or not results: return list(results) matched: list[dict] = [] + buckets: dict[str, list[dict]] = {key: [] for key in entity_keys} + for item in results: text = str(item.get("text") or "").casefold() provenance = item.get("provenance") or {} @@ -43,16 +45,47 @@ def filter_entity_scoped_results( } speaker = str(provenance.get("speaker") or "").strip().casefold() - if any( - key in text - or key == speaker - or key in prov_entities + item_entities = [ + key for key in entity_keys - ): - matched.append(item) + if key in text or key == speaker or key in prov_entities + ] + if not item_entities: + continue + matched.append(item) + for key in item_entities: + buckets[key].append(item) + + if len(matched) < min_matches: + return list(results) - # Never collapse a query to an unusably tiny evidence pool. - return matched if len(matched) >= min_matches else list(results) + if len(entity_keys) == 1 or not all(buckets[key] for key in entity_keys): + return matched + + # Round-robin the per-entity rankings, then append any remaining matched + # evidence in its original order. Items mentioning both entities are deduped. + balanced: list[dict] = [] + seen: set[str] = set() + max_len = max(len(bucket) for bucket in buckets.values()) + for index in range(max_len): + for key in entity_keys: + bucket = buckets[key] + if index >= len(bucket): + continue + item = bucket[index] + item_key = str(item.get("id") or item.get("source_path") or id(item)) + if item_key in seen: + continue + seen.add(item_key) + balanced.append(item) + + for item in matched: + item_key = str(item.get("id") or item.get("source_path") or id(item)) + if item_key in seen: + continue + seen.add(item_key) + balanced.append(item) + return balanced From e1baa9cd25ca562e3b40fc532f86d1d72182daab Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 21:58:49 +0530 Subject: [PATCH 094/109] test: require balanced multi-entity evidence --- tests/test_ranking.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/test_ranking.py b/tests/test_ranking.py index bd87d26..2e88696 100644 --- a/tests/test_ranking.py +++ b/tests/test_ranking.py @@ -1,6 +1,6 @@ from ultimate_memory.models import SearchResult from ultimate_memory.planner import plan_query -from ultimate_memory.ranking import rerank_candidates +from ultimate_memory.ranking import filter_entity_scoped_results, rerank_candidates def result(text: str, score: float = 0.5) -> SearchResult: @@ -37,3 +37,15 @@ def test_recent_query_prefers_newer_event_time(): new.provenance["event_time"] = "2023-10-01T00:00:00+00:00" ranked = rerank_candidates(query, [old, new], plan) assert "sunset" in ranked[0].text + + +def test_multi_entity_filter_balances_named_people(): + results = [ + {"id": "a1", "text": "Jean visited Paris.", "provenance": {}}, + {"id": "a2", "text": "Jean visited Rome.", "provenance": {}}, + {"id": "a3", "text": "Jean likes museums.", "provenance": {}}, + {"id": "b1", "text": "John visited Rome.", "provenance": {}}, + ] + balanced = filter_entity_scoped_results(results, ["Jean", "John"]) + assert balanced[0]["id"] == "a1" + assert balanced[1]["id"] == "b1" From afee715d2eb7b581ca9b66daebb20e54dd492708 Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 21:59:20 +0530 Subject: [PATCH 095/109] feat: prioritize named speakers in conversational entity scope --- src/ultimate_memory/ranking.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/src/ultimate_memory/ranking.py b/src/ultimate_memory/ranking.py index 0d96ec2..6c7335f 100644 --- a/src/ultimate_memory/ranking.py +++ b/src/ultimate_memory/ranking.py @@ -32,8 +32,20 @@ def filter_entity_scoped_results( if not entity_keys or not results: return list(results) + # In conversational memory, explicit speaker names are stronger entity + # anchors than other capitalized concepts in the question (e.g. LGBTQ, + # PostgreSQL, New York). If at least one query entity is a known speaker, + # scope to those speaker entities; otherwise preserve generic entity scope. + known_speakers = { + str((item.get("provenance") or {}).get("speaker") or "").strip().casefold() + for item in results + if str((item.get("provenance") or {}).get("speaker") or "").strip() + } + speaker_keys = [key for key in entity_keys if key in known_speakers] + scope_keys = speaker_keys or entity_keys + matched: list[dict] = [] - buckets: dict[str, list[dict]] = {key: [] for key in entity_keys} + buckets: dict[str, list[dict]] = {key: [] for key in scope_keys} for item in results: text = str(item.get("text") or "").casefold() @@ -47,7 +59,7 @@ def filter_entity_scoped_results( item_entities = [ key - for key in entity_keys + for key in scope_keys if key in text or key == speaker or key in prov_entities ] if not item_entities: @@ -59,7 +71,7 @@ def filter_entity_scoped_results( if len(matched) < min_matches: return list(results) - if len(entity_keys) == 1 or not all(buckets[key] for key in entity_keys): + if len(scope_keys) == 1 or not all(buckets[key] for key in scope_keys): return matched # Round-robin the per-entity rankings, then append any remaining matched @@ -68,7 +80,7 @@ def filter_entity_scoped_results( seen: set[str] = set() max_len = max(len(bucket) for bucket in buckets.values()) for index in range(max_len): - for key in entity_keys: + for key in scope_keys: bucket = buckets[key] if index >= len(bucket): continue From 1ac2a0786aad13524e75ab97696c91620e064cac Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 21:59:27 +0530 Subject: [PATCH 096/109] test: prioritize speaker identity over topical capitalized entities --- tests/test_ranking.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/test_ranking.py b/tests/test_ranking.py index 2e88696..3e92c15 100644 --- a/tests/test_ranking.py +++ b/tests/test_ranking.py @@ -49,3 +49,20 @@ def test_multi_entity_filter_balances_named_people(): balanced = filter_entity_scoped_results(results, ["Jean", "John"]) assert balanced[0]["id"] == "a1" assert balanced[1]["id"] == "b1" + + +def test_speaker_entity_takes_priority_over_capitalized_topic(): + results = [ + { + "id": "caroline", + "text": "Caroline attended a pride event.", + "provenance": {"speaker": "Caroline"}, + }, + { + "id": "melanie", + "text": "Melanie discussed LGBTQ community events.", + "provenance": {"speaker": "Melanie"}, + }, + ] + scoped = filter_entity_scoped_results(results, ["LGBTQ", "Caroline"], min_matches=1) + assert [item["id"] for item in scoped] == ["caroline"] From 8a38f038acdef00aa024f19cf4c6c7f66bbee5cb Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 22:00:39 +0530 Subject: [PATCH 097/109] fix: restrict identity predicate matching to direct identity questions --- src/ultimate_memory/answer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ultimate_memory/answer.py b/src/ultimate_memory/answer.py index 776e2e4..ad07eb6 100644 --- a/src/ultimate_memory/answer.py +++ b/src/ultimate_memory/answer.py @@ -177,7 +177,8 @@ r"\bwhat\s+is\s+.+?'s\s+(?:identity|gender)\b|" r"\bwhat\s+(?:identity|gender)\s+(?:does|is|was)\b|" r"\b(?:identify|identifies)\s+as\b|" - r"\bis\s+.+?\s+(?:transgender|nonbinary|non-binary)\b", + r"\bis\s+[A-Z][\w.-]*(?:\s+[A-Z][\w.-]*)?\s+" + r"(?:a\s+|an\s+)?(?:transgender|nonbinary|non-binary)\b", re.I, ) _IDENTITY_PHRASE_RE = re.compile( From b6d3237b4957d8a991a085eb3978e8c9649f69bb Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 22:04:07 +0530 Subject: [PATCH 098/109] fix: preserve relevant evidence inside moderate session contexts --- src/ultimate_memory/answer.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/ultimate_memory/answer.py b/src/ultimate_memory/answer.py index ad07eb6..f6ba58d 100644 --- a/src/ultimate_memory/answer.py +++ b/src/ultimate_memory/answer.py @@ -1022,8 +1022,10 @@ def _is_usable_context(item: _ContextItem) -> bool: return True if item.memory_type in _PREFERRED_MEMORY_TYPES and len(text) >= 8: return True - # Giant raw session dumps drown extractive QA — keep only shorter evidence. - if len(text) > 700 and item.memory_type in {"log", "note"}: + # Structured reasoning works sentence-by-sentence, so moderately long + # session evidence is still useful. Only reject truly oversized raw dumps; + # the context compiler already enforces the global packet budget. + if len(text) > 3000 and item.memory_type in {"log", "note"}: return False if len(text) < 8 and not re.search(r"\b(19|20)\d{2}\b", text): return False From 3affbf7d9dadbd1f74b98c8a86715dd7edce8883 Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 22:04:15 +0530 Subject: [PATCH 099/109] test: preserve relevant evidence in moderate session logs --- tests/test_answer.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/test_answer.py b/tests/test_answer.py index f0962ae..6295adb 100644 --- a/tests/test_answer.py +++ b/tests/test_answer.py @@ -440,3 +440,16 @@ def test_shared_city_cleanup_keeps_city_not_time_modifier(): ] answer = synthesize_answer("Which city have both Jean and John visited?", contexts) assert answer.lower() == "rome" + + +def test_structured_reasoner_keeps_relevant_sentence_in_moderate_log(): + filler = "Unrelated small talk. " * 45 + contexts = [ + { + "text": filler + " Caroline: I've known this group of friends for 4 years.", + "memory_type": "log", + "score": 0.7, + } + ] + answer = synthesize_answer("How long has Caroline had this group of friends?", contexts) + assert answer.lower() == "4 years" From 630d5c4feddc453084e923ea6dd0bf2fa6bbba02 Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 22:04:52 +0530 Subject: [PATCH 100/109] fix: trust high-confidence single structured list values --- src/ultimate_memory/answer.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/ultimate_memory/answer.py b/src/ultimate_memory/answer.py index f6ba58d..91585c2 100644 --- a/src/ultimate_memory/answer.py +++ b/src/ultimate_memory/answer.py @@ -830,7 +830,11 @@ def _shared_entity_answer( return _truncate(surface, max_chars) -def _list_answer(question: str, normalized: list[_ContextItem], max_chars: int) -> str | None: +def _list_answer( + question: str, + normalized: list[_ContextItem], + max_chars: int, +) -> tuple[str, bool] | None: question_words = _content_words(question) entities = _question_entities(question) scored_sentences: list[tuple[float, str]] = [] @@ -887,7 +891,7 @@ def _list_answer(question: str, normalized: list[_ContextItem], max_chars: int) if values: compact = ", ".join(values) - return _truncate(compact, max_chars) + return _truncate(compact, max_chars), True # Last-resort evidence aggregation when no structured values were extractable. chosen: list[str] = [] @@ -899,7 +903,7 @@ def _list_answer(question: str, normalized: list[_ContextItem], max_chars: int) used += len(sentence) + 2 if len(chosen) >= 2: break - return _truncate(" ".join(chosen), max_chars) if chosen else None + return (_truncate(" ".join(chosen), max_chars), False) if chosen else None def _normalize_contexts( @@ -1455,10 +1459,11 @@ def synthesize_answer( # Distributed/list questions may require evidence from multiple contexts. # Only use aggregation when at least two distinct high-relevance sentences exist. if list_question: - list_answer = _list_answer(question, normalized, max_chars) - if list_answer: + list_result = _list_answer(question, normalized, max_chars) + if list_result: + list_answer, structured_values = list_result parts = _split_sentences(list_answer) - if "," in list_answer or len(parts) >= 2 or any( + if structured_values or "," in list_answer or len(parts) >= 2 or any( cue in list_answer.lower() for cue in ( "visited", "trip to", "offer", "provid", From 875e58da8ac3be594f9d68ac84c01bf87be6700b Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 22:05:02 +0530 Subject: [PATCH 101/109] test: trust single structured list extraction --- tests/test_answer.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/test_answer.py b/tests/test_answer.py index 6295adb..3984885 100644 --- a/tests/test_answer.py +++ b/tests/test_answer.py @@ -453,3 +453,15 @@ def test_structured_reasoner_keeps_relevant_sentence_in_moderate_log(): ] answer = synthesize_answer("How long has Caroline had this group of friends?", contexts) assert answer.lower() == "4 years" + + +def test_single_structured_list_value_is_trusted(): + contexts = [ + { + "text": "Melanie: Been running longer since our last chat - a great way to destress.", + "score": 0.9, + }, + {"text": "Melanie: Thanks, Caroline!", "score": 1.0}, + ] + answer = synthesize_answer("What does Melanie do to destress?", contexts) + assert answer.lower() == "running" From d88e3159c102889613595c2d9ee555d410b26e35 Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 22:06:17 +0530 Subject: [PATCH 102/109] feat: diversify context across sessions for multi-evidence queries --- src/ultimate_memory/router.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/ultimate_memory/router.py b/src/ultimate_memory/router.py index 2300cc7..21736cd 100644 --- a/src/ultimate_memory/router.py +++ b/src/ultimate_memory/router.py @@ -358,12 +358,21 @@ def answer( else: rich_contexts.sort(key=lambda item: float(item.get("score") or 0.0), reverse=True) - context_budget = 18000 if plan.kind == "multi_hop" else 12000 + if plan.multi_evidence and not plan.requires_bridge: + context_budget = 20000 + max_per_source = 3 + elif plan.requires_bridge: + context_budget = 18000 + max_per_source = 8 + else: + context_budget = 12000 + max_per_source = 5 + rich_contexts = compile_context_packet( rich_contexts, max_chars=context_budget, max_items=max(limit * 3, 18), - max_per_source=8 if plan.kind == "multi_hop" else 5, + max_per_source=max_per_source, ) use_local_llm = use_llm_from_env() if use_llm is None else use_llm From 086443807b9729363d286f496feb3062bc9db3a8 Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 22:07:30 +0530 Subject: [PATCH 103/109] feat: preserve chunk metadata in keyword retrieval --- src/ultimate_memory/store.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/ultimate_memory/store.py b/src/ultimate_memory/store.py index 31157d5..57a6047 100644 --- a/src/ultimate_memory/store.py +++ b/src/ultimate_memory/store.py @@ -238,7 +238,8 @@ def keyword_search( rows = conn.execute( f""" select f.id, f.title, f.text, f.source_path, f.memory_type, - bm25(memory_fts) as score + bm25(memory_fts) as score, + i.metadata_json, i.created_at, i.project_path from memory_fts f join memory_index i on i.id = f.id where memory_fts match ? @@ -259,7 +260,8 @@ def keyword_search( like_params.append(limit) rows = conn.execute( f""" - select id, title, text, source_path, memory_type, 0.0 as score + select id, title, text, source_path, memory_type, 0.0 as score, + metadata_json, created_at, project_path from memory_index where (title like ? or text like ?) {fallback_clause} From 43df5ade421666ecbb9b90bf1c4b8a07c14b1b4c Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 22:07:36 +0530 Subject: [PATCH 104/109] feat: expose conversational pair provenance in keyword search --- src/ultimate_memory/router.py | 38 +++++++++++++++++++++++++---------- 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/src/ultimate_memory/router.py b/src/ultimate_memory/router.py index 21736cd..0e37185 100644 --- a/src/ultimate_memory/router.py +++ b/src/ultimate_memory/router.py @@ -1,6 +1,7 @@ from __future__ import annotations import concurrent.futures +import json import logging import re from datetime import UTC, datetime @@ -448,18 +449,33 @@ def _vector(): def _keyword(): rows = self.store.keyword_search(query, actual_limit) - return [ - SearchResult( - id=row["id"], - title=row["title"], - text=row["text"], - source_path=row["source_path"], - memory_type=row["memory_type"], - score=self._keyword_score(query, row["text"]), - provenance={"source": "sqlite-fts"}, + output: list[SearchResult] = [] + for row in rows: + try: + metadata = json.loads(row.get("metadata_json") or "{}") + except (TypeError, json.JSONDecodeError): + metadata = {} + output.append( + SearchResult( + id=row["id"], + title=row["title"], + text=row["text"], + source_path=row["source_path"], + memory_type=row["memory_type"], + score=self._keyword_score(query, row["text"]), + provenance={ + "source": "sqlite-fts", + "metadata": metadata, + "session_id": metadata.get("session_id"), + "question_speaker": metadata.get("question_speaker"), + "answer_speaker": metadata.get("answer_speaker"), + "pair": bool(metadata.get("pair")), + "created_at": row.get("created_at"), + "project_path": row.get("project_path"), + }, + ) ) - for row in rows - ] + return output def _basic(): return self.basic.search(query, actual_limit, include_cli=False) From 7ce9a7cec1bbc5be8c1c566e6f6b9552fede32ad Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 22:09:10 +0530 Subject: [PATCH 105/109] feat: make conversational entity scope attribution-aware --- src/ultimate_memory/ranking.py | 108 ++++++++++++++++++++++++--------- 1 file changed, 79 insertions(+), 29 deletions(-) diff --git a/src/ultimate_memory/ranking.py b/src/ultimate_memory/ranking.py index 6c7335f..feec1ea 100644 --- a/src/ultimate_memory/ranking.py +++ b/src/ultimate_memory/ranking.py @@ -16,57 +16,107 @@ def _tokens(text: str) -> set[str]: } +def _provenance_value(provenance: dict, key: str): + """Read a provenance field across direct, metadata, and vector payload forms.""" + value = provenance.get(key) + if value not in (None, "", [], {}): + return value + metadata = provenance.get("metadata") + if isinstance(metadata, dict): + value = metadata.get(key) + if value not in (None, "", [], {}): + return value + payload = provenance.get("payload") + if isinstance(payload, dict): + value = payload.get(key) + if value not in (None, "", [], {}): + return value + return None + + def filter_entity_scoped_results( results: list[dict], entities: list[str], *, min_matches: int = 2, ) -> list[dict]: - """Prefer and balance evidence attached to entities named in the query. + """Prefer evidence *attributed to* entities named in the query. - For one entity this is a hard topical gate when enough evidence exists. - For multiple named entities, matched evidence is interleaved so one person's - larger memory history cannot crowd the other person out of the context packet. + Conversational memory frequently contains windows with two speakers. A name + appearing in the question half of a pair is weaker evidence than a direct + turn or the answer half of that pair. This gate therefore separates strong + attribution from mere mentions before balancing multiple named entities. """ entity_keys = [entity.strip().casefold() for entity in entities if entity.strip()] if not entity_keys or not results: return list(results) - # In conversational memory, explicit speaker names are stronger entity - # anchors than other capitalized concepts in the question (e.g. LGBTQ, - # PostgreSQL, New York). If at least one query entity is a known speaker, - # scope to those speaker entities; otherwise preserve generic entity scope. - known_speakers = { - str((item.get("provenance") or {}).get("speaker") or "").strip().casefold() - for item in results - if str((item.get("provenance") or {}).get("speaker") or "").strip() - } + known_speakers: set[str] = set() + for item in results: + provenance = item.get("provenance") or {} + for key in ("speaker", "answer_speaker"): + speaker = str(_provenance_value(provenance, key) or "").strip().casefold() + if speaker: + known_speakers.add(speaker) + speaker_keys = [key for key in entity_keys if key in known_speakers] scope_keys = speaker_keys or entity_keys - matched: list[dict] = [] - buckets: dict[str, list[dict]] = {key: [] for key in scope_keys} + strong: list[dict] = [] + weak: list[dict] = [] + strong_buckets: dict[str, list[dict]] = {key: [] for key in scope_keys} + weak_buckets: dict[str, list[dict]] = {key: [] for key in scope_keys} for item in results: text = str(item.get("text") or "").casefold() provenance = item.get("provenance") or {} prov_entities = { str(entity).strip().casefold() - for entity in provenance.get("entities") or [] + for entity in (_provenance_value(provenance, "entities") or []) if str(entity).strip() } - speaker = str(provenance.get("speaker") or "").strip().casefold() - - item_entities = [ - key + speaker = str(_provenance_value(provenance, "speaker") or "").strip().casefold() + answer_speaker = str( + _provenance_value(provenance, "answer_speaker") or "" + ).strip().casefold() + question_speaker = str( + _provenance_value(provenance, "question_speaker") or "" + ).strip().casefold() + + strong_entities: list[str] = [] + weak_entities: list[str] = [] + for key in scope_keys: + if key == speaker or key == answer_speaker or key in prov_entities: + strong_entities.append(key) + elif key in text or key == question_speaker: + weak_entities.append(key) + + if strong_entities: + strong.append(item) + for key in strong_entities: + strong_buckets[key].append(item) + elif weak_entities: + weak.append(item) + for key in weak_entities: + weak_buckets[key].append(item) + + # Strongly attributed evidence is preferred whenever it is sufficiently + # populated. Fall back to textual/question-side mentions only when necessary. + if len(strong) >= min_matches: + matched = strong + buckets = strong_buckets + else: + matched = list(strong) + seen = {str(item.get("id") or item.get("source_path") or id(item)) for item in matched} + for item in weak: + item_key = str(item.get("id") or item.get("source_path") or id(item)) + if item_key not in seen: + matched.append(item) + seen.add(item_key) + buckets = { + key: [*strong_buckets[key], *weak_buckets[key]] for key in scope_keys - if key in text or key == speaker or key in prov_entities - ] - if not item_entities: - continue - matched.append(item) - for key in item_entities: - buckets[key].append(item) + } if len(matched) < min_matches: return list(results) @@ -74,8 +124,8 @@ def filter_entity_scoped_results( if len(scope_keys) == 1 or not all(buckets[key] for key in scope_keys): return matched - # Round-robin the per-entity rankings, then append any remaining matched - # evidence in its original order. Items mentioning both entities are deduped. + # Round-robin per-entity evidence so one person's larger history cannot + # crowd another named person out of the context packet. balanced: list[dict] = [] seen: set[str] = set() max_len = max(len(bucket) for bucket in buckets.values()) From d6c685a4a4e51f37e0ca3c7a7ac4916863102eec Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 22:09:20 +0530 Subject: [PATCH 106/109] test: cover question-answer attribution in entity scope --- tests/test_ranking.py | 61 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/tests/test_ranking.py b/tests/test_ranking.py index 3e92c15..71eed68 100644 --- a/tests/test_ranking.py +++ b/tests/test_ranking.py @@ -66,3 +66,64 @@ def test_speaker_entity_takes_priority_over_capitalized_topic(): ] scoped = filter_entity_scoped_results(results, ["LGBTQ", "Caroline"], min_matches=1) assert [item["id"] for item in scoped] == ["caroline"] + + +def test_question_speaker_only_pair_is_weak_evidence(): + results = [ + { + "id": "pair", + "text": "Melanie: What books have you read?\nCaroline: I read Dune.", + "provenance": { + "pair": True, + "question_speaker": "Melanie", + "answer_speaker": "Caroline", + }, + }, + { + "id": "direct", + "text": "Melanie: I read The Hobbit last year.", + "provenance": {"speaker": "Melanie", "direct_turn": True}, + }, + ] + scoped = filter_entity_scoped_results(results, ["Melanie"], min_matches=1) + assert [item["id"] for item in scoped] == ["direct"] + + +def test_pair_answer_speaker_is_strong_evidence(): + results = [ + { + "id": "pair", + "text": "Caroline: What books have you read?\nMelanie: I read The Hobbit.", + "provenance": { + "pair": True, + "question_speaker": "Caroline", + "answer_speaker": "Melanie", + }, + }, + { + "id": "noise", + "text": "Caroline mentioned Melanie while discussing art.", + "provenance": {}, + }, + ] + scoped = filter_entity_scoped_results(results, ["Melanie"], min_matches=1) + assert [item["id"] for item in scoped] == ["pair"] + + +def test_nested_vector_payload_preserves_answer_attribution(): + results = [ + { + "id": "vector-pair", + "text": "Caroline: What do you do to relax?\nMelanie: I go running.", + "provenance": { + "source": "qdrant", + "payload": { + "pair": True, + "question_speaker": "Caroline", + "answer_speaker": "Melanie", + }, + }, + } + ] + scoped = filter_entity_scoped_results(results, ["Melanie"], min_matches=1) + assert [item["id"] for item in scoped] == ["vector-pair"] From 6497426e52f18c96529694f2a6890cf6d10e71a5 Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 22:09:33 +0530 Subject: [PATCH 107/109] fix: propagate project scope to SQLite keyword retrieval --- src/ultimate_memory/router.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/ultimate_memory/router.py b/src/ultimate_memory/router.py index 0e37185..a8a7fd0 100644 --- a/src/ultimate_memory/router.py +++ b/src/ultimate_memory/router.py @@ -448,7 +448,11 @@ def _vector(): ) if self._vector_ready else [] def _keyword(): - rows = self.store.keyword_search(query, actual_limit) + rows = self.store.keyword_search( + query, + actual_limit, + project_path=project_path, + ) output: list[SearchResult] = [] for row in rows: try: From 352b481919012d1a6717c6526e360fa35a019f82 Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 22:11:50 +0530 Subject: [PATCH 108/109] fix: retain weak conversational evidence after strong attribution --- src/ultimate_memory/ranking.py | 35 +++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/src/ultimate_memory/ranking.py b/src/ultimate_memory/ranking.py index feec1ea..56f5ff8 100644 --- a/src/ultimate_memory/ranking.py +++ b/src/ultimate_memory/ranking.py @@ -100,23 +100,24 @@ def filter_entity_scoped_results( for key in weak_entities: weak_buckets[key].append(item) - # Strongly attributed evidence is preferred whenever it is sufficiently - # populated. Fall back to textual/question-side mentions only when necessary. - if len(strong) >= min_matches: - matched = strong - buckets = strong_buckets - else: - matched = list(strong) - seen = {str(item.get("id") or item.get("source_path") or id(item)) for item in matched} - for item in weak: - item_key = str(item.get("id") or item.get("source_path") or id(item)) - if item_key not in seen: - matched.append(item) - seen.add(item_key) - buckets = { - key: [*strong_buckets[key], *weak_buckets[key]] - for key in scope_keys - } + # Strongly attributed evidence is ordered first, but weaker mention / + # question-side evidence is retained as recall support. Hard-dropping it can + # lose conversational premises whose answer lives in an adjacent turn. + matched = list(strong) + seen = { + str(item.get("id") or item.get("source_path") or id(item)) + for item in matched + } + for item in weak: + item_key = str(item.get("id") or item.get("source_path") or id(item)) + if item_key not in seen: + matched.append(item) + seen.add(item_key) + + buckets = { + key: [*strong_buckets[key], *weak_buckets[key]] + for key in scope_keys + } if len(matched) < min_matches: return list(results) From 139abc501286836691929f28b347ae153e5b2027 Mon Sep 17 00:00:00 2001 From: Cookie_Cat21 Date: Sun, 20 Sep 2026 22:12:09 +0530 Subject: [PATCH 109/109] test: require strong-first attribution without recall loss --- tests/test_ranking.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_ranking.py b/tests/test_ranking.py index 71eed68..376fdfd 100644 --- a/tests/test_ranking.py +++ b/tests/test_ranking.py @@ -86,7 +86,8 @@ def test_question_speaker_only_pair_is_weak_evidence(): }, ] scoped = filter_entity_scoped_results(results, ["Melanie"], min_matches=1) - assert [item["id"] for item in scoped] == ["direct"] + assert scoped[0]["id"] == "direct" + assert {item["id"] for item in scoped} == {"direct", "pair"} def test_pair_answer_speaker_is_strong_evidence(): @@ -107,7 +108,7 @@ def test_pair_answer_speaker_is_strong_evidence(): }, ] scoped = filter_entity_scoped_results(results, ["Melanie"], min_matches=1) - assert [item["id"] for item in scoped] == ["pair"] + assert scoped[0]["id"] == "pair" def test_nested_vector_payload_preserves_answer_attribution():