diff --git a/.github/workflows/full-clean-benchmark.yml b/.github/workflows/full-clean-benchmark.yml index 00a9d7dd..5e4b860a 100644 --- a/.github/workflows/full-clean-benchmark.yml +++ b/.github/workflows/full-clean-benchmark.yml @@ -4,10 +4,16 @@ on: pull_request: workflow_dispatch: +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: clean-benchmark: runs-on: ubuntu-latest timeout-minutes: 20 + env: + ULTIMATE_MEMORY_LOCAL_SEMANTIC: "1" steps: - uses: actions/checkout@v4 - name: Install uv diff --git a/.github/workflows/memeval.yml b/.github/workflows/memeval.yml new file mode 100644 index 00000000..1e52d042 --- /dev/null +++ b/.github/workflows/memeval.yml @@ -0,0 +1,92 @@ +name: Independent MemEval + +on: + pull_request: + workflow_dispatch: + inputs: + samples: + description: "Number of LoCoMo conversations" + required: true + default: "1" + with_judge: + description: "Run GPT judge in addition to token F1" + required: true + type: boolean + default: false + +concurrency: + group: independent-memeval + cancel-in-progress: true + +jobs: + memeval: + runs-on: ubuntu-latest + timeout-minutes: 60 + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + LLM_MODEL: gpt-4.1-mini + EMBEDDING_MODEL: text-embedding-3-small + steps: + - name: Check API key + id: api_key + run: | + if [ -z "$OPENAI_API_KEY" ]; then + echo "::notice::OPENAI_API_KEY Actions secret is not configured; independent MemEval is skipped." + echo "available=false" >> "$GITHUB_OUTPUT" + else + echo "available=true" >> "$GITHUB_OUTPUT" + fi + + - name: Checkout Ultimate Memory + if: steps.api_key.outputs.available == 'true' + uses: actions/checkout@v4 + with: + path: ultimate-memory + + - name: Checkout Prosus MemEval + if: steps.api_key.outputs.available == 'true' + uses: actions/checkout@v4 + with: + repository: ProsusAI/MemEval + path: MemEval + + - name: Install uv + if: steps.api_key.outputs.available == 'true' + uses: astral-sh/setup-uv@v3 + + - name: Install MemEval and Ultimate Memory + if: steps.api_key.outputs.available == 'true' + working-directory: MemEval + run: | + uv sync + uv pip install httpx + uv pip install -e ../ultimate-memory + cp ../ultimate-memory/integrations/memeval/ultimate_memory.py src/agents_memory/systems/ultimate_memory.py + + - name: Run fair head-to-head + if: steps.api_key.outputs.available == 'true' + working-directory: MemEval + shell: bash + run: | + SAMPLES="1" + EXTRA="--skip-judge" + if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then + SAMPLES="${{ inputs.samples }}" + if [ "${{ inputs.with_judge }}" = "true" ]; then + EXTRA="" + fi + fi + uv run python scripts/run_full_benchmark.py \ + --systems ultimate_memory,propmem \ + --num-samples "$SAMPLES" \ + --llm-model "$LLM_MODEL" \ + $EXTRA \ + --output-dir benchmark-output + + - name: Upload benchmark results + if: always() && steps.api_key.outputs.available == 'true' + uses: actions/upload-artifact@v4 + with: + name: independent-memeval-results + path: MemEval/benchmark-output/ + if-no-files-found: warn diff --git a/.github/workflows/reader-benchmark.yml b/.github/workflows/reader-benchmark.yml index d3213abc..ccefdfab 100644 --- a/.github/workflows/reader-benchmark.yml +++ b/.github/workflows/reader-benchmark.yml @@ -4,12 +4,17 @@ on: pull_request: workflow_dispatch: +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: reader-benchmark: runs-on: ubuntu-latest timeout-minutes: 20 env: TOKENIZERS_PARALLELISM: "false" + ULTIMATE_MEMORY_LOCAL_SEMANTIC: "1" steps: - uses: actions/checkout@v4 - name: Install uv diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 8d784dab..7f794433 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -6,6 +6,10 @@ on: pull_request: branches: [master] +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: test: runs-on: ubuntu-latest diff --git a/evals/run_clean_benchmarks.py b/evals/run_clean_benchmarks.py index 9317c970..5a1ff6ca 100644 --- a/evals/run_clean_benchmarks.py +++ b/evals/run_clean_benchmarks.py @@ -156,6 +156,7 @@ def run( planner_correct = 0 planner_confusion: dict[str, dict[str, int]] = defaultdict(lambda: defaultdict(int)) asked = 0 + diagnostics: list[dict] = [] started = time.perf_counter() for sample in data: @@ -209,6 +210,19 @@ def run( planner_total += 1 planner_correct += int(planned_kind == expected_kind) planner_confusion[expected_kind][planned_kind] += 1 + diagnostics.append({ + "sample_id": sample_id, + "category": category, + "question": qa["question"], + "gold": golds, + "answer": result["answer"], + "token_f1": round(100 * score, 2), + "evidence_recall": round(100 * evidence_score, 2), + "gold_token_coverage": round(100 * coverage, 2), + "planned_kind": planned_kind, + "expected_kind": expected_kind, + "top_contexts": [str(x)[:320] for x in contexts_used[:4]], + }) asked += 1 if max_questions is not None and asked >= max_questions: @@ -258,6 +272,17 @@ def run( "by_category": by_category, "use_llm": use_llm, "use_reader": use_reader, + "worst_answer_failures": sorted( + diagnostics, + key=lambda item: (item["token_f1"], -item["gold_token_coverage"]), + )[:12], + "retrieval_failures": sorted( + diagnostics, + key=lambda item: (item["gold_token_coverage"], item["evidence_recall"]), + )[:12], + "planner_misses": [ + item for item in diagnostics if item["planned_kind"] != item["expected_kind"] + ][:12], } diff --git a/integrations/memeval/README.md b/integrations/memeval/README.md new file mode 100644 index 00000000..f1ff85c0 --- /dev/null +++ b/integrations/memeval/README.md @@ -0,0 +1,54 @@ +# Ultimate Memory on ProsusAI MemEval + +ProsusAI/MemEval is the primary external fair-comparison gate for Ultimate Memory. +It standardizes the answer LLM, embedding model, scoring pipeline, and token accounting +across memory systems. + +At the time this integration was added, MemEval's published LoCoMo leader is PropMem +at 0.605 token-F1 and 0.823 judge score. + +## Run head-to-head + +Clone MemEval next to Ultimate Memory, install both projects, and copy the adapter: + +```bash +git clone https://github.com/ProsusAI/MemEval.git +cd MemEval +uv sync --all-extras +uv pip install -e ../ultimate-memory +cp ../ultimate-memory/integrations/memeval/ultimate_memory.py \ + src/agents_memory/systems/ultimate_memory.py +``` + +Set `OPENAI_API_KEY`, then run one conversation first: + +```bash +uv run python scripts/run_full_benchmark.py \ + --systems ultimate_memory,propmem \ + --num-samples 1 \ + --llm-model gpt-4.1-mini \ + --skip-judge +``` + +For the full externally comparable LoCoMo run: + +```bash +uv run python scripts/run_full_benchmark.py \ + --systems ultimate_memory,propmem \ + --num-samples 10 \ + --llm-model gpt-4.1-mini +``` + +Do not claim a leaderboard position from Ultimate Memory's internal clean harness. +Use the MemEval result for cross-system claims. + +## Fairness + +The adapter: + +- ingests only raw conversation sessions; +- never receives the gold question category; +- uses MemEval's supplied answer model; +- uses `text-embedding-3-small` for semantic fallback; +- leaves MemEval's scoring and judge code untouched; +- counts answer-LLM tokens through MemEval's normal OpenAI instrumentation. diff --git a/integrations/memeval/ultimate_memory.py b/integrations/memeval/ultimate_memory.py new file mode 100644 index 00000000..bfca2c79 --- /dev/null +++ b/integrations/memeval/ultimate_memory.py @@ -0,0 +1,396 @@ +"""ProsusAI/MemEval adapter for Ultimate Memory. + +Copy this file into MemEval's ``src/agents_memory/systems/ultimate_memory.py`` +and install Ultimate Memory in the same environment. MemEval will discover it +automatically through its system registry. + +The adapter intentionally uses the benchmark-provided answer model and +text-embedding-3-small so comparisons with PropMem use the same reader and +embedding family rather than Ultimate Memory's local extractive reader. +""" +from __future__ import annotations + +import json +import os +import re +from concurrent.futures import ThreadPoolExecutor, as_completed +from pathlib import Path +from tempfile import TemporaryDirectory + +from openai import OpenAI + +from agents_memory.systems._helpers import _qa_results +from ultimate_memory.config import ( + BasicMemoryConfig, + DashboardConfig, + Neo4jConfig, + PathsConfig, + QdrantConfig, + RetrievalConfig, + Settings, +) +from ultimate_memory.dates import parse_loose_date +from ultimate_memory.local_semantic import LocalSemanticIndex +from ultimate_memory.models import CompiledMemory +from ultimate_memory.router import MemoryRouter + + +SYSTEM_INFO = { + "architecture": ( + "atomic bi-temporal memory + entity-scoped hybrid retrieval + " + "conversation windows + adaptive multi-hop planning" + ), + "infrastructure": "SQLite FTS + in-process OpenAI semantic retrieval; no external DB required", +} + + +COMPILE_PROMPT = """Compile durable atomic memories from this raw conversation session. + +Conversation participants: {participants} +Session date: {session_date} + +Raw turns: +{turns} + +Rules: +1. Extract only information supported by the turns. Never invent or infer unstated facts. +2. Each memory must be atomic: one durable fact, preference, decision, procedure, event, + relationship, status, ownership fact, activity, or plan. +3. Name the subject explicitly. Replace first-person pronouns with the speaker name. +4. Preserve exact names, places, titles, dates, quantities, and technical values verbatim. +5. Convert relative dates to absolute dates using the session date whenever possible. For example, resolve yesterday, last week, this month, three months ago, and next Tuesday into an explicit calendar date or month/year. +6. Keep separate facts separate; do not merge unrelated information. +7. Include the dialogue IDs that directly support each memory. +8. entities must contain the people/organizations/places directly involved. +9. memory_type must be one of: fact, preference, decision, procedure. +10. Extract EVERY durable fact, even minor ones; do not stop after the most salient facts. +11. Do not extract greetings, compliments, filler, or questions that contain no answer. +12. Confidence is 0-1 and should reflect how explicitly the memory is stated. + +Return JSON: +{{"memories": [ + {{ + "text": "explicit atomic memory", + "memory_type": "fact", + "entities": ["Entity"], + "source_dia_ids": ["D1:2"], + "confidence": 0.95 + }} +]}} +""" + + +ANSWER_PROMPT = """Answer the question using ONLY the memory evidence below. + +Known conversation participants: {participants} + +Evidence: +{evidence} + +Question: {question} + +Rules: +1. Reason from the evidence, including across multiple evidence items when needed. +2. Keep facts attached to the correct person; do not transfer another person's facts. +3. For temporal questions, use the dates/timestamps in the evidence and resolve relative dates. +4. For lists or multi-part questions, include every supported item and deduplicate them. +5. For questions asking what would/could/likely happen, make the minimal inference supported by evidence. +6. Give a direct compact answer, not an explanation. Prefer exact words from the evidence. +7. If the evidence does not support an answer, return "None". +8. Answer in the same language as the question. + +Return JSON exactly as: +{{"reasoning": "brief evidence-grounded reasoning", "answer": "direct compact answer"}} +""" + + +def _settings(root: Path) -> Settings: + vault = root / "vault" + private = root / "private" + vault.mkdir(parents=True, exist_ok=True) + return Settings( + paths=PathsConfig( + repo_root=root, + basic_memory_vault=vault, + private_store=private, + ), + retrieval=RetrievalConfig( + collection_name="memeval", + embedding_model="BAAI/bge-small-en-v1.5", + chunk_chars=900, + chunk_overlap=120, + default_limit=12, + bootstrap_token_budget_chars=6000, + ), + qdrant=QdrantConfig(url="http://127.0.0.1:1"), + neo4j=Neo4jConfig( + uri="bolt://127.0.0.1:1", + user="neo4j", + password="disabled", + ), + basic_memory=BasicMemoryConfig(project="memeval", cli="basic-memory-missing"), + dashboard=DashboardConfig(host="127.0.0.1", port=8787), + ) + + +def _session_text(conversation: dict, session_key: str) -> str: + turns = conversation.get(session_key) or [] + when = conversation.get(f"{session_key}_date_time") or "" + lines = ["---"] + if when: + lines.extend([f"created_at: {when}", f"session_date: {when}"]) + lines.extend(["---", ""]) + for turn in turns: + speaker = str(turn.get("speaker") or "Speaker") + dia_id = str(turn.get("dia_id") or "") + text = str(turn.get("text") or "").strip() + prefix = f"[{dia_id}] " if dia_id else "" + lines.append(f"{prefix}{speaker}: {text}") + return "\n".join(lines) + + +def _compile_session( + client: OpenAI, + model: str, + *, + participants: list[str], + session_date: str, + session_text: str, +) -> list[CompiledMemory]: + turn_lines = [ + line + for line in session_text.splitlines() + if line.strip().startswith("[D") + ] + if not turn_lines: + return [] + response = client.chat.completions.create( + model=model, + messages=[ + { + "role": "user", + "content": COMPILE_PROMPT.format( + participants=", ".join(participants) or "unknown", + session_date=session_date or "unknown", + turns="\n".join(turn_lines), + ), + } + ], + response_format={"type": "json_object"}, + temperature=0, + max_tokens=8192, + ) + content = response.choices[0].message.content or "{}" + try: + payload = json.loads(content) + except json.JSONDecodeError: + return [] + raw_memories = payload.get("memories") or payload.get("facts") or [] + if not isinstance(raw_memories, list): + return [] + + compiled: list[CompiledMemory] = [] + for raw in raw_memories[:160]: + if not isinstance(raw, dict): + continue + try: + compiled.append(CompiledMemory.model_validate(raw)) + except Exception: + continue + return compiled + + +def _participants(conversation: dict) -> list[str]: + names: list[str] = [] + for key in ("speaker_a", "speaker_b"): + value = str(conversation.get(key) or "").strip() + if value and value not in names: + names.append(value) + if names: + return names + for key, turns in conversation.items(): + if not key.startswith("session_") or key.endswith("_date_time"): + continue + if not isinstance(turns, list): + continue + for turn in turns: + speaker = str(turn.get("speaker") or "").strip() + if speaker and speaker not in names: + names.append(speaker) + return names[:8] + + + +def _is_inferential(question: str) -> bool: + lower = question.lower().strip() + return bool( + lower.startswith(("would ", "could ", "should ")) + or re.match(r"^what\s+(?:would|might|could)\b", lower) + or re.match(r"^(?:do you think|how would|is it likely)\b", lower) + or " likely " in lower + or " might " in lower + or (" prefer " in lower and "?" in question) + ) + + +def run( + conv: dict, + llm_model: str, + run_judge: bool, + category_names: dict | None = None, + judge_fn: str | None = None, +) -> list[dict]: + client = OpenAI(api_key=os.environ["OPENAI_API_KEY"]) + conversation = conv["conversation"] + participants = _participants(conversation) + + with TemporaryDirectory(prefix="ultimate-memory-memeval-") as temp_dir: + root = Path(temp_dir) + project_path = str(root / "conversation") + router = MemoryRouter(_settings(root)) + + # Use the exact embedding family from the MemEval comparison rig. + def openai_embed(texts: list[str]) -> list[list[float]]: + if not texts: + return [] + response = client.embeddings.create( + model=os.environ.get("EMBEDDING_MODEL", "text-embedding-3-small"), + input=texts, + ) + ordered = sorted(response.data, key=lambda item: item.index) + return [list(item.embedding) for item in ordered] + + router._local_semantic = LocalSemanticIndex(openai_embed) + + previous_semantic = os.environ.get("ULTIMATE_MEMORY_LOCAL_SEMANTIC") + os.environ["ULTIMATE_MEMORY_LOCAL_SEMANTIC"] = "1" + try: + session_keys = sorted( + ( + key + for key in conversation + if key.startswith("session_") and not key.endswith("_date_time") + ), + key=lambda key: int(key.split("_")[1]), + ) + compiled_total = 0 + session_payloads: list[tuple[str, str, str]] = [] + for session_key in session_keys: + raw_session = _session_text(conversation, session_key) + router.ingest_log( + client="memeval", + session_id=session_key, + transcript_or_path=raw_session, + project_path=project_path, + tags=["memeval"], + ) + session_date = str( + conversation.get(f"{session_key}_date_time") or "" + ) + session_payloads.append((session_key, session_date, raw_session)) + + compiled_by_session: dict[str, list[CompiledMemory]] = {} + with ThreadPoolExecutor(max_workers=min(10, max(1, len(session_payloads)))) as pool: + futures = { + pool.submit( + _compile_session, + client, + llm_model, + participants=participants, + session_date=session_date, + session_text=raw_session, + ): session_key + for session_key, session_date, raw_session in session_payloads + } + for future in as_completed(futures): + session_key = futures[future] + try: + compiled_by_session[session_key] = future.result() + except Exception as exc: + print(f" Compile error ({session_key}): {exc}") + compiled_by_session[session_key] = [] + + for session_key, session_date, _ in session_payloads: + compiled = compiled_by_session.get(session_key, []) + parsed_date = parse_loose_date(session_date) + event_time = parsed_date.isoformat() if parsed_date else None + outcome = router.ingest_compiled_memories( + compiled, + session_id=session_key, + project_path=project_path, + event_time=event_time, + compiler=f"memeval:{llm_model}", + ) + compiled_total += int(outcome.get("created") or 0) + + print( + f" Ingested: sessions={len(session_keys)}, " + f"compiled_memories={compiled_total}" + ) + + def answer_fn(question: str) -> str: + result = router.answer( + question, + project_path=project_path, + limit=16, + use_llm=False, + ) + contexts: list[str] = [] + seen: set[str] = set() + for raw in result.get("contexts_used") or []: + text = str(raw).strip() + key = text.casefold() + if not text or key in seen: + continue + seen.add(key) + contexts.append(text) + if len(contexts) >= 20: + break + + evidence = "\n\n".join( + f"[Memory {index}] {text}" + for index, text in enumerate(contexts, start=1) + ) + response = client.chat.completions.create( + model=llm_model, + messages=[ + { + "role": "user", + "content": ANSWER_PROMPT.format( + participants=", ".join(participants) or "unknown", + evidence=evidence or "(no evidence retrieved)", + question=( + question + + ( + "\n\nThis is an inference question. Draw the minimal logical conclusion supported by the named entity's own facts; do not return None merely because the conclusion is not stated verbatim." + if _is_inferential(question) + else "" + ) + ), + ), + } + ], + response_format={"type": "json_object"}, + temperature=0, + max_tokens=512, + ) + content = response.choices[0].message.content or "{}" + try: + payload = json.loads(content) + answer = str(payload.get("answer") or "").strip() + except json.JSONDecodeError: + answer = content.strip() + return answer or "None" + + return _qa_results( + conv, + answer_fn, + run_judge, + category_names=category_names, + judge_fn=judge_fn, + ) + finally: + if previous_semantic is None: + os.environ.pop("ULTIMATE_MEMORY_LOCAL_SEMANTIC", None) + else: + os.environ["ULTIMATE_MEMORY_LOCAL_SEMANTIC"] = previous_semantic diff --git a/src/ultimate_memory/answer.py b/src/ultimate_memory/answer.py index 6a6d40a3..91585c27 100644 --- a/src/ultimate_memory/answer.py +++ b/src/ultimate_memory/answer.py @@ -80,7 +80,13 @@ _SENTENCE_SPLIT = re.compile(r"(?<=[.!?])\s+|\n+") _ENTITY_RE = re.compile(r"\b([A-Z][a-z]+(?:\s+[A-Z][a-z]+)?)\b") _DATE_PATTERNS: list[re.Pattern[str]] = [ - # Day Month Year: 7 May 2023 + # Day Month Year, with optional comma: 7 May 2023 / 7 May, 2023 + re.compile( + r"\b\d{1,2}\s+" + r"(?:January|February|March|April|May|June|July|August|September|" + r"October|November|December)(?:,\s*|\s+)\d{4}\b", + re.I, + ), re.compile( r"\b\d{1,2}\s+" r"(?:January|February|March|April|May|June|July|August|September|" @@ -112,7 +118,7 @@ r"October|November|December)\s+\d{4}\b", re.I, ), - re.compile(r"\b\d+\s+years?\s+ago\b", re.I), + re.compile(r"\b(?:a\s+few|several|\d+)\s+years?\s+ago\b", re.I), re.compile(r"\b(?:in|on|during)\s+(?:the\s+)?(?:year\s+)?((?:19|20)\d{2})\b", re.I), re.compile(r"\b(?:19|20)\d{2}\b"), re.compile(r"\b(?:last|next)\s+(?:Monday|Tuesday|Wednesday|Thursday|Friday|Saturday|Sunday)\b", re.I), @@ -122,13 +128,23 @@ r"^(?:last|next|this)\s+(?:monday|tuesday|wednesday|thursday|friday|saturday|sunday|" r"week|weekend|month|year)$|" r"^(?:yesterday|today|tomorrow|recently|earlier|later)$|" - r"^(?:a few days ago|two days ago|2 days ago|last night)$", + r"^(?:a few days ago|a few years ago|several years ago|two days ago|2 days ago|last night)$", re.I, ) +_NUMBER_WORD = r"(?:one|two|three|four|five|six|seven|eight|nine|ten|eleven|twelve|a\s+few|several)" _DURATION_SPAN_RE = re.compile( - r"\b(\d+\s+years?(?:\s+ago)?|\d+\s+months?(?:\s+ago)?|\d+\s+weeks?(?:\s+ago)?|" - r"for\s+\d+\s+years?|\dover\s+\d+\s+years?)\b", + rf"\b((?:\d+|{_NUMBER_WORD})\s+years?(?:\s+ago)?|" + rf"(?:\d+|{_NUMBER_WORD})\s+months?(?:\s+ago)?|" + rf"(?:\d+|{_NUMBER_WORD})\s+weeks?(?:\s+ago)?|" + rf"(?:for|over)\s+(?:\d+|{_NUMBER_WORD})\s+(?:years?|months?|weeks?))\b", + re.I, +) + +_GREETING_ONLY_RE = re.compile( + r"^(?:\[D\d+:\d+\]\s*)?(?:[A-Z][a-z]+\s*:\s*)?" + r"(?:hey|hi|hello|thanks(?:\s+a\s+bunch)?|thank\s+you|wow|great|awesome|nice|cool|sure|yep|yeah)" + r"(?:[\s,!.'-]+[A-Z][a-z]+)?[!?.]*$", re.I, ) @@ -158,13 +174,28 @@ ) _DIA_TURN_RE = re.compile(r"^\[D\d+:\d+\]") _IDENTITY_QUESTION_RE = re.compile( - r"\bidentity\b|\b(?:gender|transgender)\b|what is .+'s (?:identity|gender)", + r"\bwhat\s+is\s+.+?'s\s+(?:identity|gender)\b|" + r"\bwhat\s+(?:identity|gender)\s+(?:does|is|was)\b|" + r"\b(?:identify|identifies)\s+as\b|" + r"\bis\s+[A-Z][\w.-]*(?:\s+[A-Z][\w.-]*)?\s+" + r"(?:a\s+|an\s+)?(?:transgender|nonbinary|non-binary)\b", re.I, ) _IDENTITY_PHRASE_RE = re.compile( r"\btransgender\b|\b(?:trans\s+)?woman\b|\b(?:trans\s+)?man\b|\bidentity\b", re.I, ) +_EXPLICIT_IDENTITY_RE = re.compile( + r"\b(transgender\s+(?:woman|man)|trans\s+(?:woman|man)|nonbinary|non-binary)\b", + re.I, +) +_FAVORITE_VALUE_RE = re.compile( + r"\b(?:my|his|her|their|[A-Z][a-z]+'s)\s+" + r"(?:favorite|favourite)\s+[^.!?,:]{0,50}?\s+(?:is|are)\s+([^.!?,;]{1,70})|" + r"\b([^.!?,;]{1,50})\s+(?:is|are)\s+" + r"(?:my|his|her|their|[A-Z][a-z]+'s)\s+(?:favorite|favourite)\b", + re.I, +) _JUNK_LINE_PATTERNS: tuple[re.Pattern[str], ...] = ( re.compile(r"^#"), @@ -299,6 +330,42 @@ def _is_identity_question(question: str) -> bool: return bool(_IDENTITY_QUESTION_RE.search(question)) +def _favorite_subject(question: str) -> bool: + return bool(re.search(r"\bfavou?rite\b", question, re.I)) + + +def _precise_scalar_answer(question: str, normalized: list["_ContextItem"]) -> str | None: + """Return high-precision scalar values before generic span ranking.""" + identity = _is_identity_question(question) + favorite = _favorite_subject(question) + + if identity: + for item in normalized: + match = _EXPLICIT_IDENTITY_RE.search(item.text) + if match: + return match.group(1).strip() + + if favorite: + q_words = _content_words(question) + entities = _question_entities(question) + candidates: list[tuple[float, str]] = [] + for item in normalized: + for sentence in _split_sentences(item.text): + match = _FAVORITE_VALUE_RE.search(sentence) + if not match: + continue + value = (match.group(1) or match.group(2) or "").strip(" .,:;-") + if not value: + continue + score = _overlap_score(q_words, sentence, entities) + candidates.append((score, value)) + if candidates: + candidates.sort(key=lambda pair: (pair[0], -len(pair[1])), reverse=True) + return candidates[0][1] + + return None + + def _dialogue_turn_bonus( text: str, *, @@ -375,6 +442,30 @@ def _extract_date_spans(sentence: str) -> list[str]: return spans +def _display_date(span: str) -> str: + """Render ISO-style dates as compact human-readable dates.""" + value = span.strip() + match = re.fullmatch(r"(\d{4})-(\d{2})-(\d{2})(?:[T ][^\s]+)?", value) + if match: + year, month, day = map(int, match.groups()) + months = ( + "January", "February", "March", "April", "May", "June", + "July", "August", "September", "October", "November", "December", + ) + if 1 <= month <= 12 and 1 <= day <= 31: + return f"{day} {months[month - 1]} {year}" + match = re.fullmatch(r"(\d{4})-(\d{2})", value) + if match: + year, month = map(int, match.groups()) + months = ( + "January", "February", "March", "April", "May", "June", + "July", "August", "September", "October", "November", "December", + ) + if 1 <= month <= 12: + return f"{months[month - 1]} {year}" + return value + + def _is_relative_only_date(span: str) -> bool: return bool(_RELATIVE_ONLY_DATE_RE.match(span.strip())) @@ -432,6 +523,7 @@ class _ContextItem: memory_type: str = "note" provenance: dict[str, Any] | None = None score: float = 0.0 + session_date: str | None = None @dataclass(frozen=True) @@ -463,25 +555,381 @@ def _filter_context_text(text: str) -> str: return "\n".join(kept).strip() +def _context_session_date(text: str, provenance: dict[str, Any] | None = None) -> str | None: + provenance = provenance or {} + for key in ("session_date", "event_time", "created_at"): + value = provenance.get(key) + if value: + spans = _extract_date_spans(str(value)) + if spans: + return spans[0] + if str(value).strip(): + return str(value).strip() + for line in text.splitlines()[:8]: + lower = line.strip().lower() + if lower.startswith(("session_date:", "created_at:", "event_time:", "date:")): + value = line.split(":", 1)[1].strip() + spans = _extract_date_spans(value) + return spans[0] if spans else value + return None + + +def _is_list_question(question: str) -> bool: + lower = question.lower() + return bool( + re.search( + r"\bboth\b|\ball\b|\bin\s+what\s+ways\b|" + r"\bwhich\s+(?!(?:is|was|does|did|has|have)\b)[a-z]+s\b|" + r"\bwhat\s+(?!(?:is|was|does|did|has|have)\b)[a-z]+s\b|" + r"\bwhat\s+does\s+.+?\s+(?:offer|provide|include|do\s+to)\b|" + r"\bin\s+what\s+ways\b|" + r"\bwhat\s+do\s+.+?'s\s+[a-z]+s\s+(?:like|enjoy|prefer|do)\b|" + r"\bwhere\s+has\s+.+?\s+(?:camped|traveled|travelled|visited|stayed|lived)\b", + lower, + ) + ) + + +def _clean_place_value(value: str) -> str: + value = value.strip(" ,.;:-") + value = re.sub(r"^(?:the)\s+", "", value, flags=re.I) + value = re.sub( + r"\s+(?:last|next|this)\s+" + r"(?:spring|summer|autumn|fall|winter|year|month|week|weekend|" + r"monday|tuesday|wednesday|thursday|friday|saturday|sunday)\b.*$", + "", + value, + flags=re.I, + ) + value = re.sub(r"\s+(?:a\s+few|several|\d+)\s+(?:days?|weeks?|months?|years?)\s+ago\b.*$", "", value, flags=re.I) + return value.strip(" ,.;:-") + + +def _compact_list_values(question: str, sentence: str) -> list[str]: + """Extract compact candidate values from a relevant sentence. + + This is deliberately schema/generic: it recognizes relation shapes rather + than benchmark entities or known answers. + """ + lower_q = question.lower() + values: list[str] = [] + + # Quoted works/titles are high-precision list values. + if re.search(r"\bbooks?|titles?|movies?|films?|songs?|works?\b", lower_q): + values.extend( + match.group(1).strip() + for match in re.finditer(r'["“]([^"”]{2,90})["”]', sentence) + ) + + # Travel / location histories. + if re.search(r"\b(?:city|cities|place|places|state|states|country|countries|where)\b", lower_q): + for pattern in ( + re.compile( + r"\b(?:visited|went|traveled|travelled|vacationed|camped|stayed|lived)" + r"\s+(?:in|at|to|near|on)?\s*(?:the\s+)?" + r"([a-z][a-z-]+(?:\s+[a-z][a-z-]+){0,2})", + re.I, + ), + re.compile( + r"\b(?:trip|vacation|camping)\s+(?:in|at|to|near|on)\s+" + r"(?:the\s+)?([a-z][a-z-]+(?:\s+[a-z][a-z-]+){0,2})", + re.I, + ), + ): + values.extend( + place + for match in pattern.finditer(sentence) + if (place := _clean_place_value(match.group(1))) + ) + + # Product/service/class offerings: capture coordinated objects after the verb. + if re.search(r"\b(?:offer|provide|include|services?|classes?|training|workshops?)\b", lower_q): + for match in re.finditer( + r"\b(?:offers?|provides?|includes?|has)\s+([^.!?]{3,140})", + sentence, + re.I, + ): + phrase = match.group(1) + phrase = re.split(r"\b(?:because|so that|which|where|when)\b", phrase, maxsplit=1, flags=re.I)[0] + values.extend( + part.strip(" ,.;:-") + for part in re.split(r",|\band\b|\bor\b", phrase, flags=re.I) + if 2 <= len(part.strip()) <= 80 + ) + + # Generic "I do/read/paint/attend X and Y" histories. Use only when the + # question itself asks for a plural/set answer. + if re.search(r"\b(?:what|which)\s+[a-z]+s\b|\bwhat\s+.+?\s+has\b", lower_q): + for match in re.finditer( + r"\b(?:do|does|did|done|read|reads|painted|paints|attended|attends|" + r"participated\s+in|practiced|practises|practices|tried|uses?|enjoys?|likes?)\s+" + r"([^.!?]{2,120})", + sentence, + re.I, + ): + phrase = re.split( + r"\b(?:because|since|when|while|which|that|to\s+help|to\s+make)\b", + match.group(1), + maxsplit=1, + flags=re.I, + )[0] + values.extend( + part.strip(" ,.;:-") + for part in re.split(r",|\band\b|\bor\b", phrase, flags=re.I) + if 2 <= len(part.strip()) <= 70 + ) + + # Generic activity / participation values. + if re.search( + r"\b(?:activities?|hobbies?|partake|destress|de-stress|in what ways|participat|events?)\b", + lower_q, + ): + for pattern in ( + re.compile( + r"\b(?:been|started|kept|enjoys?|likes?|loves?|go|goes|went)\s+" + r"(?:to\s+)?([a-z][a-z-]+ing)\b", + re.I, + ), + re.compile( + r"\bsigned\s+up\s+for\s+(?:a\s+|an\s+)?" + r"([a-z][a-z-]+)(?:\s+class|\s+course|\s+workshop)\b", + re.I, + ), + re.compile( + r"\b(?:attended|joined|participated\s+in|went\s+to)\s+" + r"(?:a\s+|an\s+|the\s+)?([^,.!?]{2,70})", + re.I, + ), + ): + for match in pattern.finditer(sentence): + value = match.group(1).strip(" ,.;:-") + if 2 <= len(value) <= 70: + values.append(value) + + # De-duplicate and discard obvious dialogue scaffolding. + cleaned: list[str] = [] + seen: set[str] = set() + for value in values: + value = re.sub(r"^(?:a|an|the)\s+", "", value.strip(), flags=re.I) + value = re.sub(r"^(?:my|our|his|her|their)\s+", "", value, flags=re.I) + if not value or _GREETING_ONLY_RE.match(value): + continue + if value.lower() in {"it", "them", "this", "that", "things", "stuff"}: + continue + key = value.casefold() + if key in seen: + continue + seen.add(key) + cleaned.append(value) + return cleaned + + +def _stem_shared_token(token: str) -> str: + lower = token.casefold().strip(".,;:!?'\"") + for suffix in ("ing", "ed", "es", "s"): + if len(lower) > len(suffix) + 3 and lower.endswith(suffix): + lower = lower[: -len(suffix)] + break + return lower + + +def _shared_entity_answer( + question: str, + normalized: list[_ContextItem], + max_chars: int, +) -> str | None: + """Find compact values independently supported for every named entity.""" + if not re.search(r"\bboth\b|\bin common\b|\bshared\b|\beach\b", question, re.I): + return None + + entities = sorted(_question_entities(question)) + if len(entities) < 2: + return None + + per_entity_values: dict[str, list[str]] = {entity: [] for entity in entities} + per_entity_tokens: dict[str, Counter[str]] = {entity: Counter() for entity in entities} + q_words = _content_words(question) + + for item in normalized: + for sentence in _split_sentences(item.text): + lower = sentence.casefold() + matched_entities = [entity for entity in entities if entity in lower] + if not matched_entities: + continue + + compact = _compact_list_values(question, sentence) + for entity in matched_entities: + per_entity_values[entity].extend(compact) + + # Generic relation/action fallback for non-list commonality questions. + # Keep only content words not already present in the question/entity. + for token in _content_words(sentence): + stem = _stem_shared_token(token) + if ( + stem + and stem not in {_stem_shared_token(word) for word in q_words} + and stem not in {_stem_shared_token(e) for e in entities} + and len(stem) >= 4 + ): + per_entity_tokens[entity][stem] += 1 + + # Exact compact-value intersection first (cities, titles, services, etc.). + value_sets: list[dict[str, str]] = [] + for entity in entities: + mapping: dict[str, str] = {} + for value in per_entity_values[entity]: + key = " ".join(_stem_shared_token(tok) for tok in _tokens(value)) + if key: + mapping.setdefault(key, value) + value_sets.append(mapping) + + if value_sets and all(value_sets): + common = set(value_sets[0]) + for mapping in value_sets[1:]: + common &= set(mapping) + if common: + values = [value_sets[0][key] for key in sorted(common)] + return _truncate(", ".join(values[:8]), max_chars) + + # Fallback: intersect salient content stems across each person's evidence. + token_sets = [set(counter) for counter in per_entity_tokens.values()] + if not token_sets or not all(token_sets): + return None + shared = set.intersection(*token_sets) + if not shared: + return None + + generic = { + "have", "with", "from", "that", "this", "they", "their", "your", "just", + "really", "about", "been", "want", "like", "love", "great", "good", "also", + "make", "help", "thing", "time", "need", "when", "what", "both", + "last", "next", "this", "year", "month", "week", "spring", "summer", + "autumn", "fall", "winter", "ago", "recent", "recently", + } + ranked = [ + token + for token in shared + if token not in generic and token not in {_stem_shared_token(x) for x in q_words} + ] + if not ranked: + return None + + # Prefer tokens repeatedly attested across people. + ranked.sort( + key=lambda token: sum(per_entity_tokens[e][token] for e in entities), + reverse=True, + ) + best = ranked[0] + # Recover a readable surface form from evidence. + variants: Counter[str] = Counter() + for item in normalized: + for token in re.findall(r"[A-Za-z][A-Za-z'-]{2,}", item.text): + if _stem_shared_token(token) == best: + variants[token] += 1 + surface = variants.most_common(1)[0][0] if variants else best + return _truncate(surface, max_chars) + + +def _list_answer( + question: str, + normalized: list[_ContextItem], + max_chars: int, +) -> tuple[str, bool] | None: + question_words = _content_words(question) + entities = _question_entities(question) + scored_sentences: list[tuple[float, str]] = [] + seen_sentences: set[str] = set() + + for item in normalized: + for sentence in _split_sentences(item.text): + if len(sentence) < 8: + continue + overlap = _overlap_score(question_words, sentence, entities) + lower = sentence.lower() + relation_bonus = 0.0 + if re.search( + r"\b(?:visit|visited|went to|trip to|travel(?:ed|led)? to|" + r"read|painted|attended|participated|camped|vacationed)\b", + lower, + ): + relation_bonus += 0.45 + if re.search( + r"\b(?:offer|offering|provide|provides|classes|workshops|training|services)\b", + lower, + ): + relation_bonus += 0.45 + if overlap < 0.12 and relation_bonus == 0.0: + continue + key = re.sub(r"\s+", " ", sentence.strip()).lower() + if key in seen_sentences: + continue + seen_sentences.add(key) + scored_sentences.append( + ( + overlap + relation_bonus + min(item.score, 1.0) * 0.12, + sentence.strip(), + ) + ) + + if not scored_sentences: + return None + scored_sentences.sort(key=lambda item: item[0], reverse=True) + + values: list[str] = [] + seen_values: set[str] = set() + for _, sentence in scored_sentences[:10]: + for value in _compact_list_values(question, sentence): + key = value.casefold() + if key in seen_values: + continue + seen_values.add(key) + values.append(value) + if len(values) >= 8: + break + if len(values) >= 8: + break + + if values: + compact = ", ".join(values) + return _truncate(compact, max_chars), True + + # Last-resort evidence aggregation when no structured values were extractable. + chosen: list[str] = [] + used = 0 + for _, sentence in scored_sentences[:6]: + if used + len(sentence) > max_chars and chosen: + continue + chosen.append(sentence) + used += len(sentence) + 2 + if len(chosen) >= 2: + break + return (_truncate(" ".join(chosen), max_chars), False) if chosen else None + + def _normalize_contexts( contexts: list[str] | list[dict[str, Any]], ) -> list[_ContextItem]: items: list[_ContextItem] = [] for raw in contexts: if isinstance(raw, str): + session_date = _context_session_date(raw) text = _filter_context_text(raw) if text: - items.append(_ContextItem(text=text)) + items.append(_ContextItem(text=text, session_date=session_date)) continue - text = _filter_context_text(str(raw.get("text") or "")) + raw_text = str(raw.get("text") or "") + provenance = raw.get("provenance") if isinstance(raw.get("provenance"), dict) else {} + session_date = _context_session_date(raw_text, provenance) + text = _filter_context_text(raw_text) if not text: continue items.append( _ContextItem( text=text, memory_type=str(raw.get("memory_type") or "note"), - provenance=raw.get("provenance") if isinstance(raw.get("provenance"), dict) else {}, + provenance=provenance, score=float(raw.get("score") or 0.0), + session_date=session_date, ) ) return items @@ -513,6 +961,11 @@ def _context_metadata_bonus( if provenance.get("hop"): bonus += 0.45 + if provenance.get("conversation_neighbor"): + bonus += 0.28 + if provenance.get("source") == "conversation-window": + bonus += 0.22 + if provenance.get("chain_reachable"): bonus += 0.14 bonus += min(0.28, 0.08 * int(provenance.get("chain_target_hits") or 0)) @@ -573,8 +1026,10 @@ def _is_usable_context(item: _ContextItem) -> bool: return True if item.memory_type in _PREFERRED_MEMORY_TYPES and len(text) >= 8: return True - # Giant raw session dumps drown extractive QA — keep only shorter evidence. - if len(text) > 700 and item.memory_type in {"log", "note"}: + # Structured reasoning works sentence-by-sentence, so moderately long + # session evidence is still useful. Only reject truly oversized raw dumps; + # the context compiler already enforces the global packet budget. + if len(text) > 3000 and item.memory_type in {"log", "note"}: return False if len(text) < 8 and not re.search(r"\b(19|20)\d{2}\b", text): return False @@ -654,6 +1109,11 @@ def _score_candidate( score += 0.9 lower = span.lower() + if _GREETING_ONLY_RE.match(span.strip()): + score -= 1.4 + if _GREETING_ONLY_RE.match(sentence.strip()): + score -= 0.8 + if kind == "yes_no": if any(cue in lower for cue in _NEG_CUES): score += 0.2 @@ -792,6 +1252,10 @@ def synthesize_answer( if not normalized: return "" + precise = _precise_scalar_answer(question, normalized) + if precise: + return _truncate(precise, max_chars) + temporal_bias = _question_temporal_bias(question) kind = _question_kind(question) question_words = _content_words(question) @@ -799,11 +1263,25 @@ def synthesize_answer( occupation_question = _is_occupation_question(question) identity_question = _is_identity_question(question) + list_question = _is_list_question(question) + + shared_answer = _shared_entity_answer(question, normalized, max_chars) + if shared_answer: + return shared_answer + # For "when" questions, prefer contexts that actually contain date spans. if kind == "when": - dated_only = [item for item in normalized if _extract_date_spans(item.text)] - if dated_only: - normalized = dated_only + duration_question = bool( + re.search(r"\bhow long\b|\bhow many\s+(?:years?|months?|weeks?)\b", question, re.I) + ) + temporal_only = [ + item for item in normalized + if _extract_date_spans(item.text) + or _extract_duration_spans(item.text) + or item.session_date + ] + if temporal_only: + normalized = temporal_only if kind == "yes_no": sentences: list[tuple[float, str]] = [] @@ -858,7 +1336,11 @@ def synthesize_answer( if kind == "when": duration_question = bool( - re.search(r"\bhow long\b|\bhow many years\b|\byears? ago\b", question, re.I) + re.search( + r"\bhow long\b|\bhow many\s+(?:years?|months?|weeks?)\b|\byears? ago\b", + question, + re.I, + ) ) # Prefer a date that co-occurs with question entities/content in the same sentence. dated: list[tuple[float, str]] = [] @@ -883,6 +1365,13 @@ def synthesize_answer( spans = spans or _extract_date_spans(sentence) spans = _prefer_absolute_date_spans(spans) for date in spans: + # Duration questions should answer with a duration, not the + # session timestamp that happens to anchor the evidence. + duration_bonus = ( + 1.25 + if duration_question and _extract_duration_spans(date) + else 0.0 + ) # Penalize dates that are just session stamps without topical words. topical = overlap + ( 0.4 if any(w in sentence.lower() for w in question_words) else 0.0 @@ -893,10 +1382,22 @@ def synthesize_answer( # when overlap is otherwise similar. dated.append( ( - meta + topical + abs_bonus + min(len(date), 40) * 0.015, + meta + topical + abs_bonus + duration_bonus + + min(len(date), 40) * 0.015, date, ) ) + if item.session_date and not duration_question: + topical_sentences = [ + sentence for sentence in _split_sentences(item.text) + if _overlap_score(question_words, sentence, entities) >= 0.22 + ] + if topical_sentences: + best_overlap = max( + _overlap_score(question_words, sentence, entities) + for sentence in topical_sentences + ) + dated.append((meta + best_overlap + 0.55, item.session_date)) if dated: dated.sort(key=lambda x: (x[0], len(x[1])), reverse=True) # If the top hit is relative-only, skip down to an absolute one. @@ -904,8 +1405,8 @@ def synthesize_answer( if not _is_relative_only_date(date) or all( _is_relative_only_date(d) for _, d in dated ): - return _truncate(date, max_chars) - return _truncate(dated[0][1], max_chars) + return _truncate(_display_date(date), max_chars) + return _truncate(_display_date(dated[0][1]), max_chars) dates = _prefer_absolute_date_spans( _extract_duration_spans(best.text) or _extract_date_spans(best.text) @@ -913,7 +1414,7 @@ def synthesize_answer( or _extract_date_spans(best.sentence) ) if dates: - return _truncate(dates[0], max_chars) + return _truncate(_display_date(dates[0]), max_chars) # Avoid vague relative answers when no absolute date is available. if re.search( r"\b(?:last|next|this)\s+(?:monday|tuesday|wednesday|thursday|friday|saturday|sunday|week|weekend|month)\b", @@ -925,7 +1426,7 @@ def synthesize_answer( _extract_date_spans(cand.text) or _extract_date_spans(cand.sentence) ) if alt and not _is_relative_only_date(alt[0]): - return _truncate(alt[0], max_chars) + return _truncate(_display_date(alt[0]), max_chars) if kind == "where": locs = _extract_location_spans(best.text) or _extract_location_spans(best.sentence) @@ -955,6 +1456,22 @@ def synthesize_answer( head = re.split(r"\s+for\s+|\s+as\s+", obj, maxsplit=1)[0].strip() return _truncate(head or obj, max_chars) + # Distributed/list questions may require evidence from multiple contexts. + # Only use aggregation when at least two distinct high-relevance sentences exist. + if list_question: + list_result = _list_answer(question, normalized, max_chars) + if list_result: + list_answer, structured_values = list_result + parts = _split_sentences(list_answer) + if structured_values or "," in list_answer or len(parts) >= 2 or any( + cue in list_answer.lower() + for cue in ( + "visited", "trip to", "offer", "provid", + "classes", "workshops", "training", + ) + ): + return list_answer + # Prefer a tight span when it still overlaps the question. answer = _strip_supersession_tail(best.text if best.score >= 0.2 and len(best.text) <= max_chars else best.sentence) if answer: diff --git a/src/ultimate_memory/context_compiler.py b/src/ultimate_memory/context_compiler.py new file mode 100644 index 00000000..9eb64cac --- /dev/null +++ b/src/ultimate_memory/context_compiler.py @@ -0,0 +1,93 @@ +"""Compact retrieved evidence into a diverse, token-efficient context packet.""" +from __future__ import annotations + +import re +from collections import Counter + +_WORD_RE = re.compile(r"[A-Za-z0-9_-]{2,}") + + +def _tokens(text: str) -> set[str]: + return {match.group(0).casefold() for match in _WORD_RE.finditer(text)} + + +def _similarity(left: str, right: str) -> float: + a = _tokens(left) + b = _tokens(right) + if not a or not b: + return 0.0 + return len(a & b) / len(a | b) + + +def _source_bucket(item: dict) -> str: + provenance = item.get("provenance") or {} + session_id = provenance.get("session_id") + if session_id: + return f"session:{session_id}" + source = str(provenance.get("source") or "") + source_path = str(item.get("source_path") or "") + return source_path or source or str(item.get("id") or "unknown") + + +def compile_context_packet( + contexts: list[dict], + *, + max_chars: int = 12000, + max_items: int = 24, + near_duplicate_threshold: float = 0.84, + max_per_source: int = 6, +) -> list[dict]: + """Select high-ranked, diverse evidence under a deterministic size budget. + + The input order is treated as ranking order. Near-duplicates and repeated + copies of the same session are suppressed, but small direct-turn evidence + can coexist with a richer conversation window when they are materially + different. + """ + if not contexts or max_chars <= 0 or max_items <= 0: + return [] + + selected: list[dict] = [] + selected_texts: list[str] = [] + source_counts: Counter[str] = Counter() + used_chars = 0 + + for item in contexts: + text = str(item.get("text") or "").strip() + if not text: + continue + + bucket = _source_bucket(item) + if source_counts[bucket] >= max_per_source: + continue + + if any( + _similarity(text, existing) >= near_duplicate_threshold + for existing in selected_texts + ): + continue + + remaining = max_chars - used_chars + if remaining <= 0: + break + + # Keep evidence boundaries intact whenever possible. Only truncate one + # oversized first item rather than filling the packet with fragments. + if len(text) > remaining: + if selected: + continue + clipped = text[:remaining].rsplit(" ", 1)[0].rstrip() + if not clipped: + continue + item = dict(item) + item["text"] = clipped + text = clipped + + selected.append(item) + selected_texts.append(text) + source_counts[bucket] += 1 + used_chars += len(text) + 2 + if len(selected) >= max_items: + break + + return selected diff --git a/src/ultimate_memory/dates.py b/src/ultimate_memory/dates.py index 79fae955..4a79cb58 100644 --- a/src/ultimate_memory/dates.py +++ b/src/ultimate_memory/dates.py @@ -71,6 +71,23 @@ def format_day_month_year(when: datetime) -> str: return f"{when.day} {when.strftime('%B')} {when.year}" +def format_month_year(when: datetime) -> str: + return f"{when.strftime('%B')} {when.year}" + + +def _shift_month(anchor: datetime, delta: int) -> datetime: + month0 = anchor.year * 12 + (anchor.month - 1) + delta + year, month_index = divmod(month0, 12) + return datetime(year, month_index + 1, 1) + + +def _previous_weekday(anchor: datetime, weekday: int) -> datetime: + days_back = (anchor.weekday() - weekday) % 7 + if days_back == 0: + days_back = 7 + return anchor - timedelta(days=days_back) + + def resolve_relative_dates(text: str, anchor: datetime | None) -> str: """Replace yesterday/today/last year/etc. with absolute dates using *anchor*.""" if anchor is None: @@ -84,11 +101,34 @@ def resolve_relative_dates(text: str, anchor: datetime | None) -> str: (r"\bthis\s+year\b", f"in {anchor.year}"), ( r"\blast\s+month\b", - f"in {format_day_month_year(anchor.replace(day=1) - timedelta(days=1))}", + f"in {format_month_year(_shift_month(anchor, -1))}", ), + (r"\bthis\s+month\b", f"in {format_month_year(anchor)}"), + (r"\bnext\s+month\b", f"in {format_month_year(_shift_month(anchor, 1))}"), + (r"\blast\s+week\b", f"around {format_day_month_year(anchor - timedelta(days=7))}"), + (r"\bthis\s+week\b", f"around {format_day_month_year(anchor)}"), + (r"\bnext\s+week\b", f"around {format_day_month_year(anchor + timedelta(days=7))}"), (r"\ba\s+year\s+ago\b", f"in {anchor.year - 1}"), (r"\btwo\s+years\s+ago\b", f"in {anchor.year - 2}"), ] for pattern, value in replacements: out = re.sub(pattern, value, out, flags=re.IGNORECASE) + + weekday_names = { + "monday": 0, + "tuesday": 1, + "wednesday": 2, + "thursday": 3, + "friday": 4, + "saturday": 5, + "sunday": 6, + } + for name, weekday in weekday_names.items(): + when = _previous_weekday(anchor, weekday) + out = re.sub( + rf"\blast\s+{name}\b", + f"on {format_day_month_year(when)}", + out, + flags=re.IGNORECASE, + ) return out diff --git a/src/ultimate_memory/local_semantic.py b/src/ultimate_memory/local_semantic.py new file mode 100644 index 00000000..e76f705e --- /dev/null +++ b/src/ultimate_memory/local_semantic.py @@ -0,0 +1,95 @@ +"""Optional in-process semantic retrieval for local/fallback mode. + +This reuses the configured FastEmbed model and keeps embeddings in process memory. +It is deliberately optional so normal unit tests and lightweight installs do not +need to download a model. Enable with ULTIMATE_MEMORY_LOCAL_SEMANTIC=1. +""" +from __future__ import annotations + +import math +import os + +from .models import AtomicMemory, SearchResult + + +def enabled() -> bool: + return os.environ.get("ULTIMATE_MEMORY_LOCAL_SEMANTIC", "0").strip().lower() in { + "1", "true", "yes", "on", + } + + +def cosine(left: list[float], right: list[float]) -> float: + if not left or not right or len(left) != len(right): + return 0.0 + dot = sum(a * b for a, b in zip(left, right, strict=True)) + ln = math.sqrt(sum(a * a for a in left)) + rn = math.sqrt(sum(b * b for b in right)) + if ln == 0.0 or rn == 0.0: + return 0.0 + return dot / (ln * rn) + + +class LocalSemanticIndex: + def __init__(self, embed) -> None: + self._embed = embed + self._vectors: dict[str, tuple[str, list[float]]] = {} + + def _ensure(self, atoms: list[AtomicMemory]) -> None: + missing = [ + atom for atom in atoms + if atom.id not in self._vectors or self._vectors[atom.id][0] != atom.content_hash + ] + if not missing: + return + vectors = self._embed([atom.text for atom in missing]) + for atom, vector in zip(missing, vectors, strict=True): + self._vectors[atom.id] = (atom.content_hash, vector) + + def search( + self, + query: str, + atoms: list[AtomicMemory], + *, + limit: int, + ) -> list[SearchResult]: + if not query.strip() or not atoms: + return [] + self._ensure(atoms) + query_vector = self._embed([query])[0] + + scored: list[tuple[float, AtomicMemory]] = [] + for atom in atoms: + cached = self._vectors.get(atom.id) + if cached is None: + continue + score = cosine(query_vector, cached[1]) + scored.append((score, atom)) + scored.sort(key=lambda item: item[0], reverse=True) + + out: list[SearchResult] = [] + for score, atom in scored[:limit]: + out.append( + SearchResult( + id=atom.id, + text=atom.text, + title=f"{atom.memory_type.value}: {atom.text[:72]}", + source_path=f"atom://{atom.id}", + memory_type=atom.memory_type.value, + score=max(0.0, min((score + 1.0) / 2.0, 1.0)), + provenance={ + "source": "local-semantic", + "semantic_score": score, + "salience": atom.salience, + "valid_from": atom.valid_from, + "valid_until": atom.valid_until, + "superseded_by": atom.superseded_by, + "entities": atom.entities, + "project_path": atom.project_path, + "dia_id": atom.metadata.get("dia_id"), + "speaker": atom.metadata.get("speaker"), + "direct_turn": bool(atom.metadata.get("dia_id")), + "claim": atom.metadata.get("claim"), + }, + ) + ) + return out diff --git a/src/ultimate_memory/models.py b/src/ultimate_memory/models.py index 8c0a0dff..846709f6 100644 --- a/src/ultimate_memory/models.py +++ b/src/ultimate_memory/models.py @@ -61,6 +61,18 @@ class ReflectionPayload(BaseModel): open_questions: list[str] = Field(default_factory=list) +class CompiledMemory(BaseModel): + """Evidence-linked atomic memory produced by an optional external compiler.""" + + text: str + memory_type: MemoryType = MemoryType.FACT + entities: list[str] = Field(default_factory=list) + source_dia_ids: list[str] = Field(default_factory=list) + confidence: float = Field(default=0.8, ge=0.0, le=1.0) + valid_from: str | None = None + metadata: dict[str, Any] = Field(default_factory=dict) + + class AtomicMemory(BaseModel): """Typed, bi-temporal, salience-tracked memory unit.""" diff --git a/src/ultimate_memory/planner.py b/src/ultimate_memory/planner.py index 194b7af3..d49b9d2d 100644 --- a/src/ultimate_memory/planner.py +++ b/src/ultimate_memory/planner.py @@ -22,6 +22,8 @@ class QueryPlan(BaseModel): memory_types: list[str] = Field(default_factory=list) expansions: list[str] = Field(default_factory=list) signals: list[str] = Field(default_factory=list) + multi_evidence: bool = False + requires_bridge: bool = False _CAPITALIZED = re.compile(r"\b([A-Z][a-zA-Z0-9_.-]*(?:\s+[A-Z][a-zA-Z0-9_.-]*){0,3})\b") @@ -37,7 +39,8 @@ class QueryPlan(BaseModel): r"\bhow\s+many\s+(?:years?|months?|weeks?|days?|hours?)\b|" r"\b(?:years?|months?|weeks?|days?|hours?)\s+ago\b|" r"\b(?:since|until|during)\b|" - r"\b(?:before|after|earlier|later|previously|formerly|prior)\b" + r"\b(?:before|after|earlier|later|previously|formerly|prior)\b|" + r"\b(?:recent|recently|latest|newest|most\s+recent)\b" ) _COLLECTIVE_RE = re.compile( @@ -54,6 +57,19 @@ class QueryPlan(BaseModel): re.I, ) +_LIST_OR_SET_RE = re.compile( + r"\bwhich\s+(?!(?:does|has|is|was|this)\b)(?:[a-z]+s|cities|places|countries|states|books|games|activities|items|things|ways|types|kinds)\b|" + r"\bwhat\s+(?!(?:does|has|is|was|this)\b)(?:[a-z]+s|cities|places|countries|states|books|games|activities|items|things|ways|types|kinds)\b|" + r"\bwhat\s+does\s+.+?\s+(?:offer|provide|include|do\s+to)\b|" + r"\b(?:all|multiple|several)\s+(?:[a-z]+s|cities|places|books|activities|items|things|ways|types)\b|" + r"\bwhere\s+has\s+.+?\s+(?:camped|traveled|travelled|visited|stayed|lived)\b|" + r"\bin\s+what\s+ways\b|" + r"\bhow\s+(?:does|is|has)\s+.+?\s+(?:participat|involv|contribut)\w*\b|" + r"\bwhat\s+do\s+.+?'s\s+[a-z]+s\s+(?:like|enjoy|prefer|do)\b|" + r"\bwhat\s+.+?\s+has\s+.+?\s+(?:done|read|visited|attended|participated|painted|tried|used)\b", + re.I, +) + def _is_temporal_question(question: str) -> bool: return bool(_TEMPORAL_QUESTION_RE.search(question.lower())) @@ -61,7 +77,13 @@ def _is_temporal_question(question: str) -> bool: def _is_collective_multi_hop(question: str, entities: list[str]) -> bool: lower = question.lower() - if _COLLECTIVE_RE.search(lower): + if _COLLECTIVE_RE.search(lower) or _LIST_OR_SET_RE.search(lower): + return True + if re.search( + r"\bhow\s+long\b.*\b(?:take|took|until|from|between|before|after)\b|" + r"\b(?:duration|elapsed|time\s+between)\b", + lower, + ): return True if len(entities) >= 2 and re.search(r"\b(?:and|versus|vs\.?|compared?\s+to)\b", lower): return True @@ -85,6 +107,20 @@ def _entities(question: str) -> list[str]: return found[:8] +def _requires_bridge(question: str) -> bool: + """Whether answering requires traversing an unnamed relationship chain.""" + lower = question.lower() + possessives = len(re.findall(r"\b[\w.-]+'s\b", question)) + relation_hits = sum( + 1 for word in _RELATION_WORDS + if re.search(rf"\b{re.escape(word)}\b", lower) + ) + chained_of = len( + re.findall(r"\bof\s+(?:the\s+)?(?:\w+\s+){0,2}(?:of|for|at)\b", lower) + ) + return possessives >= 2 or chained_of > 0 or (possessives >= 1 and relation_hits >= 1) + + def _hop_depth(question: str, entities: list[str] | None = None) -> int: lower = question.lower() entities = entities or _entities(question) @@ -104,7 +140,7 @@ def _hop_depth(question: str, entities: list[str] | None = None) -> int: def _memory_types(question: str) -> list[str]: lower = question.lower() types: list[str] = [] - if re.search(r"\bprefer|preference|always|never|style|likes?\b", lower): + if re.search(r"\bprefer|preference|favorite|favourite|always|never|style|likes?\b", lower): types.append("preference") if re.search(r"\bdecision|decide|decided|chose|chosen|why did we|why was\b", lower): types.append("decision") @@ -123,16 +159,34 @@ def _expansions(question: str) -> list[str]: expansions.append("work job employer role") if re.search(r"\blive|lives|location|based|where\b", lower): expansions.append("location lives based moved") - if re.search(r"\bprefer|preference|likes?\b", lower): - expansions.append("preference prefer likes") + if re.search(r"\bprefer|preference|favorite|favourite|likes?\b", lower): + expansions.append("preference prefer favorite favourite likes") if re.search(r"\bdecision|decide|chose|chosen|why\b", lower): expansions.append("decision chose reason rationale") if re.search(r"\bhow do|how to|steps?|procedure|process\b", lower): expansions.append("procedure steps process") if re.search(r"\bbefore|previous|formerly|used to|prior\b", lower): expansions.append("previous formerly before historical") + if re.search(r"\bvisit|visited|trip|travel|cities|places\b", lower): + expansions.append("visited travel trip city place") + if re.search(r"\boffer|offers|offering|provide|provides|services\b", lower): + expansions.append("offer provides services classes workshops training") + if re.search(r"\brelationship|dating|married|single|partner\b", lower): + expansions.append("relationship status dating married single partner") + if re.search(r"\bidentity|gender|transgender|nonbinary|non-binary\b", lower): + expansions.append("identity gender transgender nonbinary") + if re.search(r"\bcareer|profession|education|educaton|field|study|degree\b", lower): + expansions.append("career education study degree certification profession") + if re.search(r"\bactivities?|hobbies?|destress|de-stress|relax|leisure\b", lower): + expansions.append("activity hobby recreation leisure destress relax") + if re.search(r"\bevents?|participat|community|involvement|attend|joined?\b", lower): + expansions.append("event attended participated joined involvement community") + if re.search(r"\bpaint|painting|artwork|art\b", lower): + expansions.append("painting artwork canvas subject landscape") if _is_temporal_question(question): expansions.append("date year month day when duration time") + if re.search(r"\bhow\s+long\b|\bduration\b|\belapsed\b", lower): + expansions.append("started began finished completed opened duration elapsed") return list(dict.fromkeys(expansions)) @@ -150,7 +204,10 @@ def plan_query(question: str, *, as_of: str | None = None) -> QueryPlan: temporal_mode = "historical" include_superseded = True signals.append("historical_language") - elif re.search(r"\b(now|current|currently|today|latest)\b", lower): + elif re.search( + r"\b(now|current|currently|today|latest|recent|recently|newest|most\s+recent)\b", + lower, + ): temporal_mode = "current" include_superseded = False signals.append("current_language") @@ -159,12 +216,14 @@ def plan_query(question: str, *, as_of: str | None = None) -> QueryPlan: include_superseded = False entities = _entities(question) + multi_evidence = _is_collective_multi_hop(question, entities) + requires_bridge = _requires_bridge(question) depth = _hop_depth(question, entities) temporal_question = _is_temporal_question(question) if depth > 1: kind = "multi_hop" signals.append(f"relation_chain_depth_{depth}") - elif temporal_mode in {"historical", "as_of"} or temporal_question: + elif temporal_mode != "unspecified" or temporal_question: kind = "temporal" else: kind = "single_hop" @@ -178,4 +237,6 @@ def plan_query(question: str, *, as_of: str | None = None) -> QueryPlan: memory_types=_memory_types(question), expansions=_expansions(question), signals=signals, + multi_evidence=multi_evidence, + requires_bridge=requires_bridge, ) diff --git a/src/ultimate_memory/ranking.py b/src/ultimate_memory/ranking.py index f8bc258b..56f5ff8d 100644 --- a/src/ultimate_memory/ranking.py +++ b/src/ultimate_memory/ranking.py @@ -2,6 +2,7 @@ from __future__ import annotations import re +from datetime import datetime from .models import SearchResult from .planner import QueryPlan @@ -15,6 +16,180 @@ def _tokens(text: str) -> set[str]: } +def _provenance_value(provenance: dict, key: str): + """Read a provenance field across direct, metadata, and vector payload forms.""" + value = provenance.get(key) + if value not in (None, "", [], {}): + return value + metadata = provenance.get("metadata") + if isinstance(metadata, dict): + value = metadata.get(key) + if value not in (None, "", [], {}): + return value + payload = provenance.get("payload") + if isinstance(payload, dict): + value = payload.get(key) + if value not in (None, "", [], {}): + return value + return None + + +def filter_entity_scoped_results( + results: list[dict], + entities: list[str], + *, + min_matches: int = 2, +) -> list[dict]: + """Prefer evidence *attributed to* entities named in the query. + + Conversational memory frequently contains windows with two speakers. A name + appearing in the question half of a pair is weaker evidence than a direct + turn or the answer half of that pair. This gate therefore separates strong + attribution from mere mentions before balancing multiple named entities. + """ + entity_keys = [entity.strip().casefold() for entity in entities if entity.strip()] + if not entity_keys or not results: + return list(results) + + known_speakers: set[str] = set() + for item in results: + provenance = item.get("provenance") or {} + for key in ("speaker", "answer_speaker"): + speaker = str(_provenance_value(provenance, key) or "").strip().casefold() + if speaker: + known_speakers.add(speaker) + + speaker_keys = [key for key in entity_keys if key in known_speakers] + scope_keys = speaker_keys or entity_keys + + strong: list[dict] = [] + weak: list[dict] = [] + strong_buckets: dict[str, list[dict]] = {key: [] for key in scope_keys} + weak_buckets: dict[str, list[dict]] = {key: [] for key in scope_keys} + + for item in results: + text = str(item.get("text") or "").casefold() + provenance = item.get("provenance") or {} + prov_entities = { + str(entity).strip().casefold() + for entity in (_provenance_value(provenance, "entities") or []) + if str(entity).strip() + } + speaker = str(_provenance_value(provenance, "speaker") or "").strip().casefold() + answer_speaker = str( + _provenance_value(provenance, "answer_speaker") or "" + ).strip().casefold() + question_speaker = str( + _provenance_value(provenance, "question_speaker") or "" + ).strip().casefold() + + strong_entities: list[str] = [] + weak_entities: list[str] = [] + for key in scope_keys: + if key == speaker or key == answer_speaker or key in prov_entities: + strong_entities.append(key) + elif key in text or key == question_speaker: + weak_entities.append(key) + + if strong_entities: + strong.append(item) + for key in strong_entities: + strong_buckets[key].append(item) + elif weak_entities: + weak.append(item) + for key in weak_entities: + weak_buckets[key].append(item) + + # Strongly attributed evidence is ordered first, but weaker mention / + # question-side evidence is retained as recall support. Hard-dropping it can + # lose conversational premises whose answer lives in an adjacent turn. + matched = list(strong) + seen = { + str(item.get("id") or item.get("source_path") or id(item)) + for item in matched + } + for item in weak: + item_key = str(item.get("id") or item.get("source_path") or id(item)) + if item_key not in seen: + matched.append(item) + seen.add(item_key) + + buckets = { + key: [*strong_buckets[key], *weak_buckets[key]] + for key in scope_keys + } + + if len(matched) < min_matches: + return list(results) + + if len(scope_keys) == 1 or not all(buckets[key] for key in scope_keys): + return matched + + # Round-robin per-entity evidence so one person's larger history cannot + # crowd another named person out of the context packet. + balanced: list[dict] = [] + seen: set[str] = set() + max_len = max(len(bucket) for bucket in buckets.values()) + for index in range(max_len): + for key in scope_keys: + bucket = buckets[key] + if index >= len(bucket): + continue + item = bucket[index] + item_key = str(item.get("id") or item.get("source_path") or id(item)) + if item_key in seen: + continue + seen.add(item_key) + balanced.append(item) + + for item in matched: + item_key = str(item.get("id") or item.get("source_path") or id(item)) + if item_key in seen: + continue + seen.add(item_key) + balanced.append(item) + return balanced + + + +_NUMBER_WORDS = { + "one": "1", "two": "2", "three": "3", "four": "4", "five": "5", + "six": "6", "seven": "7", "eight": "8", "nine": "9", "ten": "10", + "eleven": "11", "twelve": "12", +} + + +def _duration_quantities(text: str) -> set[tuple[str, str]]: + out: set[tuple[str, str]] = set() + pattern = re.compile( + r"\b(\d+|one|two|three|four|five|six|seven|eight|nine|ten|eleven|twelve)" + r"\s+(years?|months?|weeks?|days?|hours?)\b", + re.I, + ) + for match in pattern.finditer(text): + raw = match.group(1).lower() + number = _NUMBER_WORDS.get(raw, raw) + unit = match.group(2).lower().rstrip("s") + out.add((number, unit)) + return out + + + +def _event_timestamp(result: SearchResult) -> float | None: + provenance = result.provenance or {} + raw = provenance.get("event_time") or provenance.get("created_at") + if not raw: + metadata = provenance.get("metadata") + if isinstance(metadata, dict): + raw = metadata.get("event_time") or metadata.get("created_at") + if not raw: + return None + try: + return datetime.fromisoformat(str(raw).replace("Z", "+00:00")).timestamp() + except (ValueError, TypeError): + return None + + def rerank_candidates( query: str, results: list[SearchResult], @@ -22,6 +197,10 @@ def rerank_candidates( ) -> list[SearchResult]: q_tokens = _tokens(query) wanted_types = set(plan.memory_types) + query_quantities = _duration_quantities(query) + timestamps = [ts for result in results if (ts := _event_timestamp(result)) is not None] + newest_timestamp = max(timestamps) if timestamps else None + oldest_timestamp = min(timestamps) if timestamps else None for result in results: score = float(result.score) @@ -35,6 +214,18 @@ def rerank_candidates( entity_hits = sum(1 for entity in plan.entities if entity.lower() in text_lower) score += min(0.18, 0.07 * entity_hits) + if query_quantities: + result_quantities = _duration_quantities(result.text) + if result_quantities: + if query_quantities & result_quantities: + score += 0.18 + elif any( + q_unit == r_unit + for _, q_unit in query_quantities + for _, r_unit in result_quantities + ): + score -= 0.18 + if result.memory_type in wanted_types: score += 0.08 @@ -43,8 +234,32 @@ def rerank_candidates( if isinstance(claim, dict): score += 0.04 * float(claim.get("confidence") or 0.0) - if plan.temporal_mode == "current" and provenance.get("valid_until"): - score -= 0.35 + # Direct dialogue/tool observations are primary evidence. Derived reflections + # remain useful, but should not outrank an equally relevant original turn. + if provenance.get("direct_turn") or provenance.get("dia_id"): + direct_overlap = len(q_tokens & r_tokens) / len(q_tokens) if q_tokens else 0.0 + score += 0.12 + 0.18 * direct_overlap + + if provenance.get("compiled"): + confidence = float(provenance.get("compiler_confidence") or 0.0) + score += min(0.10, max(0.0, confidence) * 0.10) + + if "auto-extracted from" in text_lower: + score -= 0.08 + + if plan.temporal_mode == "current": + if provenance.get("valid_until"): + score -= 0.35 + event_ts = _event_timestamp(result) + if ( + event_ts is not None + and newest_timestamp is not None + and oldest_timestamp is not None + and newest_timestamp > oldest_timestamp + ): + recency = (event_ts - oldest_timestamp) / (newest_timestamp - oldest_timestamp) + score += 0.22 * recency + provenance["recency_score"] = round(recency, 6) elif plan.temporal_mode in {"historical", "as_of"} and provenance.get("valid_until"): score += 0.08 diff --git a/src/ultimate_memory/reader.py b/src/ultimate_memory/reader.py index bd7f72f2..952973ba 100644 --- a/src/ultimate_memory/reader.py +++ b/src/ultimate_memory/reader.py @@ -22,6 +22,20 @@ r"^(?:is|are|was|were|do|does|did|has|have|had|can|could|would|will|should|may|might)\b", re.I, ) +_TEMPORAL = re.compile( + r"\bwhen\b|\bwhat\s+(?:date|year|month|day)\b|" + r"\bhow\s+long\b|\bhow\s+many\s+(?:years?|months?|weeks?|days?)\b|" + r"\b(?:recent|recently|latest|newest|most\s+recent)\b", + re.I, +) +_DISTRIBUTED = re.compile( + r"\bboth\b|\ball\b|" + r"\b(?:what|which)\s+(?!(?:does|has|is|was|this)\b)[a-z]+s\b" + r".*\b(?:has|have|did|does|are|were)\b|" + r"\bwhere\s+has\s+.+?\s+(?:camped|traveled|travelled|visited|stayed|lived)\b|" + r"\bwhat\s+does\s+.+?\s+(?:offer|provide|include|do\s+to)\b", + re.I, +) def reader_model_name() -> str: @@ -65,10 +79,17 @@ def answer(self, question: str, contexts: list[str] | list[dict[str, Any]]) -> s if not trimmed: return "" - # Extractive SQuAD readers cannot faithfully synthesize yes/no answers. - # Keep the deterministic fallback for that answer shape. - if _YES_NO.match(question.strip()): - return synthesize_answer(question, contexts) + # Span readers cannot faithfully synthesize yes/no, temporal metadata, + # or answers distributed across multiple memories. Route those shapes + # through the deterministic structured reasoner first. + if ( + _YES_NO.match(question.strip()) + or _TEMPORAL.search(question) + or _DISTRIBUTED.search(question) + ): + structured = synthesize_answer(question, contexts) + if structured: + return structured self._ensure_loaded() payloads = [{"question": question, "context": text} for text in trimmed] diff --git a/src/ultimate_memory/router.py b/src/ultimate_memory/router.py index 658946dd..a8a7fd0d 100644 --- a/src/ultimate_memory/router.py +++ b/src/ultimate_memory/router.py @@ -1,6 +1,7 @@ from __future__ import annotations import concurrent.futures +import json import logging import re from datetime import UTC, datetime @@ -23,6 +24,7 @@ from .chain import rank_evidence_chain from .claims import ensure_claim_metadata, structured_conflict_score from .config import Settings, load_settings +from .context_compiler import compile_context_packet from .dates import parse_loose_date, resolve_relative_dates from .extraction import ( extract_from_transcript, @@ -33,6 +35,7 @@ from .models import ( AtomicMemory, AuditEvent, + CompiledMemory, MemoryChunk, MemoryType, ReflectionPayload, @@ -49,8 +52,9 @@ normalize_entity, ) from .llm_answer import use_llm_from_env +from .local_semantic import LocalSemanticIndex, enabled as local_semantic_enabled from .planner import plan_query -from .ranking import rerank_candidates +from .ranking import filter_entity_scoped_results, rerank_candidates from .store import LocalStore logger = logging.getLogger(__name__) @@ -88,6 +92,7 @@ def __init__(self, settings: Settings | None = None) -> None: self.graph = GraphAdapter(self.settings.neo4j) self._vector_ready_cache = False self._graph_ready_cache = False + self._local_semantic = LocalSemanticIndex(self.vector.embed) @property def _vector_ready(self) -> bool: @@ -174,7 +179,13 @@ def answer( if token.lower() not in stop and len(token) > 2 ] dense_query = " ".join(dense_terms[:10]) or question - search_queries = list(dict.fromkeys([dense_query, question, *plan.expansions])) + expanded_queries = [ + f"{dense_query} {expansion}".strip() + for expansion in plan.expansions + ] + search_queries = list( + dict.fromkeys([dense_query, question, *expanded_queries]) + ) rich_contexts: list[dict] = [] search_result: dict = {"results": []} @@ -201,13 +212,85 @@ def answer( ) rich_contexts.extend(item for item in typed["results"] if item.get("text")) + # Direct and temporal questions should stay attached to the named + # subject. Multi-hop questions deliberately skip this hard gate because + # they must traverse bridge entities. + if not plan.requires_bridge and plan.entities: + rich_contexts = filter_entity_scoped_results( + rich_contexts, + plan.entities, + min_matches=2, + ) + + # Conversation adjacency is evidence: a retrieved question/request turn + # is frequently answered by the immediately preceding or following turn. + neighbor_seen: set[str] = { + str(item.get("id") or item.get("source_path") or "") + for item in rich_contexts + } + for anchor in list(rich_contexts[: max(limit, 8)]): + provenance = anchor.get("provenance") or {} + session_id = str(provenance.get("session_id") or "") + dia_id = str(provenance.get("dia_id") or "") + if not session_id or not dia_id: + continue + for neighbor in self.store.neighboring_turn_atoms( + session_id, + dia_id, + radius=1, + project_path=project_path, + ): + if neighbor.id in neighbor_seen: + continue + neighbor_result = self._atom_to_search_result( + neighbor, + score=max(float(anchor.get("score") or 0.0) - 0.04, 0.45), + ).model_dump() + nprov = dict(neighbor_result.get("provenance") or {}) + nprov["conversation_neighbor"] = True + nprov["neighbor_of"] = dia_id + neighbor_result["provenance"] = nprov + rich_contexts.append(neighbor_result) + neighbor_seen.add(neighbor.id) + + anchor_text = str(anchor.get("text") or "").strip() + if "?" in anchor_text or "?" in neighbor.text: + window_id = f"window:{session_id}:{dia_id}:{neighbor.metadata.get('dia_id', '')}" + if window_id not in neighbor_seen: + rich_contexts.append( + { + "id": window_id, + "title": "conversation evidence window", + "text": f"{anchor_text}\n{neighbor.text}", + "source_path": window_id, + "memory_type": "fact", + "score": min( + float(anchor.get("score") or 0.0) + 0.03, + 1.25, + ), + "provenance": { + "source": "conversation-window", + "conversation_neighbor": True, + "session_id": session_id, + "anchor_dia_id": dia_id, + "neighbor_dia_id": neighbor.metadata.get("dia_id"), + "project_path": project_path, + }, + } + ) + neighbor_seen.add(window_id) + initial_results = search_result.get("results") or [] seen_entity_keys = {normalize_entity(entity) for entity in plan.entities} - frontier = extract_hop_entities( - question, - initial_results, - exclude=None, - strict=plan.hop_depth > 1, + frontier = ( + extract_hop_entities( + question, + initial_results, + exclude=None, + strict=True, + ) + if plan.requires_bridge + else [] ) hop_entities: list[str] = list(frontier) hop_searches: list[dict] = [] @@ -271,11 +354,27 @@ def answer( depth += 1 rich_contexts = merge_contexts(rich_contexts, []) - if plan.kind == "multi_hop": + if plan.requires_bridge: rich_contexts = rank_evidence_chain(question, rich_contexts) else: rich_contexts.sort(key=lambda item: float(item.get("score") or 0.0), reverse=True) - rich_contexts = rich_contexts[: max(limit * 3, 18)] + + if plan.multi_evidence and not plan.requires_bridge: + context_budget = 20000 + max_per_source = 3 + elif plan.requires_bridge: + context_budget = 18000 + max_per_source = 8 + else: + context_budget = 12000 + max_per_source = 5 + + rich_contexts = compile_context_packet( + rich_contexts, + max_chars=context_budget, + max_items=max(limit * 3, 18), + max_per_source=max_per_source, + ) use_local_llm = use_llm_from_env() if use_llm is None else use_llm context_texts = [str(item.get("text") or "") for item in rich_contexts if item.get("text")] @@ -309,6 +408,8 @@ def answer( "answer": answer_text, "f1_text": f1_ready_text(answer_text), "contexts_used": context_texts, + "context_chars": sum(len(text) for text in context_texts), + "context_items": len(context_texts), "search": search_result, "query_plan": plan.model_dump(), "hop_entities": list(dict.fromkeys(hop_entities)), @@ -329,6 +430,7 @@ def search( ) -> dict: actual_limit = limit or self.settings.retrieval.default_limit self.store.refresh_salience(limit=200) + query_plan = plan_query(query, as_of=as_of) temporal_query = as_of is None and is_temporal_query(query) if temporal_query: @@ -346,19 +448,38 @@ def _vector(): ) if self._vector_ready else [] def _keyword(): - rows = self.store.keyword_search(query, actual_limit) - return [ - SearchResult( - id=row["id"], - title=row["title"], - text=row["text"], - source_path=row["source_path"], - memory_type=row["memory_type"], - score=self._keyword_score(query, row["text"]), - provenance={"source": "sqlite-fts"}, + rows = self.store.keyword_search( + query, + actual_limit, + project_path=project_path, + ) + output: list[SearchResult] = [] + for row in rows: + try: + metadata = json.loads(row.get("metadata_json") or "{}") + except (TypeError, json.JSONDecodeError): + metadata = {} + output.append( + SearchResult( + id=row["id"], + title=row["title"], + text=row["text"], + source_path=row["source_path"], + memory_type=row["memory_type"], + score=self._keyword_score(query, row["text"]), + provenance={ + "source": "sqlite-fts", + "metadata": metadata, + "session_id": metadata.get("session_id"), + "question_speaker": metadata.get("question_speaker"), + "answer_speaker": metadata.get("answer_speaker"), + "pair": bool(metadata.get("pair")), + "created_at": row.get("created_at"), + "project_path": row.get("project_path"), + }, + ) ) - for row in rows - ] + return output def _basic(): return self.basic.search(query, actual_limit, include_cli=False) @@ -377,6 +498,26 @@ def _atoms(): def _graph(): return self.graph.query(query, depth=1) if self._graph_ready else [] + def _local_semantic(): + if self._vector_ready or not local_semantic_enabled(): + return [] + atoms = self.store.list_active_atoms_for_entities( + query_plan.entities, + project_path=project_path, + limit=max(actual_limit * 12, 120) + if query_plan.entities + else max(actual_limit * 20, 160), + as_of=as_of, + ) + if memory_types: + allowed = set(memory_types) + atoms = [atom for atom in atoms if atom.memory_type.value in allowed] + return self._local_semantic.search( + query, + atoms, + limit=max(actual_limit * 2, 20), + ) + with concurrent.futures.ThreadPoolExecutor(max_workers=3) as pool: fv = pool.submit(_vector) fb = pool.submit(_basic) @@ -387,14 +528,28 @@ def _graph(): vector_results: list[SearchResult] = fv.result() bm_results: list[SearchResult] = fb.result() graph_hits: list[dict] = fg.result() + semantic_results = _local_semantic() results = self._rrf_rank( [vector_results, keyword_results, bm_results, atom_results], memory_types=memory_types, limit=actual_limit * 2, ) + + # Local semantic retrieval is a recall supplement, not an equal RRF voter. + # This preserves strong lexical/direct evidence while still allowing + # paraphrased candidates to enter the final generic reranker. + existing_keys = {result.source_path or result.id for result in results} + for semantic in semantic_results: + key = semantic.source_path or semantic.id + if key in existing_keys: + continue + semantic.score *= 0.82 + semantic.provenance["supplemental_semantic"] = True + results.append(semantic) + existing_keys.add(key) + results = self._apply_salience_rerank(results) - query_plan = plan_query(query, as_of=as_of) results = rerank_candidates(query, results, query_plan)[:actual_limit] touched = [ @@ -415,6 +570,7 @@ def _graph(): "results": [result.model_dump() for result in results], "graph_hits": graph_hits[:5], "atoms_considered": len(atom_results), + "semantic_hits": len(semantic_results), "query_plan": query_plan.model_dump(), } @@ -607,6 +763,86 @@ def ingest_log( "reflection": reflection_result, } + def ingest_compiled_memories( + self, + memories: list[CompiledMemory], + *, + session_id: str, + project_path: str | None = None, + event_time: str | None = None, + compiler: str = "external", + ) -> dict: + """Commit externally compiled atomic memories while preserving evidence links. + + The compiler is optional; raw logs/turns remain the source of truth. This + method exists so local or hosted models can improve proposition quality + without coupling the core memory engine to any one LLM provider. + """ + created = 0 + duplicates = 0 + superseded = 0 + atom_ids: list[str] = [] + + for index, item in enumerate(memories): + text = item.text.strip() + if not text: + continue + source_refs = [ + f"session:{session_id}:dia:{dia_id}" + for dia_id in item.source_dia_ids + if dia_id + ] or [f"session:{session_id}"] + metadata = { + **item.metadata, + "compiled": True, + "compiler": compiler, + "compiler_confidence": item.confidence, + "session_id": session_id, + "source_dia_ids": item.source_dia_ids, + } + atom = AtomicMemory( + id=f"atom:compiled:{safe_slug(session_id, 'session')}:{index}", + text=text, + memory_type=item.memory_type, + project_path=project_path, + entities=list(dict.fromkeys(item.entities)), + source_refs=source_refs, + valid_from=item.valid_from or event_time or now_iso(), + event_time=event_time, + importance=min(1.0, max(0.45, item.confidence)), + metadata=metadata, + ) + outcome = self._ingest_atom(atom) + status = outcome.get("status") + if status == "created": + created += 1 + atom_ids.append(atom.id) + elif status == "duplicate": + duplicates += 1 + superseded += len(outcome.get("superseded") or []) + + self.store.write_audit( + AuditEvent( + action="ingest_compiled_memories", + payload={ + "session_id": session_id, + "compiler": compiler, + "received": len(memories), + "created": created, + "duplicates": duplicates, + "superseded": superseded, + }, + source_refs=[f"session:{session_id}"], + ) + ) + return { + "received": len(memories), + "created": created, + "duplicates": duplicates, + "superseded": superseded, + "atom_ids": atom_ids, + } + def reflect( self, payload: ReflectionPayload, @@ -907,9 +1143,11 @@ def _dialogue_turn_memory( stamp = event_time or now_iso() chunks: list[MemoryChunk] = [] atoms: list[AtomicMemory] = [] + resolved_turns: list[tuple[object, str]] = [] for turn in turns: resolved = resolve_relative_dates(turn.utterance, anchor) + resolved_turns.append((turn, resolved)) line_text = f"[{turn.dia_id}] {turn.speaker}: {resolved}" informative = len(resolved.strip()) >= 20 chunks.append( @@ -949,6 +1187,55 @@ def _dialogue_turn_memory( ) ) + # Index adjacent conversational pairs so question/request terms and + # their response live in the same searchable evidence unit. This improves + # discourse-aware recall without inventing or summarizing any facts. + for index in range(len(resolved_turns) - 1): + first, first_text = resolved_turns[index] + second, second_text = resolved_turns[index + 1] + first_dia = getattr(first, "dia_id", "") + second_dia = getattr(second, "dia_id", "") + first_speaker = getattr(first, "speaker", "") + second_speaker = getattr(second, "speaker", "") + is_query_like = ( + "?" in first_text + or bool( + re.search( + r"^(?:can|could|would|will|please|tell|show|explain|describe|" + r"what|when|where|who|why|how)\b", + first_text.strip(), + re.I, + ) + ) + ) + if not is_query_like: + continue + pair_text = ( + f"[{first_dia}] {first_speaker}: {first_text}\n" + f"[{second_dia}] {second_speaker}: {second_text}" + ) + chunks.append( + MemoryChunk( + id=f"pair:{safe_session}:{first_dia}:{second_dia}", + text=pair_text, + source_path=str(log_path), + title=f"{first_speaker} → {second_speaker} [{first_dia}/{second_dia}]", + memory_type=MemoryType.FACT, + project_path=project_path, + tags=[*tags, "conversation-pair"], + metadata={ + "client": client, + "session_id": session_id, + "pair": True, + "question_dia_id": first_dia, + "answer_dia_id": second_dia, + "question_speaker": first_speaker, + "answer_speaker": second_speaker, + "event_time": event_time, + }, + ) + ) + return chunks, atoms def _commit_atoms_from_reflection( @@ -1123,6 +1410,14 @@ def _atom_to_search_result(atom: AtomicMemory, *, score: float) -> SearchResult: "entities": atom.entities, "project_path": atom.project_path, "claim": atom.metadata.get("claim"), + "compiled": bool(atom.metadata.get("compiled")), + "compiler_confidence": atom.metadata.get("compiler_confidence"), + "source_dia_ids": atom.metadata.get("source_dia_ids"), + "dia_id": atom.metadata.get("dia_id"), + "speaker": atom.metadata.get("speaker"), + "session_id": atom.metadata.get("session_id"), + "direct_turn": bool(atom.metadata.get("dia_id")), + "metadata": atom.metadata, }, ) diff --git a/src/ultimate_memory/store.py b/src/ultimate_memory/store.py index 556d5e62..57a6047f 100644 --- a/src/ultimate_memory/store.py +++ b/src/ultimate_memory/store.py @@ -89,10 +89,18 @@ def _init_db(self) -> None: memory_type text not null, text text not null, metadata_json text not null, + project_path text, created_at text not null ) """ ) + columns = { + row["name"] + for row in conn.execute("pragma table_info(memory_index)").fetchall() + } + if "project_path" not in columns: + conn.execute("alter table memory_index add column project_path text") + conn.execute( """ create virtual table if not exists memory_fts using fts5( @@ -170,20 +178,22 @@ def upsert_chunk( text: str, metadata: dict, created_at: str, + project_path: str | None = None, ) -> None: conn = self._connect() with conn: conn.execute( """ insert into memory_index - (id, source_path, title, memory_type, text, metadata_json, created_at) - values (?, ?, ?, ?, ?, ?, ?) + (id, source_path, title, memory_type, text, metadata_json, project_path, created_at) + values (?, ?, ?, ?, ?, ?, ?, ?) on conflict(id) do update set source_path=excluded.source_path, title=excluded.title, memory_type=excluded.memory_type, text=excluded.text, metadata_json=excluded.metadata_json, + project_path=excluded.project_path, created_at=excluded.created_at """, ( @@ -193,6 +203,7 @@ def upsert_chunk( memory_type, text, json.dumps(metadata, ensure_ascii=True), + project_path, created_at, ), ) @@ -205,31 +216,58 @@ def upsert_chunk( (chunk_id, title, text, source_path, memory_type), ) - def keyword_search(self, query: str, limit: int = 8) -> list[dict]: + def keyword_search( + self, + query: str, + limit: int = 8, + *, + project_path: str | None = None, + ) -> list[dict]: if not query.strip(): return [] + + project_clause = "" + params: list[object] = [query] + if project_path: + project_clause = "and (i.project_path = ? or i.project_path is null or i.project_path = '')" + params.append(project_path) + params.append(limit) + with self._connect() as conn: try: rows = conn.execute( - """ - select id, title, text, source_path, memory_type, - bm25(memory_fts) as score - from memory_fts + f""" + select f.id, f.title, f.text, f.source_path, f.memory_type, + bm25(memory_fts) as score, + i.metadata_json, i.created_at, i.project_path + from memory_fts f + join memory_index i on i.id = f.id where memory_fts match ? + {project_clause} order by score limit ? """, - (query, limit), + params, ).fetchall() except sqlite3.OperationalError: + like_params: list[object] = [f"%{query}%", f"%{query}%"] + fallback_clause = "" + if project_path: + fallback_clause = ( + "and (project_path = ? or project_path is null or project_path = '')" + ) + like_params.append(project_path) + like_params.append(limit) rows = conn.execute( - """ - select id, title, text, source_path, memory_type, 0.0 as score - from memory_fts - where title like ? or text like ? + f""" + select id, title, text, source_path, memory_type, 0.0 as score, + metadata_json, created_at, project_path + from memory_index + where (title like ? or text like ?) + {fallback_clause} limit ? """, - (f"%{query}%", f"%{query}%", limit), + like_params, ).fetchall() return [dict(row) for row in rows] @@ -311,6 +349,69 @@ def get_atom(self, atom_id: str) -> AtomicMemory | None: ).fetchone() return self._row_to_atom(row) if row else None + def neighboring_turn_atoms( + self, + session_id: str, + dia_id: str, + *, + radius: int = 1, + project_path: str | None = None, + ) -> list[AtomicMemory]: + """Return nearby direct-turn atoms from the same conversation session.""" + match = re.fullmatch(r"D(\d+):(\d+)", dia_id.strip(), re.I) + if not session_id or not match or radius < 1: + return [] + dialogue_no = int(match.group(1)) + turn_no = int(match.group(2)) + + clauses = [ + "valid_until is null", + "superseded_by is null", + "id like 'atom:turn:%'", + ] + params: list[object] = [] + if project_path: + clauses.append("(project_path = ? or project_path is null or project_path = '')") + params.append(project_path) + + with self._connect() as conn: + try: + rows = conn.execute( + f""" + select * from memory_atoms + where {' and '.join(clauses)} + and json_extract(metadata_json, '$.session_id') = ? + order by created_at, id + limit 200 + """, + [*params, session_id], + ).fetchall() + except sqlite3.OperationalError: + rows = conn.execute( + f""" + select * from memory_atoms + where {' and '.join(clauses)} + and metadata_json like ? + order by created_at, id + limit 200 + """, + [*params, f'%"session_id": "{session_id}"%'], + ).fetchall() + + neighbors: list[tuple[int, AtomicMemory]] = [] + for row in rows: + atom = self._row_to_atom(row) + candidate_id = str(atom.metadata.get("dia_id") or "") + candidate = re.fullmatch(r"D(\d+):(\d+)", candidate_id, re.I) + if not candidate or int(candidate.group(1)) != dialogue_no: + continue + candidate_turn = int(candidate.group(2)) + distance = abs(candidate_turn - turn_no) + if 0 < distance <= radius: + neighbors.append((candidate_turn, atom)) + neighbors.sort(key=lambda item: item[0]) + return [atom for _, atom in neighbors] + def list_active_atoms( self, *, @@ -343,6 +444,59 @@ def list_active_atoms( rows = conn.execute(sql, params).fetchall() return [self._row_to_atom(row) for row in rows] + def list_active_atoms_for_entities( + self, + entities: list[str], + *, + project_path: str | None = None, + limit: int = 240, + as_of: str | None = None, + ) -> list[AtomicMemory]: + """Return active atoms mentioning any requested entity. + + This is used to bound local semantic search without requiring a graph or + external vector database. + """ + cleaned = [entity.strip().casefold() for entity in entities if entity.strip()] + if not cleaned: + return self.list_active_atoms( + project_path=project_path, + limit=limit, + as_of=as_of, + ) + + if as_of: + clauses = ["valid_from <= ?", "(valid_until is null or valid_until > ?)"] + params: list[object] = [as_of, as_of] + else: + clauses = ["valid_until is null", "superseded_by is null"] + params = [] + if project_path: + clauses.append("(project_path = ? or project_path is null or project_path = '')") + params.append(project_path) + + entity_clauses: list[str] = [] + for entity in cleaned: + entity_clauses.append( + "(lower(entities_json) like ? or lower(text) like ?)" + ) + pattern = f"%{entity}%" + params.extend([pattern, pattern]) + clauses.append("(" + " or ".join(entity_clauses) + ")") + params.append(limit) + + with self._connect() as conn: + rows = conn.execute( + f""" + select * from memory_atoms + where {' and '.join(clauses)} + order by salience desc, created_at desc + limit ? + """, + params, + ).fetchall() + return [self._row_to_atom(row) for row in rows] + def search_atoms( self, query: str, diff --git a/tests/test_answer.py b/tests/test_answer.py index 4bb6dd27..39848854 100644 --- a/tests/test_answer.py +++ b/tests/test_answer.py @@ -229,3 +229,239 @@ def test_answer_wraps_search_and_synthesis(self, tmp_path): assert result["search"]["results"] f1 = tokenize_f1(result["answer"], "May 2023") assert f1 >= 0.5 + + +def test_when_can_use_relevant_session_frontmatter_date(): + contexts = [ + { + "text": "---\nsession_date: 4 February, 2023\n---\n[D4:3] Jon: My group is performing at the festival this month.", + "memory_type": "log", + "score": 0.9, + } + ] + answer = synthesize_answer("When is Jon's group performing at a festival?", contexts) + assert "February" in answer and "2023" in answer + + +def test_relative_multi_year_date_is_extractable(): + contexts = [{"text": "Gina: I got my tattoo a few years ago.", "score": 1.0}] + assert synthesize_answer("When did Gina get her tattoo?", contexts).lower() == "a few years ago" + + +def test_list_question_combines_distributed_evidence(): + contexts = [ + {"text": "Jon: I visited Paris last winter.", "score": 0.9}, + {"text": "Jon: I took a trip to Rome this summer.", "score": 0.8}, + ] + answer = synthesize_answer("Which cities has Jon visited?", contexts) + assert "Paris" in answer + assert "Rome" in answer + + +def test_word_number_duration_is_preferred(): + contexts = [ + {"text": "Caroline: I've had this group of friends for four years now.", "score": 0.9}, + {"text": "---\nsession_date: 7 May 2023\n---\nCaroline: We met up recently.", "score": 0.5}, + ] + answer = synthesize_answer("How long has Caroline had this group of friends?", contexts) + assert "four years" in answer.lower() + + +def test_greeting_only_candidate_is_penalized(): + contexts = [ + {"text": "Gina: Wow!", "score": 1.0}, + {"text": "Gina: Dance feels magical to me.", "score": 0.8}, + ] + answer = synthesize_answer("How does Gina describe the feeling that dance brings?", contexts) + assert "magical" in answer.lower() + + +def test_answer_expands_to_adjacent_conversation_turn(tmp_path): + router = MemoryRouter(make_settings(tmp_path)) + transcript = """--- +session_date: 10 May 2023 +--- +[D1:1] Melanie: What pet do you have? +[D1:2] Caroline: I have a guinea pig named Clover. +[D1:3] Melanie: That sounds adorable. +""" + router.ingest_log( + client="test", + session_id="adjacency", + transcript_or_path=transcript, + project_path="/project", + ) + result = router.answer("What pet does Caroline have?", project_path="/project", limit=6) + assert any( + (item.get("provenance") or {}).get("conversation_neighbor") + for item in result["search"].get("results", []) + ) is False # neighbors are answer-context expansion, not base search output + assert "guinea pig" in " ".join(result["contexts_used"]).lower() + + +def test_list_synthesis_returns_compact_locations(): + contexts = [ + {"text": "Jon: I visited Paris last winter.", "score": 0.9}, + {"text": "Jon: I traveled to Rome this summer.", "score": 0.8}, + ] + answer = synthesize_answer("Which cities has Jon visited?", contexts) + assert "Paris" in answer and "Rome" in answer + assert len(answer) < 80 + + +def test_list_synthesis_extracts_quoted_titles(): + contexts = [ + {"text": 'Alex: I read "Dune" last month.', "score": 0.9}, + {"text": 'Alex: I also read "The Hobbit" this year.', "score": 0.8}, + ] + answer = synthesize_answer("What books has Alex read?", contexts) + assert "Dune" in answer and "The Hobbit" in answer + assert len(answer) < 80 + + +def test_ingestion_indexes_question_response_pair(tmp_path): + router = MemoryRouter(make_settings(tmp_path)) + transcript = """--- +session_date: 10 May 2023 +--- +[D1:1] Melanie: What pet do you have? +[D1:2] Caroline: I have a guinea pig named Clover. +""" + router.ingest_log( + client="test", + session_id="pair-index", + transcript_or_path=transcript, + project_path="/project", + ) + result = router.search("pet Caroline", project_path="/project", limit=10) + pair_hits = [ + item for item in result["results"] + if "What pet do you have?" in item["text"] and "guinea pig" in item["text"] + ] + assert pair_hits + + +def test_shared_city_intersection_across_people(): + contexts = [ + {"text": "Jean: I visited Rome last spring.", "score": 0.9}, + {"text": "Jean: I also visited Paris.", "score": 0.8}, + {"text": "John: I traveled to Rome last year.", "score": 0.9}, + {"text": "John: I visited Berlin too.", "score": 0.8}, + ] + answer = synthesize_answer("Which city have both Jean and John visited?", contexts) + assert "Rome" in answer + assert "Paris" not in answer + assert "Berlin" not in answer + + +def test_shared_activity_intersection_uses_entity_evidence(): + contexts = [ + {"text": "Jon: I dance whenever I need to destress.", "score": 0.9}, + {"text": "Gina: Dancing helps me relax after stressful days.", "score": 0.9}, + ] + answer = synthesize_answer("How do Jon and Gina both like to destress?", contexts) + assert "danc" in answer.lower() + + +def test_explicit_identity_label_beats_related_identity_sentence(): + contexts = [ + {"text": "Caroline: Painting helps me explore my identity and be true to myself.", "score": 1.0}, + {"text": "Caroline: I'm a transgender woman and coming out changed my life.", "score": 0.7}, + ] + answer = synthesize_answer("What is Caroline's identity?", contexts) + assert answer.lower() == "transgender woman" + + +def test_favorite_value_is_extracted_compactly(): + contexts = [ + {"text": "Gina: My favorite style of dance is Contemporary.", "score": 0.8}, + {"text": "Gina: Dance is a huge part of my life.", "score": 1.0}, + ] + answer = synthesize_answer("What is Gina's favorite style of dance?", contexts) + assert answer.lower() == "contemporary" + + +def test_generic_activity_list_synthesis(): + contexts = [ + {"text": "Melanie: I've been running farther to de-stress.", "score": 0.9}, + {"text": "Melanie: I signed up for a pottery class because it feels therapeutic.", "score": 0.8}, + ] + answer = synthesize_answer("What does Melanie do to destress?", contexts) + assert "running" in answer.lower() + assert "pottery" in answer.lower() + + +def test_generic_event_participation_synthesis(): + contexts = [ + {"text": "Caroline: I attended a pride parade downtown.", "score": 0.9}, + {"text": "Caroline: I went to a support group last week.", "score": 0.8}, + ] + answer = synthesize_answer("What events has Caroline participated in?", contexts) + assert "pride parade" in answer.lower() + assert "support group" in answer.lower() + + +def test_transgender_topic_does_not_imply_identity_question(): + contexts = [ + {"text": "Caroline is a transgender woman.", "score": 0.5}, + {"text": "Caroline: I'm going to a transgender conference in July 2023.", "score": 0.9}, + ] + answer = synthesize_answer("When is Caroline going to the transgender conference?", contexts) + assert "July" in answer and "2023" in answer + + +def test_duration_beats_session_date_for_how_long_question(): + contexts = [ + { + "text": "---\nsession_date: 13 September 2023\n---\nCaroline: I've had this group of friends for 4 years.", + "score": 0.9, + } + ] + answer = synthesize_answer("How long has Caroline had this group of friends for?", contexts) + assert answer.lower() == "4 years" + + +def test_location_list_handles_lowercase_places_and_strips_time_modifiers(): + contexts = [ + {"text": "Melanie: We camped at the beach last summer.", "score": 0.9}, + {"text": "Melanie: We camped in the forest this spring.", "score": 0.8}, + ] + answer = synthesize_answer("Where has Melanie camped?", contexts) + assert "beach" in answer.lower() + assert "forest" in answer.lower() + assert "last summer" not in answer.lower() + assert "this spring" not in answer.lower() + + +def test_shared_city_cleanup_keeps_city_not_time_modifier(): + contexts = [ + {"text": "Jean: I visited Rome last spring.", "score": 0.9}, + {"text": "John: I traveled to Rome last year.", "score": 0.9}, + ] + answer = synthesize_answer("Which city have both Jean and John visited?", contexts) + assert answer.lower() == "rome" + + +def test_structured_reasoner_keeps_relevant_sentence_in_moderate_log(): + filler = "Unrelated small talk. " * 45 + contexts = [ + { + "text": filler + " Caroline: I've known this group of friends for 4 years.", + "memory_type": "log", + "score": 0.7, + } + ] + answer = synthesize_answer("How long has Caroline had this group of friends?", contexts) + assert answer.lower() == "4 years" + + +def test_single_structured_list_value_is_trusted(): + contexts = [ + { + "text": "Melanie: Been running longer since our last chat - a great way to destress.", + "score": 0.9, + }, + {"text": "Melanie: Thanks, Caroline!", "score": 1.0}, + ] + answer = synthesize_answer("What does Melanie do to destress?", contexts) + assert answer.lower() == "running" diff --git a/tests/test_atoms.py b/tests/test_atoms.py index f8e426e3..ff030d2f 100644 --- a/tests/test_atoms.py +++ b/tests/test_atoms.py @@ -24,7 +24,7 @@ RetrievalConfig, Settings, ) -from ultimate_memory.models import AtomicMemory, MemoryType, ReflectionPayload +from ultimate_memory.models import AtomicMemory, CompiledMemory, MemoryType, ReflectionPayload from ultimate_memory.router import MemoryRouter @@ -289,3 +289,56 @@ def test_group_near_duplicates_helper(self): groups = group_near_duplicates(atoms, threshold=0.5) assert len(groups) == 1 assert {a.id for a in groups[0]} == {"1", "2"} + + +def test_entity_scoped_atom_candidates(tmp_path): + router = MemoryRouter(make_settings(tmp_path)) + router.store.upsert_atom( + AtomicMemory( + text="Caroline keeps a guinea pig named Clover.", + memory_type=MemoryType.FACT, + entities=["Caroline", "Clover"], + project_path="/project", + ) + ) + router.store.upsert_atom( + AtomicMemory( + text="Jon is opening a dance studio.", + memory_type=MemoryType.FACT, + entities=["Jon"], + project_path="/project", + ) + ) + candidates = router.store.list_active_atoms_for_entities( + ["Caroline"], + project_path="/project", + limit=10, + ) + assert any("guinea pig" in atom.text for atom in candidates) + assert all("Jon" not in atom.text for atom in candidates) + + +def test_ingest_compiled_memories_preserves_evidence_links(tmp_path): + router = MemoryRouter(make_settings(tmp_path)) + result = router.ingest_compiled_memories( + [ + CompiledMemory( + text="Caroline has a guinea pig named Clover.", + memory_type=MemoryType.FACT, + entities=["Caroline", "Clover"], + source_dia_ids=["D1:2"], + confidence=0.97, + ) + ], + session_id="session_1", + project_path="/project", + event_time="2023-05-10T00:00:00", + compiler="test-compiler", + ) + assert result["created"] == 1 + atoms = router.list_atoms(query="guinea pig", project_path="/project", limit=5) + atom = atoms["atoms"][0] + assert atom["metadata"]["compiled"] is True + assert atom["metadata"]["compiler"] == "test-compiler" + assert atom["metadata"]["source_dia_ids"] == ["D1:2"] + assert atom["source_refs"] == ["session:session_1:dia:D1:2"] diff --git a/tests/test_context_compiler.py b/tests/test_context_compiler.py new file mode 100644 index 00000000..dfd73661 --- /dev/null +++ b/tests/test_context_compiler.py @@ -0,0 +1,38 @@ +from ultimate_memory.context_compiler import compile_context_packet + + +def item(text, score=1.0, session="s1"): + return { + "id": text[:8], + "text": text, + "score": score, + "provenance": {"session_id": session}, + } + + +def test_near_duplicates_are_removed(): + contexts = [ + item("Caroline has a guinea pig named Clover."), + item("Caroline has a guinea pig named Clover!"), + item("Jon runs a dance studio.", session="s2"), + ] + packet = compile_context_packet(contexts) + assert len(packet) == 2 + + +def test_budget_is_respected(): + contexts = [ + item("alpha " * 20, session="a"), + item("beta " * 20, session="b"), + ] + packet = compile_context_packet(contexts, max_chars=130) + assert sum(len(x["text"]) + 2 for x in packet) <= 132 + + +def test_source_diversity_cap(): + contexts = [ + item(f"distinct fact number {i} with unique token x{i}", session="same") + for i in range(10) + ] + packet = compile_context_packet(contexts, max_per_source=3) + assert len(packet) == 3 diff --git a/tests/test_dates.py b/tests/test_dates.py new file mode 100644 index 00000000..924073a3 --- /dev/null +++ b/tests/test_dates.py @@ -0,0 +1,15 @@ +from ultimate_memory.dates import parse_loose_date, resolve_relative_dates + + +def test_resolve_this_and_next_month(): + anchor = parse_loose_date("15 February 2023") + assert anchor is not None + assert "February 2023" in resolve_relative_dates("this month", anchor) + assert "March 2023" in resolve_relative_dates("next month", anchor) + + +def test_resolve_last_named_weekday(): + anchor = parse_loose_date("20 July 2023") + assert anchor is not None + result = resolve_relative_dates("last Saturday", anchor) + assert "15 July 2023" in result diff --git a/tests/test_entity_scope.py b/tests/test_entity_scope.py new file mode 100644 index 00000000..731be20a --- /dev/null +++ b/tests/test_entity_scope.py @@ -0,0 +1,31 @@ +from ultimate_memory.ranking import filter_entity_scoped_results + + +def result(text, entities=None, speaker=None): + return { + "text": text, + "provenance": { + "entities": entities or [], + "speaker": speaker, + }, + } + + +def test_entity_scope_removes_other_people_when_enough_matches_exist(): + items = [ + result("Caroline lives in Boston.", ["Caroline"], "Caroline"), + result("Caroline likes painting.", ["Caroline"], "Caroline"), + result("Melanie likes pottery.", ["Melanie"], "Melanie"), + ] + scoped = filter_entity_scoped_results(items, ["Caroline"]) + assert len(scoped) == 2 + assert all("Melanie" not in item["text"] for item in scoped) + + +def test_entity_scope_falls_back_when_pool_is_too_small(): + items = [ + result("Caroline likes painting.", ["Caroline"], "Caroline"), + result("Melanie likes pottery.", ["Melanie"], "Melanie"), + ] + scoped = filter_entity_scoped_results(items, ["Caroline"], min_matches=2) + assert scoped == items diff --git a/tests/test_local_semantic.py b/tests/test_local_semantic.py new file mode 100644 index 00000000..aba18105 --- /dev/null +++ b/tests/test_local_semantic.py @@ -0,0 +1,28 @@ +from ultimate_memory.local_semantic import LocalSemanticIndex, cosine +from ultimate_memory.models import AtomicMemory, MemoryType + + +def fake_embed(texts): + vectors = [] + for text in texts: + lower = text.lower() + vectors.append([ + 1.0 if "dance" in lower or "studio" in lower else 0.0, + 1.0 if "database" in lower or "postgres" in lower else 0.0, + ]) + return vectors + + +def test_cosine_identity(): + assert cosine([1.0, 0.0], [1.0, 0.0]) == 1.0 + + +def test_semantic_index_ranks_related_atom(): + index = LocalSemanticIndex(fake_embed) + atoms = [ + AtomicMemory(text="Project uses PostgreSQL.", memory_type=MemoryType.FACT), + AtomicMemory(text="The dance studio offers classes.", memory_type=MemoryType.FACT), + ] + results = index.search("What does the studio offer?", atoms, limit=2) + assert "dance studio" in results[0].text.lower() + assert results[0].provenance["source"] == "local-semantic" diff --git a/tests/test_planner.py b/tests/test_planner.py index 567666f3..69b8ca08 100644 --- a/tests/test_planner.py +++ b/tests/test_planner.py @@ -45,3 +45,70 @@ def test_collective_both_question_triggers_multi_hop(): def test_two_entity_comparison_triggers_multi_hop(): plan = plan_query("What do Elena and Marcus have in common?") assert plan.kind == "multi_hop" + + +def test_plural_set_query_uses_multi_hop_plan(): + plan = plan_query("Which cities has Jon visited?") + assert plan.kind == "multi_hop" + assert plan.hop_depth >= 2 + + +def test_offer_query_uses_multi_evidence_plan(): + plan = plan_query("What does Jon's dance studio offer?") + assert plan.kind == "multi_hop" + + +def test_generic_plural_history_query_uses_distributed_plan(): + assert plan_query("What martial arts has Alex done?").kind == "multi_hop" + assert plan_query("What books has Alex read?").kind == "multi_hop" + assert plan_query("Where has Alex camped?").kind == "multi_hop" + + +def test_duration_between_events_uses_multi_evidence_plan(): + plan = plan_query("How long did it take Alex to open the studio?") + assert plan.kind == "multi_hop" + assert plan.hop_depth >= 2 + assert any("started" in expansion for expansion in plan.expansions) + + +def test_distributed_list_query_is_not_a_bridge_walk(): + plan = plan_query("What activities does Melanie partake in?") + assert plan.kind == "multi_hop" + assert plan.multi_evidence is True + assert plan.requires_bridge is False + + +def test_shared_named_entities_are_multi_evidence_not_unknown_bridge(): + plan = plan_query("Which city have both Jean and John visited?") + assert plan.multi_evidence is True + assert plan.requires_bridge is False + + +def test_possessive_relation_chain_requires_bridge(): + plan = plan_query("Where does Elena's sister's mentor work?") + assert plan.kind == "multi_hop" + assert plan.requires_bridge is True + + +def test_favorite_question_prefers_preference_memory(): + plan = plan_query("What is Gina's favorite style of dance?") + assert "preference" in plan.memory_types + assert any("favorite" in expansion for expansion in plan.expansions) + + +def test_recent_question_is_current_temporal(): + plan = plan_query("What did Melanie paint recently?") + assert plan.kind == "temporal" + assert plan.temporal_mode == "current" + + +def test_in_what_ways_query_is_multi_evidence_not_bridge(): + plan = plan_query("In what ways is Caroline participating in the LGBTQ community?") + assert plan.multi_evidence is True + assert plan.requires_bridge is False + + +def test_group_preference_query_is_multi_evidence(): + plan = plan_query("What do Melanie's kids like?") + assert plan.multi_evidence is True + assert plan.requires_bridge is False diff --git a/tests/test_ranking.py b/tests/test_ranking.py new file mode 100644 index 00000000..376fdfd2 --- /dev/null +++ b/tests/test_ranking.py @@ -0,0 +1,130 @@ +from ultimate_memory.models import SearchResult +from ultimate_memory.planner import plan_query +from ultimate_memory.ranking import filter_entity_scoped_results, rerank_candidates + + +def result(text: str, score: float = 0.5) -> SearchResult: + return SearchResult( + id=text, + text=text, + title=text, + source_path=text, + memory_type="fact", + score=score, + provenance={}, + ) + + +def test_matching_duration_quantity_beats_conflicting_duration(): + query = "Where did Caroline move from 4 years ago?" + ranked = rerank_candidates( + query, + [ + result("Caroline moved from Sweden four years ago."), + result("Caroline started guitar five years ago."), + ], + plan_query(query), + ) + assert "Sweden" in ranked[0].text + + +def test_recent_query_prefers_newer_event_time(): + query = "What did Melanie paint recently?" + plan = plan_query(query) + old = result("Melanie painted horses.") + old.provenance["event_time"] = "2023-01-01T00:00:00+00:00" + new = result("Melanie painted a sunset.") + new.provenance["event_time"] = "2023-10-01T00:00:00+00:00" + ranked = rerank_candidates(query, [old, new], plan) + assert "sunset" in ranked[0].text + + +def test_multi_entity_filter_balances_named_people(): + results = [ + {"id": "a1", "text": "Jean visited Paris.", "provenance": {}}, + {"id": "a2", "text": "Jean visited Rome.", "provenance": {}}, + {"id": "a3", "text": "Jean likes museums.", "provenance": {}}, + {"id": "b1", "text": "John visited Rome.", "provenance": {}}, + ] + balanced = filter_entity_scoped_results(results, ["Jean", "John"]) + assert balanced[0]["id"] == "a1" + assert balanced[1]["id"] == "b1" + + +def test_speaker_entity_takes_priority_over_capitalized_topic(): + results = [ + { + "id": "caroline", + "text": "Caroline attended a pride event.", + "provenance": {"speaker": "Caroline"}, + }, + { + "id": "melanie", + "text": "Melanie discussed LGBTQ community events.", + "provenance": {"speaker": "Melanie"}, + }, + ] + scoped = filter_entity_scoped_results(results, ["LGBTQ", "Caroline"], min_matches=1) + assert [item["id"] for item in scoped] == ["caroline"] + + +def test_question_speaker_only_pair_is_weak_evidence(): + results = [ + { + "id": "pair", + "text": "Melanie: What books have you read?\nCaroline: I read Dune.", + "provenance": { + "pair": True, + "question_speaker": "Melanie", + "answer_speaker": "Caroline", + }, + }, + { + "id": "direct", + "text": "Melanie: I read The Hobbit last year.", + "provenance": {"speaker": "Melanie", "direct_turn": True}, + }, + ] + scoped = filter_entity_scoped_results(results, ["Melanie"], min_matches=1) + assert scoped[0]["id"] == "direct" + assert {item["id"] for item in scoped} == {"direct", "pair"} + + +def test_pair_answer_speaker_is_strong_evidence(): + results = [ + { + "id": "pair", + "text": "Caroline: What books have you read?\nMelanie: I read The Hobbit.", + "provenance": { + "pair": True, + "question_speaker": "Caroline", + "answer_speaker": "Melanie", + }, + }, + { + "id": "noise", + "text": "Caroline mentioned Melanie while discussing art.", + "provenance": {}, + }, + ] + scoped = filter_entity_scoped_results(results, ["Melanie"], min_matches=1) + assert scoped[0]["id"] == "pair" + + +def test_nested_vector_payload_preserves_answer_attribution(): + results = [ + { + "id": "vector-pair", + "text": "Caroline: What do you do to relax?\nMelanie: I go running.", + "provenance": { + "source": "qdrant", + "payload": { + "pair": True, + "question_speaker": "Caroline", + "answer_speaker": "Melanie", + }, + }, + } + ] + scoped = filter_entity_scoped_results(results, ["Melanie"], min_matches=1) + assert [item["id"] for item in scoped] == ["vector-pair"] diff --git a/tests/test_reader.py b/tests/test_reader.py index 71ea3151..8822eb22 100644 --- a/tests/test_reader.py +++ b/tests/test_reader.py @@ -24,3 +24,26 @@ def test_reader_selects_highest_confidence_span_without_loading_transformers(): ], ) assert answer == "Portland" + + +def test_temporal_reader_uses_structured_reasoner_without_model(): + reader = ExtractiveReader() + contexts = [ + { + "text": "---\nsession_date: 4 February, 2023\n---\nJon: My group is performing at the festival this month.", + "memory_type": "log", + "score": 0.9, + } + ] + answer = reader.answer("When is Jon's group performing at a festival?", contexts) + assert "February" in answer and "2023" in answer + + +def test_distributed_reader_can_combine_multiple_memories_without_model(): + reader = ExtractiveReader() + contexts = [ + {"text": "Jon: I visited Paris last winter.", "score": 0.9}, + {"text": "Jon: I took a trip to Rome this summer.", "score": 0.8}, + ] + answer = reader.answer("Which cities has Jon visited?", contexts) + assert "Paris" in answer and "Rome" in answer