Ultimate Memory v1.1: diagnostics-driven generalization

.github/workflows/full-clean-benchmark.yml

@@ -4,10 +4,16 @@ on:
   pull_request:
   workflow_dispatch:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   clean-benchmark:
     runs-on: ubuntu-latest
     timeout-minutes: 20
+    env:
+      ULTIMATE_MEMORY_LOCAL_SEMANTIC: "1"
     steps:
       - uses: actions/checkout@v4
       - name: Install uv

.github/workflows/memeval.yml

@@ -0,0 +1,92 @@
+name: Independent MemEval
+
+on:
+  pull_request:
+  workflow_dispatch:
+    inputs:
+      samples:
+        description: "Number of LoCoMo conversations"
+        required: true
+        default: "1"
+      with_judge:
+        description: "Run GPT judge in addition to token F1"
+        required: true
+        type: boolean
+        default: false
+
+concurrency:
+  group: independent-memeval
+  cancel-in-progress: true
+
+jobs:
+  memeval:
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    env:
+      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+      LLM_MODEL: gpt-4.1-mini
+      EMBEDDING_MODEL: text-embedding-3-small
+    steps:
+      - name: Check API key
+        id: api_key
+        run: |
+          if [ -z "$OPENAI_API_KEY" ]; then
+            echo "::notice::OPENAI_API_KEY Actions secret is not configured; independent MemEval is skipped."
+            echo "available=false" >> "$GITHUB_OUTPUT"
+          else
+            echo "available=true" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Checkout Ultimate Memory
+        if: steps.api_key.outputs.available == 'true'
+        uses: actions/checkout@v4
+        with:
+          path: ultimate-memory
+
+      - name: Checkout Prosus MemEval
+        if: steps.api_key.outputs.available == 'true'
+        uses: actions/checkout@v4
+        with:
+          repository: ProsusAI/MemEval
+          path: MemEval
+
+      - name: Install uv
+        if: steps.api_key.outputs.available == 'true'
+        uses: astral-sh/setup-uv@v3
+
+      - name: Install MemEval and Ultimate Memory
+        if: steps.api_key.outputs.available == 'true'
+        working-directory: MemEval
+        run: |
+          uv sync
+          uv pip install httpx
+          uv pip install -e ../ultimate-memory
+          cp ../ultimate-memory/integrations/memeval/ultimate_memory.py             src/agents_memory/systems/ultimate_memory.py
+
+      - name: Run fair head-to-head
+        if: steps.api_key.outputs.available == 'true'
+        working-directory: MemEval
+        shell: bash
+        run: |
+          SAMPLES="1"
+          EXTRA="--skip-judge"
+          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
+            SAMPLES="${{ inputs.samples }}"
+            if [ "${{ inputs.with_judge }}" = "true" ]; then
+              EXTRA=""
+            fi
+          fi
+          uv run python scripts/run_full_benchmark.py \
+            --systems ultimate_memory,propmem \
+            --num-samples "$SAMPLES" \
+            --llm-model "$LLM_MODEL" \
+            $EXTRA \
+            --output-dir benchmark-output
+
+      - name: Upload benchmark results
+        if: always() && steps.api_key.outputs.available == 'true'
+        uses: actions/upload-artifact@v4
+        with:
+          name: independent-memeval-results
+          path: MemEval/benchmark-output/
+          if-no-files-found: warn

.github/workflows/reader-benchmark.yml

@@ -4,12 +4,17 @@ on:
   pull_request:
   workflow_dispatch:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   reader-benchmark:
     runs-on: ubuntu-latest
     timeout-minutes: 20
     env:
       TOKENIZERS_PARALLELISM: "false"
+      ULTIMATE_MEMORY_LOCAL_SEMANTIC: "1"
     steps:
       - uses: actions/checkout@v4
       - name: Install uv

.github/workflows/tests.yml

@@ -6,6 +6,10 @@ on:
   pull_request:
     branches: [master]
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   test:
     runs-on: ubuntu-latest

evals/run_clean_benchmarks.py

@@ -156,6 +156,7 @@ def run(
     planner_correct = 0
     planner_confusion: dict[str, dict[str, int]] = defaultdict(lambda: defaultdict(int))
     asked = 0
+    diagnostics: list[dict] = []
     started = time.perf_counter()
 
     for sample in data:
@@ -209,6 +210,19 @@ def run(
             planner_total += 1
             planner_correct += int(planned_kind == expected_kind)
             planner_confusion[expected_kind][planned_kind] += 1
+            diagnostics.append({
+                "sample_id": sample_id,
+                "category": category,
+                "question": qa["question"],
+                "gold": golds,
+                "answer": result["answer"],
+                "token_f1": round(100 * score, 2),
+                "evidence_recall": round(100 * evidence_score, 2),
+                "gold_token_coverage": round(100 * coverage, 2),
+                "planned_kind": planned_kind,
+                "expected_kind": expected_kind,
+                "top_contexts": [str(x)[:320] for x in contexts_used[:4]],
+            })
             asked += 1
 
         if max_questions is not None and asked >= max_questions:
@@ -258,6 +272,17 @@ def run(
         "by_category": by_category,
         "use_llm": use_llm,
         "use_reader": use_reader,
+        "worst_answer_failures": sorted(
+            diagnostics,
+            key=lambda item: (item["token_f1"], -item["gold_token_coverage"]),
+        )[:12],
+        "retrieval_failures": sorted(
+            diagnostics,
+            key=lambda item: (item["gold_token_coverage"], item["evidence_recall"]),
+        )[:12],
+        "planner_misses": [
+            item for item in diagnostics if item["planned_kind"] != item["expected_kind"]
+        ][:12],
     }
 
 

integrations/memeval/README.md

@@ -0,0 +1,54 @@
+# Ultimate Memory on ProsusAI MemEval
+
+ProsusAI/MemEval is the primary external fair-comparison gate for Ultimate Memory.
+It standardizes the answer LLM, embedding model, scoring pipeline, and token accounting
+across memory systems.
+
+At the time this integration was added, MemEval's published LoCoMo leader is PropMem
+at 0.605 token-F1 and 0.823 judge score.
+
+## Run head-to-head
+
+Clone MemEval next to Ultimate Memory, install both projects, and copy the adapter:
+
+```bash
+git clone https://github.com/ProsusAI/MemEval.git
+cd MemEval
+uv sync --all-extras
+uv pip install -e ../ultimate-memory
+cp ../ultimate-memory/integrations/memeval/ultimate_memory.py \
+  src/agents_memory/systems/ultimate_memory.py
+```
+
+Set `OPENAI_API_KEY`, then run one conversation first:
+
+```bash
+uv run python scripts/run_full_benchmark.py \
+  --systems ultimate_memory,propmem \
+  --num-samples 1 \
+  --llm-model gpt-4.1-mini \
+  --skip-judge
+```
+
+For the full externally comparable LoCoMo run:
+
+```bash
+uv run python scripts/run_full_benchmark.py \
+  --systems ultimate_memory,propmem \
+  --num-samples 10 \
+  --llm-model gpt-4.1-mini
+```
+
+Do not claim a leaderboard position from Ultimate Memory's internal clean harness.
+Use the MemEval result for cross-system claims.
+
+## Fairness
+
+The adapter:
+
+- ingests only raw conversation sessions;
+- never receives the gold question category;
+- uses MemEval's supplied answer model;
+- uses `text-embedding-3-small` for semantic fallback;
+- leaves MemEval's scoring and judge code untouched;
+- counts answer-LLM tokens through MemEval's normal OpenAI instrumentation.