From eec85c2ddf25832b98eca5260ac9e121fe58c49b Mon Sep 17 00:00:00 2001 From: Derek Thomas Date: Thu, 10 Sep 2026 16:32:12 -0400 Subject: [PATCH] MemoryRetriever: write the memory-retrievals.jsonl audit stream CortexHealth already checks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The per-turn retrieval audit stream was designed in ISC-107..112 and documented in MemorySystem.md and ObservabilitySystem.md as "written by MemoryRetriever", with the note "not yet populated as of 2026-05-23; infrastructure ready". The writer never landed. Release 7.40.4 then added CortexHealth.ts, which WARNs when the latest row is missing or older than 24h, so every install has shown a permanent amber memory-health line regardless of how well retrieval works. No test referenced the stream, so nothing could catch a check shipping without its producer. This adds the writer to getRelevantContext(): every uncached exit funnels through one finish() closure that appends a row shaped exactly as CortexHealth.validRetrievalRow requires — {ts, query_hash, returned_count, duration_ms} plus optional top_score. Empty-result paths are recorded too (an empty retrieval is still evidence the retriever ran); cache hits are not (they do no ranking work). The query is stored as a 16-hex sha256 prefix, never raw, since it is the principal's prompt. Writes are best-effort and swallowed: the retriever is on every turn's critical path and must not fail because a log did. Adds `bun MemoryRetriever.ts test`, following the self-test convention MemoryReviewer and seven other TOOLS already use: runs the real hot-path entry point in a child process against a throwaway corpus, then validates the produced rows with the real collectCortexEvidence/assessCortexEvidence. 12 assertions, PASS. Doc line in ObservabilitySystem.md updated from "not yet populated" to the landed behavior. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01MSS7QWGFC7BR6zTBswFw7x --- .../Observability/ObservabilitySystem.md | 2 +- .../install/LIFEOS/TOOLS/MemoryRetriever.ts | 166 ++++++++++++++++-- 2 files changed, 157 insertions(+), 11 deletions(-) diff --git a/LifeOS/install/LIFEOS/DOCUMENTATION/Observability/ObservabilitySystem.md b/LifeOS/install/LIFEOS/DOCUMENTATION/Observability/ObservabilitySystem.md index 7467d6889d..fc3811d784 100644 --- a/LifeOS/install/LIFEOS/DOCUMENTATION/Observability/ObservabilitySystem.md +++ b/LifeOS/install/LIFEOS/DOCUMENTATION/Observability/ObservabilitySystem.md @@ -52,7 +52,7 @@ JSONL Sources (local disk) | Pending proposals (Tier C queue) | `MEMORY/OBSERVABILITY/pending-proposals.jsonl` | — | `MemorySystem.add()` for `type:proposal`. Status lifecycle: pending → sent → accepted/rejected/edited/applied-elsewhere (or auto-applied without surfacing); `TERMINAL_STATUSES` in `LIFEOS/PULSE/lib/memory-proposals.ts` is the one authority on which of those count as resolved. Surfaced by that same lib on the Pulse dashboard and the inline 🧠 MEMORY line; decided via `bun LIFEOS/TOOLS/ProposalDecide.ts`. | | Identity proposals (archive) | `MEMORY/OBSERVABILITY/identity-proposals.jsonl` | — | `LIFEOS/PULSE/lib/memory-proposals.ts` surfacer. Archive of sent/accepted/rejected/edited proposals. | | Proposal replies | `MEMORY/OBSERVABILITY/proposal-replies.jsonl` | — | Pulse dashboard reply handler. Records accept/reject/edit interactions. | -| Memory retrievals | `MEMORY/OBSERVABILITY/memory-retrievals.jsonl` | — | `MemoryRetriever.getRelevantContext()` (ISC-107..112; not yet populated as of 2026-05-23; infrastructure ready). Per-turn BM25 audit. | +| Memory retrievals | `MEMORY/OBSERVABILITY/memory-retrievals.jsonl` | — | `MemoryRetriever.getRelevantContext()` (ISC-107..112; writer landed with the `test` self-check — one row per uncached BM25 run, cache hits not recorded, query stored as a 16-hex sha256 prefix, never raw). Per-turn BM25 audit. | Per-source counts are configured inline in `Pulse/Observability/observability.ts`. diff --git a/LifeOS/install/LIFEOS/TOOLS/MemoryRetriever.ts b/LifeOS/install/LIFEOS/TOOLS/MemoryRetriever.ts index dcc185e985..a929130594 100755 --- a/LifeOS/install/LIFEOS/TOOLS/MemoryRetriever.ts +++ b/LifeOS/install/LIFEOS/TOOLS/MemoryRetriever.ts @@ -34,7 +34,9 @@ for (const __k of ["LIFEOS_DIR", "LIFEOS_CONFIG_DIR", "PROJECTS_DIR"]) { * STORAGE: * Reads MEMORY/KNOWLEDGE/{People,Companies,Ideas,Research}/*.md (class: knowledge) * Reads MEMORY/LEARNING/ ** /*.md recursively (class: learning) - * NEVER writes or modifies files — read-only tool. + * NEVER writes or modifies memory files — read-only over the corpus. + * The one write is an audit row per hot-path retrieval, appended to + * MEMORY/OBSERVABILITY/memory-retrievals.jsonl (see recordRetrieval). * * ============================================================================ */ @@ -44,6 +46,7 @@ import * as fs from "fs"; import * as path from "path"; import { spawnSync } from "child_process"; import { homedir } from "node:os"; +import { createHash } from "node:crypto"; // Normalize env path vars that Claude Code injects without shell expansion (LifeOS#1404) for (const k of ["LIFEOS_DIR", "LIFEOS_CONFIG_DIR", "PROJECTS_DIR"]) { @@ -66,6 +69,17 @@ const DOMAINS = ["People", "Companies", "Ideas", "Research"]; // live in dated subdirs (FAILURES/2026-07//CONTEXT.md). const LEARNING_DIR = path.join(LIFEOS_DIR, "MEMORY", "LEARNING"); +// Per-turn retrieval audit stream. Designed in ISC-107..112 (May 2026) and +// documented in MemorySystem.md + ObservabilitySystem.md as "written by +// MemoryRetriever", but the writer never landed; CortexHealth.ts (7.40.4) +// then began WARNing on its absence, so every install sat permanently amber. +// Row shape is pinned by CortexHealth.validRetrievalRow: exactly +// {ts, query_hash, returned_count, duration_ms} plus optional top_score. +// The raw query is never persisted — it is the principal's prompt — only a +// hash, which is enough to correlate repeated retrievals. +const OBSERVABILITY_DIR = path.join(LIFEOS_DIR, "MEMORY", "OBSERVABILITY"); +const RETRIEVALS_LOG = path.join(OBSERVABILITY_DIR, "memory-retrievals.jsonl"); + // BM25 parameters const BM25_K1 = 1.5; const BM25_B = 0.75; @@ -685,6 +699,29 @@ function discoverAllItems(): KnowledgeNote[] { * excerptChars — max excerpt length per result (default 500) * typeFilter — restrict to one type */ +/** + * Append one audit row for a hot-path retrieval. Best-effort and silent: the + * retriever sits on every turn's critical path, so a full disk or a missing + * OBSERVABILITY dir must never turn into a thrown error or a stderr line. + * Cache hits are deliberately not recorded — the stream audits BM25 runs, + * and a hit does no ranking work. + */ +function recordRetrieval(row: { query_hash: string; returned_count: number; duration_ms: number; top_score?: number }): void { + try { + fs.mkdirSync(OBSERVABILITY_DIR, { recursive: true }); + const line: Record = { + ts: new Date().toISOString(), + query_hash: row.query_hash, + returned_count: row.returned_count, + duration_ms: Math.round(row.duration_ms * 1000) / 1000, + }; + if (row.top_score !== undefined) line.top_score = row.top_score; + fs.appendFileSync(RETRIEVALS_LOG, JSON.stringify(line) + "\n", "utf8"); + } catch { + /* observability must never break retrieval */ + } +} + export function getRelevantContext( query: string, options: { topK?: number; threshold?: number; excerptChars?: number; typeFilter?: string } = {}, @@ -701,18 +738,32 @@ export function getRelevantContext( return { ...cached.result, cached: true }; } + // Every uncached exit below funnels through finish() so the audit row is + // written exactly once per BM25 run, including the empty-result paths — + // an empty retrieval is still evidence the retriever ran this turn. + const startedAt = performance.now(); + const queryHash = createHash("sha256").update(cacheKey).digest("hex").slice(0, 16); + const finish = (result: GetRelevantContextResult, topScore?: number): GetRelevantContextResult => { + relevantCache.set(cacheKey, { ts: Date.now(), result }); + recordRetrieval({ + query_hash: queryHash, + returned_count: result.results.length, + duration_ms: performance.now() - startedAt, + top_score: topScore, + }); + return result; + }; + const empty: GetRelevantContextResult = { results: [], markdownBlock: "", totalSearched: 0, cached: false }; const queryTerms = tokenize(query); if (queryTerms.length === 0) { - relevantCache.set(cacheKey, { ts: Date.now(), result: empty }); - return empty; + return finish(empty); } const allNotes = discoverAllItems(); if (allNotes.length === 0) { - relevantCache.set(cacheKey, { ts: Date.now(), result: { ...empty, totalSearched: 0 } }); - return empty; + return finish({ ...empty, totalSearched: 0 }); } const avgDocLength = allNotes.reduce((sum, n) => sum + n.wordCount, 0) / allNotes.length; @@ -729,9 +780,7 @@ export function getRelevantContext( scored = scored.slice(0, topK); if (scored.length === 0) { - const result = { ...empty, totalSearched: allNotes.length }; - relevantCache.set(cacheKey, { ts: Date.now(), result }); - return result; + return finish({ ...empty, totalSearched: allNotes.length }); } const results: RelevantResultItem[] = scored.map(({ note, score }) => { @@ -745,8 +794,7 @@ export function getRelevantContext( const markdownBlock = formatRelevantBlock(results); const result: GetRelevantContextResult = { results, markdownBlock, totalSearched: allNotes.length, cached: false }; - relevantCache.set(cacheKey, { ts: Date.now(), result }); - return result; + return finish(result, scored[0].score); } /** @@ -778,6 +826,91 @@ export function clearRelevantCache(): void { relevantCache.clear(); } +// ============================================================================ +// Self-test +// ============================================================================ + +/** + * `bun MemoryRetriever.ts test` — hermetic check that the retrieval audit + * stream is actually produced, and produced in the shape CortexHealth + * enforces. Exists because the stream was documented and health-checked for + * months before any writer existed, and no test noticed: the check had a + * fixture-free WARN path and nothing exercised the producer. This closes + * that loop by running the real hot-path entry point against a throwaway + * corpus and validating the result with the real health module. + * + * Runs the probe in a child process so LIFEOS_DIR can be repointed at the + * fixture — the corpus paths are resolved once at module load, by design. + */ +async function selfTest(): Promise { + const os = await import("node:os"); + const { collectCortexEvidence, assessCortexEvidence } = await import("./CortexHealth"); + + const root = fs.mkdtempSync(path.join(os.tmpdir(), "memory-retriever-test-")); + const lifeosDir = path.join(root, "LIFEOS"); + const ideas = path.join(lifeosDir, "MEMORY", "KNOWLEDGE", "Ideas"); + fs.mkdirSync(ideas, { recursive: true }); + fs.writeFileSync( + path.join(ideas, "fixture.md"), + "---\ntype: idea\ntitle: Retrieval audit fixture\ntags: [fixture, audit]\n---\n\n" + + "The retrieval audit fixture exists so the self-test has a note that matches the probe query about audit fixtures.\n", + ); + + let failures = 0; + const fail = (msg: string) => { failures++; console.error(` FAIL ${msg}`); }; + const pass = (msg: string) => console.log(` ok ${msg}`); + + try { + // Three hot-path calls in one process: a hit, the same query again (cache + // hit, must NOT log), and a query with no matches (empty path, MUST log). + const probe = spawnSync("bun", [import.meta.path, "--probe", "retrieval audit fixture"], { + env: { ...process.env, LIFEOS_DIR: lifeosDir }, + encoding: "utf8", + }); + if (probe.status !== 0) { + fail(`probe exited ${probe.status}: ${probe.stderr}`); + return 1; + } + const report = JSON.parse(probe.stdout.trim()) as { hit: number; secondCached: boolean; miss: number }; + report.hit > 0 ? pass(`hit query returned ${report.hit} result(s)`) : fail("hit query returned nothing"); + report.secondCached ? pass("repeat query served from cache") : fail("repeat query was not cached"); + report.miss === 0 ? pass("nonsense query returned nothing") : fail(`nonsense query returned ${report.miss}`); + + const log = path.join(lifeosDir, "MEMORY", "OBSERVABILITY", "memory-retrievals.jsonl"); + if (!fs.existsSync(log)) { fail(`audit log not written: ${log}`); return 1; } + const rows = fs.readFileSync(log, "utf8").trim().split("\n").map((l) => JSON.parse(l)); + rows.length === 2 ? pass("exactly 2 rows for 3 calls (cache hit not recorded)") : fail(`expected 2 rows, got ${rows.length}`); + + for (const [i, row] of rows.entries()) { + const keys = Object.keys(row).sort().join(","); + const okKeys = keys === "duration_ms,query_hash,returned_count,ts" || keys === "duration_ms,query_hash,returned_count,top_score,ts"; + okKeys ? pass(`row ${i} has the pinned key set (${keys})`) : fail(`row ${i} has unexpected keys: ${keys}`); + /^[0-9a-f]{16}$/.test(row.query_hash) ? pass(`row ${i} query_hash is a 16-hex prefix, not the raw query`) : fail(`row ${i} query_hash malformed: ${row.query_hash}`); + } + rows[0].returned_count > 0 && rows[0].top_score !== undefined ? pass("hit row carries returned_count and top_score") : fail("hit row missing count/score"); + rows[1].returned_count === 0 && rows[1].top_score === undefined ? pass("miss row has returned_count 0 and no top_score") : fail("miss row shape wrong"); + + // The real health module must accept what the real writer produced. + const evidence = collectCortexEvidence({ root }); + evidence.retrieval.status === "ok" ? pass(`CortexHealth reads retrieval evidence as "${evidence.retrieval.status}"`) : fail(`CortexHealth retrieval status: ${evidence.retrieval.status}`); + const retrievalFindings = assessCortexEvidence(evidence).findings.filter((f) => f.id.startsWith("retrieval") && f.severity !== "ok"); + retrievalFindings.length === 0 ? pass("no retrieval findings from assessCortexEvidence") : fail(`retrieval findings: ${retrievalFindings.map((f) => f.id).join(", ")}`); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } + + console.log(failures === 0 ? "\nPASS" : `\nFAIL (${failures})`); + return failures === 0 ? 0 : 1; +} + +/** Child side of selfTest(): exercise getRelevantContext three ways, report as JSON. */ +function probe(query: string): void { + const a = getRelevantContext(query, { topK: 5, threshold: 0.2 }); + const b = getRelevantContext(query, { topK: 5, threshold: 0.2 }); + const c = getRelevantContext("zzqxv wvvqz", { topK: 5, threshold: 0.2 }); + console.log(JSON.stringify({ hit: a.results.length, secondCached: b.cached, miss: c.results.length })); +} + // ============================================================================ // Help // ============================================================================ @@ -791,6 +924,7 @@ USAGE: bun MemoryRetriever.ts "query string" --top 5 Return top 5 (default: 3) bun MemoryRetriever.ts "query string" --raw Skip compression, return raw excerpts bun MemoryRetriever.ts "query string" --budget 800 Token budget for output (default: 500) + bun MemoryRetriever.ts test Hermetic self-test of the retrieval audit stream bun MemoryRetriever.ts --help Show this help OPTIONS: @@ -822,6 +956,11 @@ EXAMPLES: // ============================================================================ async function main(): Promise { + // Subcommand dispatch ahead of parseArgs, same shape as MemoryReviewer.ts. + if (process.argv[2] === "test") { + process.exit(await selfTest()); + } + const { values, positionals } = parseArgs({ args: process.argv.slice(2), options: { @@ -829,6 +968,7 @@ async function main(): Promise { raw: { type: "boolean", short: "r", default: false }, budget: { type: "string", short: "b" }, help: { type: "boolean", short: "h", default: false }, + probe: { type: "boolean", default: false }, }, allowPositionals: true, strict: true, @@ -840,6 +980,12 @@ async function main(): Promise { } const query = positionals.join(" ").trim(); + + // Hidden child-process entry used only by `test`; never a user-facing mode. + if (values.probe) { + probe(query); + return; + } if (!query) { console.error("Error: Query string required.\n"); printHelp();