From f63934f665ae79c6163f7e2a14545dc9c481c8ee Mon Sep 17 00:00:00 2001 From: Phototonic <35551999+Phototonic@users.noreply.github.com> Date: Fri, 6 Nov 2026 17:33:24 +1100 Subject: [PATCH] fix: preserve capture links in the timeline and markdown vault --- README.md | 4 + src/services/client.ts | 10 +- src/services/markdown/backend.ts | 7 +- src/services/markdown/document.ts | 39 +++- src/services/markdown/reconcile.ts | 34 ++- tests/group-memories.test.ts | 232 ++++++++++++++++++++ tests/markdown-capture-metadata.test.ts | 273 ++++++++++++++++++++++++ tests/markdown-document.test.ts | 35 +++ web/src/lib/group-memories.ts | 47 ++-- 9 files changed, 655 insertions(+), 26 deletions(-) create mode 100644 tests/group-memories.test.ts create mode 100644 tests/markdown-capture-metadata.test.ts diff --git a/README.md b/README.md index 0dc9786f..e5b84d77 100644 --- a/README.md +++ b/README.md @@ -272,6 +272,8 @@ If you have chatted for a while and still see the message, check that auto-captu Open `http://127.0.0.1:4747` to browse the memory–prompt timeline, inspect captures, and manage the user profile. The UI has five views: **Project memories** (`/project-memories`), **User profile** (`/user-profile`), **Skill library** (`/skill-library`), **Markdown vault** (`/markdown-vault`) and **Stats** (`/stats`). Change the port with `webServerPort`, or set `webServerEnabled: false` to run without the web UI. If you bind the server beyond loopback, see [Web UI HTTP Basic Auth](#web-ui-http-basic-auth). +The timeline shows each memory and prompt once. If several capture prompts refer to one memory, its current prompt is paired with it when available on the page; earlier prompts remain visible as separate entries. + ## Usage Examples ```typescript @@ -325,6 +327,8 @@ By default memories live only in Turso under `storagePath`. Optionally set a **g } ``` +Plugin-created vault files preserve capture provenance (`promptId`, `sessionID`, and `captureTimestamp`) so reindexing can restore those fields. Legacy files without them retain any corresponding metadata already in the index. Provenance already missing from both the file and the index cannot be recovered automatically. + | Key | Default | What it does | | --------------- | ------------ | -------------------------------------------------------------------------------------------------------------------------------------------------- | | `memoryBackend` | `"database"` | `"database"` keeps content in Turso. `"markdown"` uses all `.md` files under `markdownPath` as source of truth. | diff --git a/src/services/client.ts b/src/services/client.ts index 28b94f26..78b393be 100644 --- a/src/services/client.ts +++ b/src/services/client.ts @@ -17,6 +17,7 @@ import type { MemoryType } from "../types/index.js"; import type { MemoryListFilters, MemoryRecord } from "./turso/types.js"; import { isMarkdownBackend, writeMemoryMarkdownFile } from "./markdown/backend.js"; import { pluginMemoryRelPath } from "./markdown/vault.js"; +import { readCaptureMetadata } from "./markdown/document.js"; export type MemoryScope = "project" | "all-projects"; @@ -263,6 +264,7 @@ export class LocalMemoryClient { updatedAt: now, source: typeof dynamicMetadata.source === "string" ? dynamicMetadata.source : "manual", title: displayName, + ...readCaptureMetadata(dynamicMetadata), }); dynamicMetadata.markdownRelPath = pluginMemoryRelPath(id); if (!dynamicMetadata.source) dynamicMetadata.source = "manual"; @@ -374,6 +376,7 @@ export class LocalMemoryClient { updatedAt, source: typeof existingMeta.source === "string" ? existingMeta.source : "api", title: existingMemory.display_name ? String(existingMemory.display_name) : undefined, + ...readCaptureMetadata(existingMeta), }); existingMeta.markdownRelPath = pluginMemoryRelPath(id); } @@ -406,11 +409,8 @@ export class LocalMemoryClient { Number(existingMemory.created_at), updatedAt, metadataJson, - existingMemory.session_id - ? String(existingMemory.session_id) - : parseSessionIdFromMetadata( - existingMemory.metadata ? String(existingMemory.metadata) : null - ), + parseSessionIdFromMetadata(metadataJson) ?? + (existingMemory.session_id ? String(existingMemory.session_id) : null), existingMemory.display_name ? String(existingMemory.display_name) : null, existingMemory.user_name ? String(existingMemory.user_name) : null, existingMemory.user_email ? String(existingMemory.user_email) : null, diff --git a/src/services/markdown/backend.ts b/src/services/markdown/backend.ts index 6d888638..d347b2ed 100644 --- a/src/services/markdown/backend.ts +++ b/src/services/markdown/backend.ts @@ -1,7 +1,7 @@ import { mkdirSync } from "node:fs"; import { CONFIG } from "../../config.js"; import { log } from "../logger.js"; -import { packMemoryMarkdown } from "./document.js"; +import { packMemoryMarkdown, type MemoryCaptureMetadata } from "./document.js"; import { getLastReconcileAt, listVaultIndexEntries, @@ -18,7 +18,7 @@ import { } from "./vault.js"; import { markdownVaultWatcher } from "./watcher.js"; -export interface MarkdownMemoryWriteInput { +export interface MarkdownMemoryWriteInput extends MemoryCaptureMetadata { id: string; content: string; type?: string; @@ -63,6 +63,9 @@ export function writeMemoryMarkdownFile(input: MarkdownMemoryWriteInput): string updatedAt: input.updatedAt, source: input.source, title: input.title, + promptId: input.promptId, + sessionID: input.sessionID, + captureTimestamp: input.captureTimestamp, }); markdownVaultWatcher.noteSelfWrite(relPath); writeVaultFile(root, relPath, markdown); diff --git a/src/services/markdown/document.ts b/src/services/markdown/document.ts index f8f913d3..df484b33 100644 --- a/src/services/markdown/document.ts +++ b/src/services/markdown/document.ts @@ -1,7 +1,30 @@ import { createHash } from "node:crypto"; import { basename } from "node:path"; -export interface MemoryMarkdownFrontmatter { +export interface MemoryCaptureMetadata { + promptId?: string; + sessionID?: string; + captureTimestamp?: number; +} + +/** Only persist well-formed capture provenance; other metadata stays in the index. */ +export function readCaptureMetadata(input: Record): MemoryCaptureMetadata { + const capture: MemoryCaptureMetadata = {}; + for (const key of ["promptId", "sessionID"] as const) { + const value = input[key]; + if (typeof value === "string" && value.trim()) capture[key] = value; + } + if ( + typeof input.captureTimestamp === "number" && + Number.isFinite(input.captureTimestamp) && + input.captureTimestamp >= 0 + ) { + capture.captureTimestamp = input.captureTimestamp; + } + return capture; +} + +export interface MemoryMarkdownFrontmatter extends MemoryCaptureMetadata { id?: string; type?: string; tags?: string[]; @@ -20,7 +43,7 @@ export interface ParsedMemoryMarkdown { hasFrontmatter: boolean; } -export interface PackMemoryMarkdownInput { +export interface PackMemoryMarkdownInput extends MemoryCaptureMetadata { id: string; content: string; type?: string; @@ -98,6 +121,12 @@ export function packMemoryMarkdown(input: PackMemoryMarkdownInput): string { if (input.updatedAt !== undefined) lines.push(`updatedAt: ${input.updatedAt}`); if (input.source) lines.push(`source: ${yamlScalar(input.source)}`); if (input.title) lines.push(`title: ${yamlScalar(input.title)}`); + const capture = readCaptureMetadata({ ...input }); + if (capture.promptId) lines.push(`promptId: ${yamlScalar(capture.promptId)}`); + if (capture.sessionID) lines.push(`sessionID: ${yamlScalar(capture.sessionID)}`); + if (capture.captureTimestamp !== undefined) { + lines.push(`captureTimestamp: ${capture.captureTimestamp}`); + } lines.push("---", "", input.content); return lines.join("\n"); } @@ -131,7 +160,11 @@ export function parseMemoryMarkdown(raw: string): ParsedMemoryMarkdown { continue; } const parsed = parseYamlScalar(valueRaw); - frontmatter[key] = parsed; + if (key === "promptId" || key === "sessionID" || key === "captureTimestamp") { + Object.assign(frontmatter, readCaptureMetadata({ [key]: parsed })); + } else { + frontmatter[key] = parsed; + } } return { diff --git a/src/services/markdown/reconcile.ts b/src/services/markdown/reconcile.ts index 91da677e..5c856c03 100644 --- a/src/services/markdown/reconcile.ts +++ b/src/services/markdown/reconcile.ts @@ -12,6 +12,7 @@ import { contentHash, markdownIdFromRelPath, parseMemoryMarkdown, + readCaptureMetadata, titleFromRelPath, } from "./document.js"; import { listMarkdownFiles, readVaultFile, type VaultFileInfo } from "./vault.js"; @@ -43,7 +44,10 @@ export function getLastReconcileAt(): number | null { function parseMetadata(value: unknown): Record { if (!value || typeof value !== "string") return {}; try { - return JSON.parse(value) as Record; + const parsed: unknown = JSON.parse(value); + return parsed && typeof parsed === "object" && !Array.isArray(parsed) + ? (parsed as Record) + : {}; } catch { return {}; } @@ -98,11 +102,11 @@ async function upsertFromFile(file: VaultFileInfo): Promise<"upserted" | "unchan const hash = contentHash(raw); const existing = await findMemoryAcrossShards(id); + const existingMetadata = parseMetadata(existing?.row.metadata); if (existing) { - const meta = parseMetadata(existing.row.metadata); if ( - meta.markdownContentHash === hash && - meta.markdownRelPath === file.relPath && + existingMetadata.markdownContentHash === hash && + existingMetadata.markdownRelPath === file.relPath && String(existing.row.content) === content ) { return "unchanged"; @@ -140,7 +144,15 @@ async function upsertFromFile(file: VaultFileInfo): Promise<"upserted" | "unchan }); } + // Legacy vault files omit capture fields. Keep their surviving provenance and + // other index-only metadata; validated file values take precedence when present. + const capture = { + ...readCaptureMetadata(existingMetadata), + ...readCaptureMetadata(fm), + }; const metadata = JSON.stringify({ + ...existingMetadata, + ...capture, source, markdownRelPath: file.relPath, markdownContentHash: hash, @@ -159,7 +171,18 @@ async function upsertFromFile(file: VaultFileInfo): Promise<"upserted" | "unchan createdAt, updatedAt, metadata, + sessionId: + capture.sessionID ?? + (typeof existing?.row.session_id === "string" ? existing.row.session_id : undefined), displayName: title, + userName: typeof existing?.row.user_name === "string" ? existing.row.user_name : undefined, + userEmail: typeof existing?.row.user_email === "string" ? existing.row.user_email : undefined, + projectPath: + typeof existing?.row.project_path === "string" ? existing.row.project_path : undefined, + projectName: + typeof existing?.row.project_name === "string" ? existing.row.project_name : undefined, + gitRepoUrl: + typeof existing?.row.git_repo_url === "string" ? existing.row.git_repo_url : undefined, }; const { scope, hash: scopeHash } = extractScopeFromContainerTag(containerTag); @@ -181,6 +204,9 @@ async function upsertFromFile(file: VaultFileInfo): Promise<"upserted" | "unchan await tursoShardManager.decrementVectorCount(shard.id); } await tursoVectorSearch.insertVector(db, record); + if (Number(existing?.row.is_pinned) === 1) { + await tursoVectorSearch.pinMemory(db, id); + } await tursoShardManager.incrementVectorCount(shard.id); }); diff --git a/tests/group-memories.test.ts b/tests/group-memories.test.ts new file mode 100644 index 00000000..d5634a16 --- /dev/null +++ b/tests/group-memories.test.ts @@ -0,0 +1,232 @@ +import { describe, expect, test } from "bun:test"; +import { deleteSelectedMemories } from "../web/src/lib/bulk-delete"; +import { groupMemories, partitionPinnedGroups } from "../web/src/lib/group-memories"; +import type { MemoryGroup, MemoryItem } from "../src/shared/api"; + +function memory(id: string, linkedPromptId?: string, extra: Partial = {}): MemoryItem { + return { + id, + type: "memory", + content: id, + createdAt: "2026-01-01T00:00:00Z", + linkedPromptId, + ...extra, + }; +} + +function prompt(id: string, linkedMemoryId?: string, extra: Partial = {}): MemoryItem { + return { + id, + type: "prompt", + content: id, + createdAt: "2026-01-01T00:00:00Z", + linkedMemoryId, + ...extra, + }; +} + +function permutations(items: T[]): T[][] { + if (items.length === 0) return [[]]; + return items.flatMap((item, index) => + permutations(items.filter((_, i) => i !== index)).map((rest) => [item, ...rest]) + ); +} + +function keys(groups: MemoryGroup[]): string[] { + return groups.map((group) => (group.isPair ? group.memory.id : group.item.id)); +} + +function expectPartition(items: MemoryItem[]): MemoryGroup[] { + const groups = groupMemories(items); + const members = groups.flatMap((group) => + group.isPair ? [group.memory.id, group.prompt.id] : [group.item.id] + ); + expect(members.toSorted()).toEqual([...new Set(items.map((item) => item.id))].toSorted()); + expect(new Set(keys(groups)).size).toBe(groups.length); + for (const group of groups) { + if (group.isPair) { + expect(group.memory.type).toBe("memory"); + expect(group.prompt.type).toBe("prompt"); + } + } + return groups; +} + +describe("groupMemories", () => { + test("handles an empty page", () => { + expect(groupMemories([])).toEqual([]); + }); + + test("keeps an ordinary bidirectional pair in either input order", () => { + const items = [memory("mem_1", "prompt_1"), prompt("prompt_1", "mem_1")]; + for (const order of permutations(items)) { + expect(expectPartition(order)).toEqual([ + { isPair: true, memory: items[0], prompt: items[1] }, + ]); + } + }); + + test("prefers the current capture prompt and keeps older prompts visible in every order", () => { + const currentMemory = memory("mem_1", "prompt_new"); + const oldPrompt = prompt("prompt_old", "mem_1"); + const newPrompt = prompt("prompt_new", "mem_1"); + for (const order of permutations([currentMemory, oldPrompt, newPrompt])) { + const groups = expectPartition(order); + expect(groups).toContainEqual({ isPair: true, memory: currentMemory, prompt: newPrompt }); + expect(groups).toContainEqual({ isPair: false, type: "prompt", item: oldPrompt }); + expect(new Set(keys(groups))).toEqual(new Set(["mem_1", "prompt_old"])); + } + }); + + test("pairs a prompt backlink even when a memory without promptId appears first", () => { + const items = [memory("mem_1"), prompt("prompt_1", "mem_1")]; + for (const order of permutations(items)) { + expect(expectPartition(order)).toEqual([ + { isPair: true, memory: items[0], prompt: items[1] }, + ]); + } + }); + + test("pairs a memory forward link even when an unlinked prompt appears first", () => { + const items = [memory("mem_1", "prompt_1"), prompt("prompt_1")]; + for (const order of permutations(items)) { + expect(expectPartition(order)).toEqual([ + { isPair: true, memory: items[0], prompt: items[1] }, + ]); + } + }); + + test("uses a prompt at most once when two memories claim it", () => { + for (const order of permutations([ + memory("mem_1", "prompt_1"), + memory("mem_2", "prompt_1"), + prompt("prompt_1"), + ])) { + expect(expectPartition(order)).toHaveLength(2); + } + }); + + test("preserves every item when several prompts claim a memory without a backlink", () => { + for (const order of permutations([ + memory("mem_1"), + prompt("prompt_1", "mem_1"), + prompt("prompt_2", "mem_1"), + ])) { + expect(expectPartition(order)).toHaveLength(2); + } + }); + + test("handles all page subsets without requiring linked items on that page", () => { + const items = [ + memory("mem_1", "prompt_new"), + prompt("prompt_old", "mem_1"), + prompt("prompt_new", "mem_1"), + ]; + for (let mask = 0; mask < 1 << items.length; mask++) { + for (const order of permutations(items.filter((_, index) => mask & (1 << index)))) { + expectPartition(order); + } + } + }); + + test("uses an available older prompt when the current prompt is outside the page", () => { + const m = memory("mem_1", "prompt_new_off_page"); + const p = prompt("prompt_old", "mem_1"); + for (const order of permutations([m, p])) { + expect(expectPartition(order)).toEqual([{ isPair: true, memory: m, prompt: p }]); + } + }); + + test("grouped bulk selection does not delete a shared memory twice", async () => { + const items = [ + memory("mem_1", "prompt_new"), + prompt("prompt_old", "mem_1"), + prompt("prompt_new", "mem_1"), + ]; + const calls: Array<{ endpoint: string; body: unknown }> = []; + const result = await deleteSelectedMemories( + keys(expectPartition(items)), + async (endpoint, options) => { + calls.push({ endpoint, body: JSON.parse(String(options.body)) }); + return { success: true }; + }, + items + ); + expect(result).toEqual({ success: true, deletedIds: ["prompt_old", "mem_1"] }); + expect(calls).toEqual([ + { endpoint: "/api/prompts/bulk-delete", body: { ids: ["prompt_old"], cascade: true } }, + ]); + }); + + test("leaves missing, self, and wrong-type links as standalone items", () => { + const items = [ + memory("mem_missing", "absent"), + prompt("prompt_missing", "absent"), + memory("mem_self", "mem_self"), + prompt("prompt_self", "prompt_self"), + memory("mem_wrong", "mem_missing"), + prompt("prompt_wrong", "prompt_missing"), + ]; + expect(expectPartition(items).every((group) => !group.isPair)).toBe(true); + }); + + test("deduplicates repeated input IDs", () => { + const m = memory("mem_1", "prompt_1"); + const p = prompt("prompt_1", "mem_1"); + expect(expectPartition([m, p, m, p])).toHaveLength(1); + }); + + test("retains pinned-first and memory-createdAt ordering", () => { + const groups = expectPartition([ + memory("mem_old", undefined, { createdAt: "2024-01-01T00:00:00Z" }), + memory("mem_new", undefined, { createdAt: "2025-01-01T00:00:00Z" }), + memory("mem_pinned", "prompt_pinned", { isPinned: true, createdAt: "2023-01-01T00:00:00Z" }), + prompt("prompt_pinned", "mem_pinned", { createdAt: "2026-01-01T00:00:00Z" }), + ]); + expect(keys(groups)).toEqual(["mem_pinned", "mem_new", "mem_old"]); + const partitioned = partitionPinnedGroups(groups); + expect(keys(partitioned.pinned)).toEqual(["mem_pinned"]); + expect(keys(partitioned.rest)).toEqual(["mem_new", "mem_old"]); + }); + + test("keeps first-appearance order when group dates tie", () => { + const items = [ + memory("mem_solo"), + prompt("prompt_pair", "mem_pair"), + memory("mem_pair", "prompt_pair"), + ]; + expect(keys(expectPartition(items))).toEqual(["mem_solo", "mem_pair"]); + expect(keys(expectPartition(items.toReversed()))).toEqual(["mem_pair", "mem_solo"]); + }); + + test("does not mutate input links or ordering", () => { + const items = [ + memory("mem_1", "prompt_new"), + prompt("prompt_old", "mem_1"), + prompt("prompt_new", "mem_1"), + ]; + const before = structuredClone(items); + items.forEach(Object.freeze); + expectPartition(items); + expect(items).toEqual(before); + }); + + test("partitions every two-memory/two-prompt link combination in every order", () => { + for (const a of [undefined, "prompt_1", "prompt_2"]) { + for (const b of [undefined, "prompt_1", "prompt_2"]) { + for (const c of [undefined, "mem_1", "mem_2"]) { + for (const d of [undefined, "mem_1", "mem_2"]) { + for (const order of permutations([ + memory("mem_1", a), + memory("mem_2", b), + prompt("prompt_1", c), + prompt("prompt_2", d), + ])) { + expectPartition(order); + } + } + } + } + } + }); +}); diff --git a/tests/markdown-capture-metadata.test.ts b/tests/markdown-capture-metadata.test.ts new file mode 100644 index 00000000..46ca04f3 --- /dev/null +++ b/tests/markdown-capture-metadata.test.ts @@ -0,0 +1,273 @@ +import { describe, expect, it } from "bun:test"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +const source = (path: string) => JSON.stringify(new URL(`../src/${path}`, import.meta.url).href); + +// Isolate real client/DB integration from the suite's process-wide module mocks. +function scenario(body: string): void { + const baseDir = mkdtempSync(join(tmpdir(), "markdown-capture-metadata-")); + try { + const script = join(baseDir, "scenario.mjs"); + writeFileSync( + script, + ` +import assert from "node:assert/strict"; +import { readFileSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +const baseDir = ${JSON.stringify(baseDir)}; +const { CONFIG } = await import(${source("config.js")}); +CONFIG.storagePath = join(baseDir, "storage"); +CONFIG.embeddingDimensions = 2; +CONFIG.containerTagPrefix = "opencode"; +CONFIG.memoryBackend = "markdown"; +CONFIG.markdownPath = join(baseDir, "vault"); +const { embeddingService } = await import(${source("services/embedding.js")}); +embeddingService.warmup = async () => {}; +embeddingService.clearCache = () => {}; +embeddingService.embedWithTimeout = async () => new Float32Array([1, 0]); +const { memoryClient } = await import(${source("services/client.js")}); +const { tursoConnectionManager } = await import(${source("services/turso/connection-manager.js")}); +const { tursoShardManager } = await import(${source("services/turso/shard-manager.js")}); +const { tursoVectorSearch } = await import(${source("services/turso/vector-search.js")}); +const { closeTursoAndInvalidateCaches } = await import(${source("services/turso/lifecycle.js")}); +const { reconcileMarkdownVault, reindexVaultFile } = await import(${source("services/markdown/reconcile.js")}); +const { parseMemoryMarkdown } = await import(${source("services/markdown/document.js")}); +const { applyUpdateOnCaptureDecision } = await import(${source("services/update-on-capture.js")}); +const tag = "opencode_project_abcdef1234567890"; +const initialCapture = { promptId: "prompt_initial", sessionID: "session_initial", captureTimestamp: 1234 }; +const auxiliary = { + userName: "Synthetic User", userEmail: "synthetic@example.invalid", projectPath: "/synthetic/project", + projectName: "Synthetic Project", gitRepoUrl: "https://example.invalid/project.git" +}; +async function addCapture() { + const added = await memoryClient.addMemory("Initial captured fact", tag, { + source: "auto-capture", type: "fact", ...initialCapture, ...auxiliary, + customProvenance: { importedFrom: "synthetic" }, updatedByCapture: true + }); + assert.equal(added.success, true, JSON.stringify(added)); + return added.id; +} +async function indexed(id) { + const shards = await tursoShardManager.getAllShards("project", ""); + for (const shard of shards) { + const db = await tursoConnectionManager.getConnection(shard.dbPath); + const row = await tursoVectorSearch.getMemoryById(db, id); + if (row) return { row, db, metadata: JSON.parse(String(row.metadata)) }; + } + throw new Error("Missing indexed memory: " + id); +} +function filePath(id) { return join(CONFIG.markdownPath, id + ".md"); } +function assertCapture(value, capture) { + for (const [key, expected] of Object.entries(capture)) assert.equal(value[key], expected, key); +} +function assertAuxiliary(row) { + assert.equal(row.user_name, auxiliary.userName); + assert.equal(row.user_email, auxiliary.userEmail); + assert.equal(row.project_path, auxiliary.projectPath); + assert.equal(row.project_name, auxiliary.projectName); + assert.equal(row.git_repo_url, auxiliary.gitRepoUrl); + assert.equal(Number(row.is_pinned), 1); + assert.equal(row.deleted_at, null); +} +try { + ${body} +} finally { + await closeTursoAndInvalidateCaches(); +} +` + ); + const result = Bun.spawnSync({ + cmd: [process.execPath, script], + stdout: "pipe", + stderr: "pipe", + timeout: 20_000, + }); + expect(result.exitCode, Buffer.from(result.stderr).toString()).toBe(0); + } finally { + rmSync(baseDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + } +} + +describe("markdown capture metadata", () => { + it("preserves capture provenance and index-only state through writes and reconciliation", () => { + scenario(String.raw` +const id = await addCapture(); +let file = parseMemoryMarkdown(readFileSync(filePath(id), "utf8")); +assertCapture(file.frontmatter, initialCapture); +assert.equal(file.frontmatter.customProvenance, undefined); +let current = await indexed(id); +await tursoVectorSearch.pinMemory(current.db, id); +const first = await reconcileMarkdownVault(); +assert.equal(first.upserted, 1); +assert.deepEqual(first.errors, []); +current = await indexed(id); +assertCapture(current.metadata, initialCapture); +assert.equal(current.row.session_id, initialCapture.sessionID); +assert.deepEqual(current.metadata.customProvenance, { importedFrom: "synthetic" }); +assert.equal(current.metadata.updatedByCapture, true); +assertAuxiliary(current.row); +const repeated = await reconcileMarkdownVault(); +assert.equal(repeated.unchanged, 1); +assert.equal(repeated.upserted, 0); +assert.deepEqual(repeated.errors, []); +writeFileSync(filePath(id), readFileSync(filePath(id), "utf8").replace("Initial captured fact", "Edited in the vault")); +await reindexVaultFile(id + ".md"); +current = await indexed(id); +assert.equal(current.row.content, "Edited in the vault"); +assertCapture(current.metadata, initialCapture); +assertAuxiliary(current.row); +const updatedCapture = { promptId: "prompt_updated", sessionID: "session_updated", captureTimestamp: 0 }; +// Use the actual update-on-capture path; pinned memories are intentionally excluded. +await tursoVectorSearch.unpinMemory(current.db, id); +const updated = await applyUpdateOnCaptureDecision(memoryClient, { action: "update", targetId: id, content: "Updated captured fact" }, { + summary: "Updated captured fact", type: "fact", tags: [], containerTag: tag, + metadata: { source: "auto-capture", type: "fact", tags: [], ...updatedCapture } +}, new Set([id])); +assert.equal(updated.success, true, JSON.stringify(updated)); +assert.equal(updated.action, "update"); +file = parseMemoryMarkdown(readFileSync(filePath(id), "utf8")); +assertCapture(file.frontmatter, updatedCapture); +current = await indexed(id); +assert.equal(current.row.session_id, updatedCapture.sessionID); +assertCapture(current.metadata, updatedCapture); +await tursoVectorSearch.pinMemory(current.db, id); +const afterUpdate = await reconcileMarkdownVault(); +assert.equal(afterUpdate.upserted, 1); +assert.deepEqual(afterUpdate.errors, []); +current = await indexed(id); +assert.equal(current.row.content, "Updated captured fact"); +assert.equal(current.row.session_id, updatedCapture.sessionID); +assertCapture(current.metadata, updatedCapture); +assert.deepEqual(current.metadata.customProvenance, { importedFrom: "synthetic" }); +assertAuxiliary(current.row); +`); + }); + + it("retains legacy metadata when capture fields are absent or invalid, and accepts valid file edits", () => { + scenario(String.raw` +const id = await addCapture(); +const legacy = readFileSync(filePath(id), "utf8").split("\n").filter(line => !/^(promptId|sessionID|captureTimestamp):/.test(line)).join("\n"); +writeFileSync(filePath(id), legacy); +let result = await reconcileMarkdownVault(); +assert.deepEqual(result.errors, []); +let current = await indexed(id); +assertCapture(current.metadata, initialCapture); +assert.equal(current.row.session_id, initialCapture.sessionID); +assert.deepEqual(current.metadata.customProvenance, { importedFrom: "synthetic" }); +writeFileSync(filePath(id), legacy.replace("---\n", "---\npromptId: 123\nsessionID: false\ncaptureTimestamp: -1\n")); +await reindexVaultFile(id + ".md"); +current = await indexed(id); +assertCapture(current.metadata, initialCapture); +assert.equal(current.row.session_id, initialCapture.sessionID); +writeFileSync(filePath(id), legacy.replace("---\n", '---\npromptId: "prompt_from_file"\nsessionID: "session_from_file"\ncaptureTimestamp: 5678\n')); +await reindexVaultFile(id + ".md"); +current = await indexed(id); +assertCapture(current.metadata, { promptId: "prompt_from_file", sessionID: "session_from_file", captureTimestamp: 5678 }); +assert.equal(current.row.session_id, "session_from_file"); +assert.deepEqual(current.metadata.customProvenance, { importedFrom: "synthetic" }); +`); + }); + + it("rebuilds capture provenance and the indexed session from the vault into an empty database", () => { + scenario(String.raw` +const id = await addCapture(); +await closeTursoAndInvalidateCaches(); +CONFIG.storagePath = join(baseDir, "rebuilt-storage"); +memoryClient.reset(); +const result = await reconcileMarkdownVault(); +assert.equal(result.upserted, 1); +assert.deepEqual(result.errors, []); +const current = await indexed(id); +assertCapture(current.metadata, initialCapture); +assert.equal(current.row.session_id, initialCapture.sessionID); +assert.equal(current.metadata.source, "auto-capture"); +assert.equal(current.metadata.customProvenance, undefined); +assert.equal(current.row.content, "Initial captured fact"); +const sameSession = await tursoVectorSearch.getMemoriesBySessionID(current.db, initialCapture.sessionID); +assert.deepEqual(sameSession.map(memory => memory.id), [id]); +`); + }); + + it("retains the indexed session when a legacy row has no metadata session field", () => { + scenario(String.raw` +const id = await addCapture(); +const legacy = readFileSync(filePath(id), "utf8").split("\n").filter(line => !/^(promptId|sessionID|captureTimestamp):/.test(line)).join("\n"); +writeFileSync(filePath(id), legacy); +let current = await indexed(id); +await current.db.run("UPDATE memories SET metadata = ? WHERE id = ?", [JSON.stringify({ custom: "kept" }), id]); +const result = await reconcileMarkdownVault(); +assert.deepEqual(result.errors, []); +current = await indexed(id); +assert.equal(current.row.session_id, initialCapture.sessionID); +assert.equal(current.metadata.custom, "kept"); +assert.equal(current.metadata.promptId, undefined); +`); + }); + it("moves a memory between project shards without losing provenance, pin state, or counts", () => { + scenario(String.raw` +const id = await addCapture(); +const initial = await reconcileMarkdownVault(); +assert.deepEqual(initial.errors, []); +const originalShards = await tursoShardManager.getAllShards("project", "abcdef1234567890"); +assert.equal(originalShards.length, 1); +const originalShard = originalShards[0]; +let current = await indexed(id); +await tursoVectorSearch.pinMemory(current.db, id); +const originalDb = current.db; +const destinationTag = "opencode_project_1234567890abcdef"; +writeFileSync(filePath(id), readFileSync(filePath(id), "utf8").replace( + 'containerTag: "' + tag + '"', 'containerTag: "' + destinationTag + '"' +)); +const moved = await reconcileMarkdownVault(); +assert.equal(moved.upserted, 1); +assert.equal(moved.deleted, 0); +assert.deepEqual(moved.errors, []); +const destinationShards = await tursoShardManager.getAllShards("project", "1234567890abcdef"); +assert.equal(destinationShards.length, 1); +const destinationShard = destinationShards[0]; +assert.notEqual(destinationShard.id, originalShard.id); +assert.notEqual(destinationShard.dbPath, originalShard.dbPath); +assert.equal(destinationShard.scope, "project"); +assert.equal(destinationShard.scopeHash, "1234567890abcdef"); +assert.equal(await tursoVectorSearch.getMemoryById(originalDb, id, { includeDeleted: true }), null); +assert.equal(await tursoVectorSearch.countAllVectors(originalDb), 0); +assert.equal((await tursoShardManager.getShardById(originalShard.id)).vectorCount, 0); +const destinationDb = await tursoConnectionManager.getConnection(destinationShard.dbPath); +const destinationRow = await tursoVectorSearch.getMemoryById(destinationDb, id); +assert.equal(destinationRow.container_tag, destinationTag); +assert.equal(destinationRow.session_id, initialCapture.sessionID); +assertCapture(JSON.parse(destinationRow.metadata), initialCapture); +assert.deepEqual(JSON.parse(destinationRow.metadata).customProvenance, { importedFrom: "synthetic" }); +assertAuxiliary(destinationRow); +assert.equal(await tursoVectorSearch.countAllVectors(destinationDb), 1); +assert.equal((await tursoShardManager.getShardById(destinationShard.id)).vectorCount, 1); +const repeated = await reconcileMarkdownVault(); +assert.equal(repeated.unchanged, 1); +assert.deepEqual(repeated.errors, []); +assert.equal((await tursoShardManager.getShardById(originalShard.id)).vectorCount, 0); +assert.equal((await tursoShardManager.getShardById(destinationShard.id)).vectorCount, 1); +`); + }); + + it("reindexes malformed indexed metadata using valid capture fields from the file", () => { + scenario(String.raw` +const id = await addCapture(); +for (const malformed of ['{invalid JSON', 'null', '[{"promptId":"wrong"}]']) { + const before = await indexed(id); + await before.db.run("UPDATE memories SET metadata = ? WHERE id = ?", [malformed, id]); + const result = await reconcileMarkdownVault(); + assert.equal(result.upserted, 1, malformed); + assert.deepEqual(result.errors, [], malformed); + const current = await indexed(id); + assertCapture(current.metadata, initialCapture); + assert.equal(current.row.session_id, initialCapture.sessionID); + assert.equal(current.metadata.source, "auto-capture"); + assert.equal(current.metadata.markdownRelPath, id + ".md"); + assert.equal(current.metadata[0], undefined); + assert.equal(await tursoVectorSearch.countAllVectors(current.db), 1); +} +`); + }); +}); diff --git a/tests/markdown-document.test.ts b/tests/markdown-document.test.ts index 7847987a..c9db5c55 100644 --- a/tests/markdown-document.test.ts +++ b/tests/markdown-document.test.ts @@ -3,6 +3,7 @@ import { contentHash, markdownIdFromRelPath, packMemoryMarkdown, + readCaptureMetadata, parseMemoryMarkdown, titleFromRelPath, } from "../src/services/markdown/document.js"; @@ -38,6 +39,40 @@ describe("markdown document pack/parse", () => { expect(parsed.body.trim()).toBe("Remember to use bun test"); }); + it("roundtrips capture provenance with escaped ids and a zero timestamp", () => { + const capture = { + promptId: 'prompt_"quoted":id', + sessionID: "session_with\nnewline", + captureTimestamp: 0, + }; + const parsed = parseMemoryMarkdown( + packMemoryMarkdown({ id: "mem_capture", content: "Captured fact", ...capture }) + ); + expect(readCaptureMetadata(parsed.frontmatter)).toEqual(capture); + }); + + it("rejects malformed capture fields instead of coercing them into links", () => { + const raw = `--- +id: "mem_capture" +promptId: 123 +sessionID: false +captureTimestamp: "123" +--- + +Captured fact +`; + const parsed = parseMemoryMarkdown(raw); + expect(parsed.frontmatter.promptId).toBeUndefined(); + expect(parsed.frontmatter.sessionID).toBeUndefined(); + expect(parsed.frontmatter.captureTimestamp).toBeUndefined(); + for (const captureTimestamp of [-1, NaN, Infinity]) { + expect(readCaptureMetadata({ promptId: " ", sessionID: "", captureTimestamp })).toEqual({}); + } + expect( + packMemoryMarkdown({ id: "mem_capture", content: "Captured fact", captureTimestamp: NaN }) + ).not.toContain("captureTimestamp:"); + }); + it("parses foreign wiki pages without our id", () => { const raw = `--- title: "Raneto Page" diff --git a/web/src/lib/group-memories.ts b/web/src/lib/group-memories.ts index 42032f22..1840d4fd 100644 --- a/web/src/lib/group-memories.ts +++ b/web/src/lib/group-memories.ts @@ -12,21 +12,44 @@ export function isGroupPinned(group: MemoryGroup): boolean { export function groupMemories(items: MemoryItem[]): MemoryGroup[] { const map = new Map(items.map((item) => [item.id, item])); const pairs: MemoryGroup[] = []; + const paired = new Map(); const processed = new Set(); - for (const item of items) { - if (processed.has(item.id)) continue; + function pair(memory: MemoryItem | undefined, prompt: MemoryItem | undefined) { + if ( + memory?.type !== "memory" || + prompt?.type !== "prompt" || + paired.has(memory.id) || + paired.has(prompt.id) + ) { + return; + } + const group: MemoryGroup = { isPair: true, memory, prompt }; + paired.set(memory.id, group); + paired.set(prompt.id, group); + } - if (item.type === "memory" && item.linkedPromptId && map.has(item.linkedPromptId)) { - const prompt = map.get(item.linkedPromptId)!; - pairs.push({ isPair: true, memory: item, prompt }); - processed.add(item.id); - processed.add(prompt.id); - } else if (item.type === "prompt" && item.linkedMemoryId && map.has(item.linkedMemoryId)) { - const memory = map.get(item.linkedMemoryId)!; - pairs.push({ isPair: true, memory, prompt: item }); - processed.add(item.id); - processed.add(memory.id); + // Capture updates can leave older prompts pointing at the same memory. + // Prefer the memory's current prompt before considering those backlinks. + for (const item of map.values()) { + if (item.type === "memory" && item.linkedPromptId) { + pair(item, map.get(item.linkedPromptId)); + } + } + for (const item of map.values()) { + if (item.type === "prompt" && item.linkedMemoryId) { + pair(map.get(item.linkedMemoryId), item); + } + } + + // Emit each item once, retaining first-appearance order for equal dates. + for (const item of map.values()) { + if (processed.has(item.id)) continue; + const group = paired.get(item.id); + if (group?.isPair) { + pairs.push(group); + processed.add(group.memory.id); + processed.add(group.prompt.id); } else { pairs.push({ isPair: false, type: item.type, item }); processed.add(item.id);