Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -272,6 +272,8 @@ If you have chatted for a while and still see the message, check that auto-captu

Open `http://127.0.0.1:4747` to browse the memory–prompt timeline, inspect captures, and manage the user profile. The UI has five views: **Project memories** (`/project-memories`), **User profile** (`/user-profile`), **Skill library** (`/skill-library`), **Markdown vault** (`/markdown-vault`) and **Stats** (`/stats`). Change the port with `webServerPort`, or set `webServerEnabled: false` to run without the web UI. If you bind the server beyond loopback, see [Web UI HTTP Basic Auth](#web-ui-http-basic-auth).

The timeline shows each memory and prompt once. If several capture prompts refer to one memory, its current prompt is paired with it when available on the page; earlier prompts remain visible as separate entries.

## Usage Examples

```typescript
Expand Down Expand Up @@ -325,6 +327,8 @@ By default memories live only in Turso under `storagePath`. Optionally set a **g
}
```

Plugin-created vault files preserve capture provenance (`promptId`, `sessionID`, and `captureTimestamp`) so reindexing can restore those fields. Legacy files without them retain any corresponding metadata already in the index. Provenance already missing from both the file and the index cannot be recovered automatically.

| Key | Default | What it does |
| --------------- | ------------ | -------------------------------------------------------------------------------------------------------------------------------------------------- |
| `memoryBackend` | `"database"` | `"database"` keeps content in Turso. `"markdown"` uses all `.md` files under `markdownPath` as source of truth. |
Expand Down
10 changes: 5 additions & 5 deletions src/services/client.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import type { MemoryType } from "../types/index.js";
import type { MemoryListFilters, MemoryRecord } from "./turso/types.js";
import { isMarkdownBackend, writeMemoryMarkdownFile } from "./markdown/backend.js";
import { pluginMemoryRelPath } from "./markdown/vault.js";
import { readCaptureMetadata } from "./markdown/document.js";

export type MemoryScope = "project" | "all-projects";

Expand Down Expand Up @@ -263,6 +264,7 @@ export class LocalMemoryClient {
updatedAt: now,
source: typeof dynamicMetadata.source === "string" ? dynamicMetadata.source : "manual",
title: displayName,
...readCaptureMetadata(dynamicMetadata),
});
dynamicMetadata.markdownRelPath = pluginMemoryRelPath(id);
if (!dynamicMetadata.source) dynamicMetadata.source = "manual";
Expand Down Expand Up @@ -374,6 +376,7 @@ export class LocalMemoryClient {
updatedAt,
source: typeof existingMeta.source === "string" ? existingMeta.source : "api",
title: existingMemory.display_name ? String(existingMemory.display_name) : undefined,
...readCaptureMetadata(existingMeta),
});
existingMeta.markdownRelPath = pluginMemoryRelPath(id);
}
Expand Down Expand Up @@ -406,11 +409,8 @@ export class LocalMemoryClient {
Number(existingMemory.created_at),
updatedAt,
metadataJson,
existingMemory.session_id
? String(existingMemory.session_id)
: parseSessionIdFromMetadata(
existingMemory.metadata ? String(existingMemory.metadata) : null
),
parseSessionIdFromMetadata(metadataJson) ??
(existingMemory.session_id ? String(existingMemory.session_id) : null),
existingMemory.display_name ? String(existingMemory.display_name) : null,
existingMemory.user_name ? String(existingMemory.user_name) : null,
existingMemory.user_email ? String(existingMemory.user_email) : null,
Expand Down
7 changes: 5 additions & 2 deletions src/services/markdown/backend.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import { mkdirSync } from "node:fs";
import { CONFIG } from "../../config.js";
import { log } from "../logger.js";
import { packMemoryMarkdown } from "./document.js";
import { packMemoryMarkdown, type MemoryCaptureMetadata } from "./document.js";
import {
getLastReconcileAt,
listVaultIndexEntries,
Expand All @@ -18,7 +18,7 @@ import {
} from "./vault.js";
import { markdownVaultWatcher } from "./watcher.js";

export interface MarkdownMemoryWriteInput {
export interface MarkdownMemoryWriteInput extends MemoryCaptureMetadata {
id: string;
content: string;
type?: string;
Expand Down Expand Up @@ -63,6 +63,9 @@ export function writeMemoryMarkdownFile(input: MarkdownMemoryWriteInput): string
updatedAt: input.updatedAt,
source: input.source,
title: input.title,
promptId: input.promptId,
sessionID: input.sessionID,
captureTimestamp: input.captureTimestamp,
});
markdownVaultWatcher.noteSelfWrite(relPath);
writeVaultFile(root, relPath, markdown);
Expand Down
39 changes: 36 additions & 3 deletions src/services/markdown/document.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,30 @@
import { createHash } from "node:crypto";
import { basename } from "node:path";

export interface MemoryMarkdownFrontmatter {
export interface MemoryCaptureMetadata {
promptId?: string;
sessionID?: string;
captureTimestamp?: number;
}

/** Only persist well-formed capture provenance; other metadata stays in the index. */
export function readCaptureMetadata(input: Record<string, unknown>): MemoryCaptureMetadata {
const capture: MemoryCaptureMetadata = {};
for (const key of ["promptId", "sessionID"] as const) {
const value = input[key];
if (typeof value === "string" && value.trim()) capture[key] = value;
}
if (
typeof input.captureTimestamp === "number" &&
Number.isFinite(input.captureTimestamp) &&
input.captureTimestamp >= 0
) {
capture.captureTimestamp = input.captureTimestamp;
}
return capture;
}

export interface MemoryMarkdownFrontmatter extends MemoryCaptureMetadata {
id?: string;
type?: string;
tags?: string[];
Expand All @@ -20,7 +43,7 @@ export interface ParsedMemoryMarkdown {
hasFrontmatter: boolean;
}

export interface PackMemoryMarkdownInput {
export interface PackMemoryMarkdownInput extends MemoryCaptureMetadata {
id: string;
content: string;
type?: string;
Expand Down Expand Up @@ -98,6 +121,12 @@ export function packMemoryMarkdown(input: PackMemoryMarkdownInput): string {
if (input.updatedAt !== undefined) lines.push(`updatedAt: ${input.updatedAt}`);
if (input.source) lines.push(`source: ${yamlScalar(input.source)}`);
if (input.title) lines.push(`title: ${yamlScalar(input.title)}`);
const capture = readCaptureMetadata({ ...input });
if (capture.promptId) lines.push(`promptId: ${yamlScalar(capture.promptId)}`);
if (capture.sessionID) lines.push(`sessionID: ${yamlScalar(capture.sessionID)}`);
if (capture.captureTimestamp !== undefined) {
lines.push(`captureTimestamp: ${capture.captureTimestamp}`);
}
lines.push("---", "", input.content);
return lines.join("\n");
}
Expand Down Expand Up @@ -131,7 +160,11 @@ export function parseMemoryMarkdown(raw: string): ParsedMemoryMarkdown {
continue;
}
const parsed = parseYamlScalar(valueRaw);
frontmatter[key] = parsed;
if (key === "promptId" || key === "sessionID" || key === "captureTimestamp") {
Object.assign(frontmatter, readCaptureMetadata({ [key]: parsed }));
} else {
frontmatter[key] = parsed;
}
}

return {
Expand Down
34 changes: 30 additions & 4 deletions src/services/markdown/reconcile.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import {
contentHash,
markdownIdFromRelPath,
parseMemoryMarkdown,
readCaptureMetadata,
titleFromRelPath,
} from "./document.js";
import { listMarkdownFiles, readVaultFile, type VaultFileInfo } from "./vault.js";
Expand Down Expand Up @@ -43,7 +44,10 @@ export function getLastReconcileAt(): number | null {
function parseMetadata(value: unknown): Record<string, unknown> {
if (!value || typeof value !== "string") return {};
try {
return JSON.parse(value) as Record<string, unknown>;
const parsed: unknown = JSON.parse(value);
return parsed && typeof parsed === "object" && !Array.isArray(parsed)
? (parsed as Record<string, unknown>)
: {};
} catch {
return {};
}
Expand Down Expand Up @@ -98,11 +102,11 @@ async function upsertFromFile(file: VaultFileInfo): Promise<"upserted" | "unchan

const hash = contentHash(raw);
const existing = await findMemoryAcrossShards(id);
const existingMetadata = parseMetadata(existing?.row.metadata);
if (existing) {
const meta = parseMetadata(existing.row.metadata);
if (
meta.markdownContentHash === hash &&
meta.markdownRelPath === file.relPath &&
existingMetadata.markdownContentHash === hash &&
existingMetadata.markdownRelPath === file.relPath &&
String(existing.row.content) === content
) {
return "unchanged";
Expand Down Expand Up @@ -140,7 +144,15 @@ async function upsertFromFile(file: VaultFileInfo): Promise<"upserted" | "unchan
});
}

// Legacy vault files omit capture fields. Keep their surviving provenance and
// other index-only metadata; validated file values take precedence when present.
const capture = {
...readCaptureMetadata(existingMetadata),
...readCaptureMetadata(fm),
};
const metadata = JSON.stringify({
...existingMetadata,
...capture,
source,
markdownRelPath: file.relPath,
markdownContentHash: hash,
Expand All @@ -159,7 +171,18 @@ async function upsertFromFile(file: VaultFileInfo): Promise<"upserted" | "unchan
createdAt,
updatedAt,
metadata,
sessionId:
capture.sessionID ??
(typeof existing?.row.session_id === "string" ? existing.row.session_id : undefined),
displayName: title,
userName: typeof existing?.row.user_name === "string" ? existing.row.user_name : undefined,
userEmail: typeof existing?.row.user_email === "string" ? existing.row.user_email : undefined,
projectPath:
typeof existing?.row.project_path === "string" ? existing.row.project_path : undefined,
projectName:
typeof existing?.row.project_name === "string" ? existing.row.project_name : undefined,
gitRepoUrl:
typeof existing?.row.git_repo_url === "string" ? existing.row.git_repo_url : undefined,
};

const { scope, hash: scopeHash } = extractScopeFromContainerTag(containerTag);
Expand All @@ -181,6 +204,9 @@ async function upsertFromFile(file: VaultFileInfo): Promise<"upserted" | "unchan
await tursoShardManager.decrementVectorCount(shard.id);
}
await tursoVectorSearch.insertVector(db, record);
if (Number(existing?.row.is_pinned) === 1) {
await tursoVectorSearch.pinMemory(db, id);
}
await tursoShardManager.incrementVectorCount(shard.id);
});

Expand Down
Loading
Loading