From d5a3b0e5c00d00ad6a592a4eaae377f3fae1cf83 Mon Sep 17 00:00:00 2001 From: Phototonic <35551999+Phototonic@users.noreply.github.com> Date: Fri, 6 Nov 2026 16:52:15 +1100 Subject: [PATCH] fix(markdown): preserve index when vault inventory is unsafe --- .github/i18n/README.ar.md | 4 +- .github/i18n/README.de.md | 4 +- .github/i18n/README.nl.md | 4 +- .github/i18n/README.tr.md | 4 +- .github/i18n/README.zh.md | 4 +- README.md | 4 +- src/services/markdown/reconcile.ts | 90 +++++-- src/services/markdown/vault.ts | 32 ++- tests/fixtures/markdown-reconcile-safety.ts | 264 ++++++++++++++++++++ tests/markdown-reconcile-safety.test.ts | 68 +++++ 10 files changed, 440 insertions(+), 38 deletions(-) create mode 100644 tests/fixtures/markdown-reconcile-safety.ts create mode 100644 tests/markdown-reconcile-safety.test.ts diff --git a/.github/i18n/README.ar.md b/.github/i18n/README.ar.md index cfaf7ff8..dde52913 100644 --- a/.github/i18n/README.ar.md +++ b/.github/i18n/README.ar.md @@ -281,7 +281,9 @@ memory({ mode: "import", inputPath: "./memories.json" }); - تُكتب الذكريات التي ينشئها المكوّن الإضافي كـ `{id}.md` مع YAML frontmatter + المحتوى. - تُستورد/تُفهرس الملاحظات الموجودة في المجلد (أي أسلوب صفحة Markdown)؛ وTurso مجرد فهرس بحث دلالي قابل لإعادة البناء. - تُراقَب إضافات/تعديلات/حذف الملفات ويُعاد فهرستها؛ تعرض واجهة الويب صفحة شريط جانبي **Markdown** (الحالة، قائمة الملفات، إعادة الفهرسة) في هذا الوضع فقط. -- التبديل إلى Markdown يعيد بناء الفهرس من المجلد — تُسقَط صفوف قاعدة البيانات فقط بلا ملف مطابق. صدّر أولًا بـ `memory export` إن احتجت الإبقاء على ذكريات قاعدة البيانات القديمة. +- التبديل إلى Markdown يعيد بناء الفهرس من المجلد. لا تُحذف السجلات الموجودة في قاعدة البيانات فقط والتي لا يقابلها ملف إلا بعد فحص كامل لخزنة قابلة للقراءة وغير فارغة. صدّر أولًا باستخدام `memory export` إذا أردت الاحتفاظ بذكريات قاعدة البيانات القديمة. +- تتخطى المطابقة الحذف وتصدر تحذيرًا إذا كانت الخزنة مفقودة أو غير قابلة للقراءة أو جرى فحصها أو معالجتها جزئيًا فقط أو لا تحتوي على ملفات Markdown؛ وتبقى الذكريات المفهرسة الموجودة محفوظة. استعد الخزنة أو أصلحها، ثم أعد الفهرسة. لإزالة الذاكرة المفهرسة المرتبطة بآخر ملف عمدًا، احذف تلك الذاكرة صراحةً من خلال الأداة أو واجهة الويب. +- تظل الذكريات المفهرسة التي لم تعد ملفاتها موجودة عرضة للحذف الدائم عندما تكون الخزنة سليمة وغير فارغة. صدّر البيانات أو أنشئ نسخة احتياطية أولًا: هذه الإجراءات الوقائية ليست بديلًا عن النسخ الاحتياطي ولا تكشف المزامنة غير المكتملة التي تبدو كخزنة سليمة وغير فارغة. ### التقاط رسائل الدردشة وحقنها (`chatMessage`) diff --git a/.github/i18n/README.de.md b/.github/i18n/README.de.md index 6d583b07..94e41cac 100644 --- a/.github/i18n/README.de.md +++ b/.github/i18n/README.de.md @@ -281,7 +281,9 @@ Wenn Markdown-Modus aktiv ist: - Vom Plugin erzeugte Memories werden als `{id}.md` mit YAML-Frontmatter + Body geschrieben. - Vorhandene Notizen im Ordner (beliebiger Markdown-Seitenstil) werden importiert/indexiert; Turso ist nur ein neu aufbaubarer Semantic-Search-Index. - Datei-Adds/Edits/Deletes werden beobachtet und neu indexiert; die Web-UI zeigt nur in diesem Modus eine **Markdown**-Sidebar-Seite (Status, Dateiliste, Reindex). -- Der Wechsel zu Markdown baut den Index aus dem Ordner neu — DB-only-Zeilen ohne passende Datei entfallen. Zuerst mit `memory export` exportieren, wenn du alte Datenbank-Memories behalten willst. +- Der Wechsel zu Markdown baut den Index aus dem Ordner neu auf. Reine Datenbankeinträge ohne passende Datei werden nur nach einem lesbaren, vollständigen Scan eines nicht leeren Vaults entfernt. Exportiere zuerst mit `memory export`, wenn du alte Datenbank-Memories behalten möchtest. +- Der Abgleich überspringt das Löschen und gibt eine Warnung aus, wenn der Vault fehlt, nicht lesbar ist, nur teilweise gescannt oder verarbeitet wurde oder keine Markdown-Dateien enthält; bereits indexierte Memories bleiben erhalten. Stelle den Vault wieder her oder behebe das Problem und indexiere ihn erneut. Um die indexierte Memory der letzten Datei absichtlich zu entfernen, lösche diese Memory ausdrücklich über das Tool oder die Web-UI. +- Bei einem funktionierenden, nicht leeren Vault werden indexierte Memories ohne zugehörige Datei weiterhin endgültig gelöscht. Exportiere die Daten oder erstelle vorher ein Backup: Diese Schutzmaßnahmen ersetzen kein Backup und erkennen keine unvollständige Synchronisierung, die wie ein funktionierender, nicht leerer Vault aussieht. ### Erfassung und Injection von Chat-Nachrichten (`chatMessage`) diff --git a/.github/i18n/README.nl.md b/.github/i18n/README.nl.md index 746098a2..de9a87e5 100644 --- a/.github/i18n/README.nl.md +++ b/.github/i18n/README.nl.md @@ -281,7 +281,9 @@ Wanneer Markdown-modus aan staat: - Door de plugin gemaakte herinneringen worden als `{id}.md` met YAML-frontmatter + body geschreven. - Bestaande notities in de map (elke Markdown-paginastijl) worden geïmporteerd/geïndexeerd; Turso is alleen een herbouwbare semantic-search-index. - Bestandsadds/bewerkingen/verwijderingen worden bewaakt en opnieuw geïndexeerd; de web-UI toont alleen in deze modus een **Markdown**-sidebarpagina (status, bestandenlijst, reindex). -- Overschakelen naar Markdown herbouwt de index vanuit de map — DB-only-rijen zonder bijpassend bestand verdwijnen. Exporteer eerst met `memory export` als je oude databaseherinneringen wilt bewaren. +- Overschakelen naar Markdown herbouwt de index vanuit de map. Rijen die alleen in de database staan en geen bijpassend bestand hebben, worden alleen verwijderd na een leesbare, volledige scan van een niet-lege vault. Exporteer eerst met `memory export` als je oude databaseherinneringen wilt bewaren. +- Bij het synchroniseren wordt verwijdering overgeslagen en verschijnt een waarschuwing als de vault ontbreekt, onleesbaar is, slechts gedeeltelijk is gescand of verwerkt, of geen Markdown-bestanden bevat; bestaande geïndexeerde herinneringen blijven behouden. Herstel de vault of los het probleem op en indexeer daarna opnieuw. Wil je de geïndexeerde herinnering van het laatste bestand bewust verwijderen, verwijder die herinnering dan expliciet via de tool of web-UI. +- Bij een gezonde, niet-lege vault worden geïndexeerde herinneringen waarvan de bestanden ontbreken nog steeds permanent verwijderd. Exporteer of maak eerst een back-up: deze beveiligingen vervangen geen back-up en herkennen geen onvolledige synchronisatie die eruitziet als een gezonde, niet-lege vault. ### Chatberichtvastlegging en -injectie (`chatMessage`) diff --git a/.github/i18n/README.tr.md b/.github/i18n/README.tr.md index 5ee8b272..daee7bb5 100644 --- a/.github/i18n/README.tr.md +++ b/.github/i18n/README.tr.md @@ -283,7 +283,9 @@ Markdown modu açıkken: - Eklentinin oluşturduğu bellekler YAML frontmatter + gövde ile `{id}.md` olarak yazılır. - Klasördeki mevcut notlar (herhangi bir Markdown sayfa stili) içe aktarılır/indekslenir; Turso yalnızca yeniden oluşturulabilir bir anlamsal arama indeksidir. - Dosya ekleme/düzenleme/silme izlenir ve yeniden indekslenir; Web arayüzü yalnızca bu modda bir **Markdown** kenar çubuğu sayfası gösterir (durum, dosya listesi, yeniden indeks). -- Markdown’a geçiş indeksi klasörden yeniden oluşturur — eşleşen dosyası olmayan yalnızca-DB satırları düşer. Eski veritabanı belleklerini saklamak istiyorsanız önce `memory export` ile dışa aktarın. +- Markdown’a geçiş indeksi klasörden yeniden oluşturur. Eşleşen dosyası olmayan, yalnızca veritabanında bulunan kayıtlar ancak okunabilir ve boş olmayan bir vault tamamen tarandıktan sonra silinir. Eski veritabanı belleklerini saklamak istiyorsanız önce `memory export` ile dışa aktarın. +- Vault yoksa, okunamıyorsa, yalnızca kısmen taranmış veya işlenmişse ya da hiç Markdown dosyası içermiyorsa uzlaştırma silme işlemini atlar ve uyarı verir; mevcut indekslenmiş bellekler korunur. Vault'u geri yükleyin veya sorunu giderin, ardından yeniden indeksleyin. Son dosyanın indekslenmiş belleğini bilerek kaldırmak için bu belleği araç veya Web UI üzerinden açıkça silin. +- Sağlıklı ve boş olmayan bir vault'ta, dosyaları artık bulunmayan indekslenmiş bellekler yine kalıcı olarak silinir. Önce dışa aktarın veya yedek alın: bu önlemler yedeklemenin yerini tutmaz ve sağlıklı, boş olmayan bir vault gibi görünen eksik senkronizasyonu tespit edemez. ### Sohbet mesajı yakalama ve enjekte etme (`chatMessage`) diff --git a/.github/i18n/README.zh.md b/.github/i18n/README.zh.md index d7c644ab..65cc6465 100644 --- a/.github/i18n/README.zh.md +++ b/.github/i18n/README.zh.md @@ -281,7 +281,9 @@ memory({ mode: "import", inputPath: "./memories.json" }); - 插件创建的记忆以 `{id}.md` 写入，含 YAML frontmatter + 正文。 - 文件夹中已有笔记（任意 Markdown 页面风格）会被导入/索引；Turso 仅作为可重建的语义搜索索引。 - 监视文件新增/编辑/删除并重新索引；Web UI 仅在此模式下显示 **Markdown** 侧边栏页（状态、文件列表、重新索引）。 -- 切换到 Markdown 会从文件夹重建索引 — 没有对应文件的仅数据库行会被丢弃。若需保留旧数据库记忆，请先用 `memory export` 导出。 +- 切换到 Markdown 会从文件夹重建索引。只有在对可读取、非空的 vault 完成完整扫描后，才会删除没有对应文件的纯数据库记录。如需保留旧数据库记忆，请先用 `memory export` 导出。 +- 如果 vault 缺失、不可读取、仅完成部分扫描或处理，或不包含任何 Markdown 文件，对账会跳过删除并发出警告，保留已有的索引记忆。恢复或修复 vault 后，请重新索引。如果确实要移除最后一个文件对应的索引记忆，请通过工具或 Web UI 明确删除该记忆。 +- 对于正常且非空的 vault，对应文件已不存在的索引记忆仍会被永久删除。请先导出或备份：这些保护措施不能替代备份，也无法识别看似正常且非空、实则尚未完成同步的 vault。 ### 聊天消息捕获与注入（`chatMessage`） diff --git a/README.md b/README.md index 0dc9786f..a132ebd7 100644 --- a/README.md +++ b/README.md @@ -335,7 +335,9 @@ When Markdown mode is on: - Plugin-created memories are written as `{id}.md` with YAML frontmatter + body. - Existing notes in the folder (any Markdown page style) are imported/indexed; Turso is only a rebuildable semantic-search index. - File adds/edits/deletes are watched and re-indexed; the Web UI shows a **Markdown** sidebar page (status, file list, reindex) only in this mode. -- Switching to Markdown rebuilds the index from the folder — DB-only rows without a matching file are dropped. Export with `memory export` first if you need to keep old database memories. +- Switching to Markdown rebuilds the index from the folder. DB-only rows without a matching file are dropped only after a readable, complete scan of a nonempty vault. Export with `memory export` first if you need to keep old database memories. +- Reconciliation skips deletion and warns when the vault is missing, unreadable, only partially scanned or processed, or contains no Markdown files; existing indexed memories are retained. Restore or fix the vault, then reindex. To intentionally remove the final file's indexed memory, explicitly delete that memory through the tool or Web UI. +- A healthy, nonempty vault still permits permanent deletion of indexed memories whose files are absent. Export or back up first: these safeguards are not a backup and cannot detect an incomplete sync that appears to be a healthy, nonempty vault. ### Chat message capture and injection (`chatMessage`) diff --git a/src/services/markdown/reconcile.ts b/src/services/markdown/reconcile.ts index 91da677e..00f4de00 100644 --- a/src/services/markdown/reconcile.ts +++ b/src/services/markdown/reconcile.ts @@ -14,13 +14,20 @@ import { parseMemoryMarkdown, titleFromRelPath, } from "./document.js"; -import { listMarkdownFiles, readVaultFile, type VaultFileInfo } from "./vault.js"; +import { + listMarkdownFiles, + readVaultFile, + scanMarkdownVault, + type VaultFileInfo, +} from "./vault.js"; export interface ReconcileResult { scanned: number; upserted: number; unchanged: number; deleted: number; + /** Missing files are not evidence of deletion when the inventory is unsafe. */ + deletionSkipped?: "empty-vault" | "incomplete-scan"; errors: Array<{ path: string; error: string }>; finishedAt: number; } @@ -83,9 +90,11 @@ async function resolveContainerTag(fmContainerTag: string | undefined): Promise< return getUserTagInfo().tag; } -async function upsertFromFile(file: VaultFileInfo): Promise<"upserted" | "unchanged"> { - const raw = readVaultFile(CONFIG.markdownPath!, file.relPath); - const parsed = parseMemoryMarkdown(raw); +async function upsertFromFile( + file: VaultFileInfo, + raw: string, + parsed = parseMemoryMarkdown(raw) +): Promise<"upserted" | "unchanged"> { const fm = parsed.frontmatter; const id = @@ -208,6 +217,11 @@ async function collectIndexedMarkdownIds(): Promise< } export async function reconcileMarkdownVault(): Promise { + return reconcileVault(); +} + +/** A watcher deletion validates the whole inventory but only removes its own path. */ +async function reconcileVault(deletedRelPath?: string): Promise { const vaultRoot = CONFIG.markdownPath; if (!vaultRoot) { throw new Error("markdownPath is not configured"); @@ -216,14 +230,14 @@ export async function reconcileMarkdownVault(): Promise { await ensureTursoReady(); await embeddingService.warmup(); - const files = listMarkdownFiles(vaultRoot); + const { files, errors } = scanMarkdownVault(vaultRoot); const seenIds = new Set(); const result: ReconcileResult = { scanned: files.length, upserted: 0, unchanged: 0, deleted: 0, - errors: [], + errors, finishedAt: Date.now(), }; @@ -236,7 +250,9 @@ export async function reconcileMarkdownVault(): Promise { ? parsed.frontmatter.id.trim() : markdownIdFromRelPath(file.relPath); seenIds.add(id); - const outcome = await upsertFromFile(file); + // Use the same snapshot for identity and content. A second read could see + // a different id after an editor/sync write and delete the new row below. + const outcome = await upsertFromFile(file, raw, parsed); if (outcome === "upserted") result.upserted++; else result.unchanged++; } catch (error) { @@ -252,16 +268,40 @@ export async function reconcileMarkdownVault(): Promise { } const indexed = await collectIndexedMarkdownIds(); - for (const [id] of indexed) { - if (seenIds.has(id)) continue; - try { - const deleted = await deleteMemoryById(id); - if (deleted) result.deleted++; - } catch (error) { - result.errors.push({ - path: id, - error: error instanceof Error ? error.message : String(error), - }); + if (result.errors.length > 0) { + result.deletionSkipped = "incomplete-scan"; + } else if (files.length === 0 && indexed.size > 0) { + // An intentionally emptied vault cannot be distinguished from a missing + // mount or incomplete sync. Explicit memory deletion remains available. + result.deletionSkipped = "empty-vault"; + } + + if (result.deletionSkipped) { + log("markdown reconcile warning: deletion skipped; existing index retained", { + reason: result.deletionSkipped, + path: vaultRoot, + indexed: indexed.size, + errors: result.errors, + }); + } else { + for (const [id, info] of indexed) { + if (seenIds.has(id)) continue; + if ( + deletedRelPath !== undefined && + info.relPath !== deletedRelPath && + id !== markdownIdFromRelPath(deletedRelPath) + ) { + continue; + } + try { + const deleted = await deleteMemoryById(id); + if (deleted) result.deleted++; + } catch (error) { + result.errors.push({ + path: id, + error: error instanceof Error ? error.message : String(error), + }); + } } } @@ -271,6 +311,7 @@ export async function reconcileMarkdownVault(): Promise { upserted: result.upserted, unchanged: result.unchanged, deleted: result.deleted, + deletionSkipped: result.deletionSkipped, errors: result.errors.length, }); return result; @@ -328,17 +369,14 @@ export async function reindexVaultFile(relPath: string): Promise { if (!vaultRoot) throw new Error("markdownPath is not configured"); await ensureTursoReady(); await embeddingService.warmup(); - const files = listMarkdownFiles(vaultRoot); + const { files } = scanMarkdownVault(vaultRoot); const file = files.find((f) => f.relPath === relPath); if (!file) { - // File gone — drop any id derived from path or previously stored for this path. - const indexed = await collectIndexedMarkdownIds(); - for (const [id, info] of indexed) { - if (info.relPath === relPath || id === markdownIdFromRelPath(relPath)) { - await deleteMemoryById(id); - } - } + // Watcher events also arrive when the vault disappears or cannot be read. + // Require the same complete, nonempty inventory as a full reconciliation. + // This also preserves an explicit id when a file was renamed in the vault. + await reconcileVault(relPath); return; } - await upsertFromFile(file); + await upsertFromFile(file, readVaultFile(vaultRoot, file.relPath)); } diff --git a/src/services/markdown/vault.ts b/src/services/markdown/vault.ts index b17c74fd..67c8587d 100644 --- a/src/services/markdown/vault.ts +++ b/src/services/markdown/vault.ts @@ -27,6 +27,12 @@ export interface VaultFileInfo { size: number; } +export interface VaultScanResult { + files: VaultFileInfo[]; + /** A partial inventory must never be used to infer that memories were deleted. */ + errors: Array<{ path: string; error: string }>; +} + function normalizeRel(relPath: string): string { return relPath.replace(/\\/g, "/").replace(/^\/+/, ""); } @@ -66,17 +72,24 @@ function shouldSkipDir(name: string): boolean { return false; } -export function listMarkdownFiles(vaultRoot: string): VaultFileInfo[] { +export function scanMarkdownVault(vaultRoot: string): VaultScanResult { const root = resolve(vaultRoot); - if (!existsSync(root)) return []; - const out: VaultFileInfo[] = []; + const errors: VaultScanResult["errors"] = []; + + function recordError(absPath: string, error: unknown): void { + errors.push({ + path: normalizeRel(relative(root, absPath)) || ".", + error: error instanceof Error ? error.message : String(error), + }); + } function walk(dir: string): void { let entries; try { entries = readdirSync(dir, { withFileTypes: true }); - } catch { + } catch (error) { + recordError(dir, error); return; } for (const entry of entries) { @@ -91,7 +104,9 @@ export function listMarkdownFiles(vaultRoot: string): VaultFileInfo[] { let st; try { st = statSync(abs); - } catch { + if (!st.isFile()) throw new Error("Markdown file is no longer a regular file"); + } catch (error) { + recordError(abs, error); continue; } const rel = normalizeRel(relative(root, abs)); @@ -106,7 +121,12 @@ export function listMarkdownFiles(vaultRoot: string): VaultFileInfo[] { walk(root); out.sort((a, b) => a.relPath.localeCompare(b.relPath)); - return out; + return { files: out, errors }; +} + +/** Best-effort listing for display; reconciliation must use the scan's errors too. */ +export function listMarkdownFiles(vaultRoot: string): VaultFileInfo[] { + return scanMarkdownVault(vaultRoot).files; } export function readVaultFile(vaultRoot: string, relPath: string): string { diff --git a/tests/fixtures/markdown-reconcile-safety.ts b/tests/fixtures/markdown-reconcile-safety.ts new file mode 100644 index 00000000..70201220 --- /dev/null +++ b/tests/fixtures/markdown-reconcile-safety.ts @@ -0,0 +1,264 @@ +import assert from "node:assert/strict"; +import * as fs from "node:fs"; +import { dirname, join } from "node:path"; +import { spyOn } from "bun:test"; + +const [scenario, home] = process.argv.slice(2); +const vault = join(home!, "vault"); +const { CONFIG } = await import("../../src/config.js"); +Object.assign(CONFIG, { + storagePath: join(home!, "storage"), + markdownPath: vault, + memoryBackend: "markdown", + embeddingDimensions: 2, +}); +const { embeddingService } = await import("../../src/services/embedding.js"); +embeddingService.warmup = async () => {}; +embeddingService.embedWithTimeout = async () => new Float32Array([1, 0]); +const { reconcileMarkdownVault, reindexVaultFile } = + await import("../../src/services/markdown/reconcile.js"); +const { tursoShardManager } = await import("../../src/services/turso/shard-manager.js"); +const { tursoConnectionManager } = await import("../../src/services/turso/connection-manager.js"); +const { tursoVectorSearch } = await import("../../src/services/turso/vector-search.js"); +const { startMarkdownBackend, stopMarkdownBackend } = + await import("../../src/services/markdown/backend.js"); +const { getUserTagInfo } = await import("../../src/services/tags.js"); +const tag = getUserTagInfo().tag; + +function note(path: string, id: string, body = `Content for ${id}`) { + fs.mkdirSync(dirname(join(vault, path)), { recursive: true }); + fs.writeFileSync(join(vault, path), `---\nid: "${id}"\ncontainerTag: "${tag}"\n---\n\n${body}\n`); +} +async function snapshot() { + const shards = await tursoShardManager.getAllShards("user", ""); + const rows = []; + for (const shard of shards) { + const db = await tursoConnectionManager.getConnection(shard.dbPath); + rows.push( + ...(await db.all("SELECT id, content, metadata, deleted_at FROM memories ORDER BY id")) + ); + } + return { rows, counts: shards.map((s) => s.vectorCount) }; +} +async function ids() { + return (await snapshot()).rows + .filter((r) => r.deleted_at === null) + .map((r) => r.id) + .sort(); +} +let restore: (() => void) | undefined; +try { + fs.mkdirSync(vault, { recursive: true }); + if (scenario === "fresh-empty") { + const result = await reconcileMarkdownVault(); + assert.equal(result.deleted, 0); + assert.equal(result.deletionSkipped, undefined); + assert.deepEqual(result.errors, []); + } else { + note("a.md", "a"); + note("nested/b.md", "b"); + note("c.md", "c"); + assert.equal((await reconcileMarkdownVault()).upserted, 3); + // Preserve unrelated metadata and a trashed row too, not just live IDs. + const shard = (await tursoShardManager.getAllShards("user", ""))[0]!; + const db = await tursoConnectionManager.getConnection(shard.dbPath); + await tursoVectorSearch.patchMemoryMetadata(db, "a", { custom: "keep-me" }); + await tursoVectorSearch.insertVector(db, { + id: "trashed", + content: "Trash", + vector: new Float32Array([1, 0]), + containerTag: tag, + createdAt: 1, + updatedAt: 1, + metadata: '{"custom":"trash-metadata"}', + }); + await tursoShardManager.incrementVectorCount(shard.id); + await tursoVectorSearch.softDeleteVector(db, "trashed"); + if (scenario === "incremental-delete") { + await tursoVectorSearch.insertVector(db, { + id: "unrelated", + content: "Database-only memory", + vector: new Float32Array([1, 0]), + containerTag: tag, + createdAt: 1, + updatedAt: 1, + metadata: '{"custom":"unrelated-metadata"}', + }); + await tursoShardManager.incrementVectorCount(shard.id); + } + const before = await snapshot(); + let reason: "empty-vault" | "incomplete-scan" = "incomplete-scan"; + let errorPath = "."; + let readableAddition = false; + if ( + ["missing-root", "startup-missing-root", "incremental-missing", "restored-vault"].includes( + scenario! + ) + ) { + fs.renameSync(vault, `${vault}-saved`); + if (scenario === "startup-missing-root") reason = "empty-vault"; + } else if (["empty-root", "incremental-empty", "no-markdown-files"].includes(scenario!)) { + fs.rmSync(vault, { recursive: true }); + fs.mkdirSync(vault); + reason = "empty-vault"; + if (scenario === "no-markdown-files") { + fs.writeFileSync(join(vault, "readme.txt"), "Not a Markdown note"); + note(".obsidian/ignored.md", "ignored-obsidian"); + note(".trash/ignored.md", "ignored-trash"); + } + } else if (scenario === "root-is-file") { + fs.rmSync(vault, { recursive: true }); + fs.writeFileSync(vault, "not a directory"); + } else if ( + ["unreadable-root", "partial-directory", "incremental-partial"].includes(scenario!) + ) { + const target = scenario === "unreadable-root" ? vault : join(vault, "nested"); + errorPath = scenario === "unreadable-root" ? "." : "nested"; + if (scenario !== "unreadable-root") { + note("new.md", "new"); + readableAddition = true; + } + const original = fs.readdirSync; + const spy = spyOn(fs, "readdirSync").mockImplementation((( + ...args: Parameters + ) => { + if (String(args[0]) === target) throw new Error("injected EACCES"); + return original(...args); + }) as typeof original); + restore = () => spy.mockRestore(); + } else if (["failed-stat", "failed-read"].includes(scenario!)) { + errorPath = "nested/b.md"; + note("new.md", "new"); + readableAddition = true; + const method = scenario === "failed-stat" ? "statSync" : "readFileSync"; + const original = fs[method]; + const spy = spyOn(fs, method).mockImplementation(((...args: unknown[]) => { + if (String(args[0]) === join(vault, errorPath)) throw new Error("injected EIO"); + return Reflect.apply(original, fs, args); + }) as typeof original); + restore = () => spy.mockRestore(); + } else if (["failed-upsert", "empty-note"].includes(scenario!)) { + fs.unlinkSync(join(vault, "c.md")); + errorPath = "new.md"; + if (scenario === "empty-note") fs.writeFileSync(join(vault, errorPath), ""); + else { + note("new.md", "new"); + const original = tursoVectorSearch.insertVector.bind(tursoVectorSearch); + const spy = spyOn(tursoVectorSearch, "insertVector").mockImplementation( + async (db, record) => { + if (record.id === "new") throw new Error("injected insert failure"); + return original(db, record); + } + ); + restore = () => spy.mockRestore(); + } + } else if (["healthy-majority-delete", "incremental-delete"].includes(scenario!)) { + fs.unlinkSync(join(vault, "a.md")); + fs.unlinkSync(join(vault, "nested/b.md")); + } else if (scenario === "incremental-rename") { + fs.renameSync(join(vault, "a.md"), join(vault, "renamed.md")); + } else if (scenario === "single-read") { + const original = fs.readFileSync; + let reads = 0; + const spy = spyOn(fs, "readFileSync").mockImplementation((( + ...args: Parameters + ) => { + if (String(args[0]) === join(vault, "a.md")) { + reads++; + if (reads > 1) return '---\nid: "different-id"\n---\nChanged during read'; + } + return original(...args); + }) as typeof original); + restore = () => { + spy.mockRestore(); + assert.equal(reads, 1, "full reconcile must read each file once"); + }; + } + + if (scenario!.startsWith("incremental-")) { + await reindexVaultFile(scenario === "incremental-partial" ? "nested/b.md" : "a.md"); + } else { + const result = + scenario === "startup-missing-root" + ? await startMarkdownBackend() + : await reconcileMarkdownVault(); + assert.ok(result); + if (!["healthy-majority-delete", "single-read"].includes(scenario!)) { + assert.equal(result.deleted, 0, "unsafe scan must not delete any indexed memory"); + assert.equal(result.deletionSkipped, reason); + if (reason === "incomplete-scan") + assert.ok(result.errors.some((e) => e.path === errorPath)); + if (readableAddition) assert.equal(result.upserted, 1); + } else if (scenario === "healthy-majority-delete") assert.equal(result.deleted, 2); + } + restore?.(); + restore = undefined; + if ( + ![ + "healthy-majority-delete", + "incremental-delete", + "incremental-rename", + "single-read", + ].includes(scenario!) + ) { + const warning = fs + .readFileSync(join(home!, "test.log"), "utf-8") + .split("\n") + .find((line) => + line.includes("markdown reconcile warning: deletion skipped; existing index retained") + ); + assert.ok(warning, "skipped deletion must emit a warning"); + assert.ok(warning.includes(JSON.stringify(reason)), "warning must identify the skip reason"); + } + if (scenario === "incremental-delete") { + assert.deepEqual(await ids(), ["b", "c", "unrelated"]); + assert.deepEqual((await snapshot()).counts, [4]); + await reindexVaultFile("nested/b.md"); + assert.deepEqual(await ids(), ["c", "unrelated"]); + const after = await snapshot(); + assert.deepEqual(after.counts, [3]); + assert.deepEqual( + after.rows.find((r) => r.id === "unrelated"), + before.rows.find((r) => r.id === "unrelated") + ); + } else if (scenario === "healthy-majority-delete") { + assert.deepEqual(await ids(), ["c"]); + assert.deepEqual((await snapshot()).counts, [2]); + } else if (scenario === "incremental-rename") { + assert.deepEqual(await ids(), ["a", "b", "c"]); + const row = (await snapshot()).rows.find((r) => r.id === "a")!; + assert.equal(JSON.parse(String(row.metadata)).markdownRelPath, "renamed.md"); + } else if (readableAddition) { + const after = await snapshot(); + assert.deepEqual( + after.rows.filter((r) => r.id !== "new"), + before.rows + ); + assert.deepEqual(after.counts, [5]); + } else + assert.deepEqual( + await snapshot(), + before, + "rows, metadata, trash and shard counts must be preserved" + ); + if (scenario === "restored-vault") { + fs.renameSync(`${vault}-saved`, vault); + fs.unlinkSync(join(vault, "a.md")); + note("new.md", "new"); + const result = await reconcileMarkdownVault(); + assert.equal(result.deletionSkipped, undefined); + assert.equal(result.deleted, 1); + assert.equal(result.upserted, 1); + assert.deepEqual(await ids(), ["b", "c", "new"]); + } + const trash = (await snapshot()).rows.find((r) => r.id === "trashed"); + assert.deepEqual( + trash, + before.rows.find((r) => r.id === "trashed") + ); + } +} finally { + restore?.(); + stopMarkdownBackend(); + await tursoConnectionManager.closeAll(); +} diff --git a/tests/markdown-reconcile-safety.test.ts b/tests/markdown-reconcile-safety.test.ts new file mode 100644 index 00000000..db9a131d --- /dev/null +++ b/tests/markdown-reconcile-safety.test.ts @@ -0,0 +1,68 @@ +import { describe, expect, it } from "bun:test"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +// Separate processes isolate CONFIG, the native database, and filesystem spies +// from the suite's shared module mocks. Every scenario uses actual shard storage. +describe("markdown reconciliation deletion safety", () => { + for (const scenario of [ + "missing-root", + "empty-root", + "no-markdown-files", + "startup-missing-root", + "root-is-file", + "unreadable-root", + "partial-directory", + "failed-stat", + "failed-read", + "failed-upsert", + "empty-note", + "healthy-majority-delete", + "fresh-empty", + "incremental-missing", + "incremental-empty", + "incremental-partial", + "incremental-delete", + "incremental-rename", + "single-read", + "restored-vault", + ]) { + it( + scenario, + async () => { + const home = mkdtempSync(join(tmpdir(), "md-reconcile-safety-")); + try { + const child = Bun.spawn( + [ + process.execPath, + join(import.meta.dir, "fixtures/markdown-reconcile-safety.ts"), + scenario, + home, + ], + { + env: { + ...process.env, + HOME: home, + USERPROFILE: home, + XDG_CONFIG_HOME: home, + OPENCODE_MEM_LOG_FILE: join(home, "test.log"), + }, + stdout: "pipe", + stderr: "pipe", + } + ); + const [stdout, stderr, code] = await Promise.all([ + new Response(child.stdout).text(), + new Response(child.stderr).text(), + child.exited, + ]); + expect(code, stdout + stderr).toBe(0); + } finally { + rmSync(home, { recursive: true, force: true }); + } + }, + 30_000 + ); + } +});