From 94c5a19139f96c9b3cc32e17f4e76bbece1d88a6 Mon Sep 17 00:00:00 2001 From: Luan Nguyen Date: Wed, 11 Nov 2026 21:27:26 +1100 Subject: [PATCH] feat(notes): add library metadata and passage anchor models --- web/src/library/model.ts | 373 ++++++++++++++++++ web/src/library/validation.ts | 480 +++++++++++++++++++++++ web/src/links/model.ts | 60 +++ web/src/links/scanner.ts | 187 +++++++++ web/src/study/sourceModel.ts | 245 ++++++++++++ web/tests/data/note-links-limits.test.ts | 121 ++++++ 6 files changed, 1466 insertions(+) create mode 100644 web/src/library/model.ts create mode 100644 web/src/library/validation.ts create mode 100644 web/src/links/model.ts create mode 100644 web/src/links/scanner.ts create mode 100644 web/src/study/sourceModel.ts create mode 100644 web/tests/data/note-links-limits.test.ts diff --git a/web/src/library/model.ts b/web/src/library/model.ts new file mode 100644 index 0000000..2919181 --- /dev/null +++ b/web/src/library/model.ts @@ -0,0 +1,373 @@ +import type { Note, RichNode } from "../domain/contracts"; +import type { + LibrarySnapshot, + LibraryState, + LibraryCommonState, + NoteVersion, + Tag, + TagColor, +} from "./contracts"; +import { LibraryCapacityError } from "./contracts"; +import { LIBRARY_LIMITS, jsonBytes } from "./validation"; + +export function emptyLibrary(revision = 0): LibraryState { + return { + version: 2, + epoch: crypto.randomUUID(), + revision, + tags: [], + noteMetadata: [], + versions: [], + preferences: { sort: "edited" }, + receipts: [], + legacyDuplicateTags: [], + anchors: [], + sourceReceipts: [], + }; +} +export function internTag( + state: LibraryState, + name: string, + now: string, + color: TagColor = null, +): Tag { + const existing = state.tags.find( + (tag) => tag.state === "active" && tag.name === name, + ); + if (existing) return existing; + if (state.tags.length >= LIBRARY_LIMITS.tags) + throw new LibraryCapacityError(); + const tag: Tag = { + id: crypto.randomUUID(), + name, + color, + revision: 1, + createdAt: now, + updatedAt: now, + state: "active", + mergedInto: null, + }; + state.tags.push(tag); + return tag; +} +export function syncMembership( + state: LibraryState, + note: Note, + now: string, +): void { + const tagIds = note.tags.map((name) => internTag(state, name, now).id); + const existing = state.noteMetadata.find((meta) => meta.noteId === note.id); + if (existing) { + if (JSON.stringify(existing.tagIds) !== JSON.stringify(tagIds)) { + existing.tagIds = tagIds; + existing.revision++; + } + } else + state.noteMetadata.push({ + noteId: note.id, + revision: 1, + pinned: false, + tagIds, + }); +} +export function adoptLibrary( + notes: Note[], + original: unknown[], + now: string, +): LibraryState { + const state = emptyLibrary(); + for (const note of notes) syncMembership(state, note, now); + for (const raw of original) { + const note = raw as Note; + if (new Set(note.tags).size !== note.tags.length) + state.legacyDuplicateTags.push({ + noteId: note.id, + values: [...note.tags], + }); + } + return state; +} +export function snapshot( + state: LibraryCommonState, + notes: Note[], +): LibrarySnapshot { + return structuredClone({ + token: { epoch: state.epoch, revision: state.revision }, + notes, + tags: state.tags, + metadata: state.noteMetadata, + preferences: state.preferences, + }); +} +export function sameDocument(a: unknown, b: unknown): boolean { + const canonical = (value: unknown) => + JSON.stringify(value, (_key, entry: unknown) => { + if (!entry || typeof entry !== "object" || Array.isArray(entry)) + return entry; + return Object.fromEntries( + Object.entries(entry).sort(([a], [b]) => a.localeCompare(b)), + ); + }); + return canonical(a) === canonical(b); +} +export function captureVersion( + state: LibraryState, + note: Note, + reason: NoteVersion["reason"], + now: string, +): string { + const existing = state.versions.find( + (version) => + version.noteId === note.id && version.noteRevision === note.revision, + ); + if (existing) return existing.id; + const meta = state.noteMetadata.find((meta) => meta.noteId === note.id)!; + const version: NoteVersion = { + id: crypto.randomUUID(), + noteId: note.id, + noteRevision: note.revision, + capturedAt: now, + reason, + title: note.title, + document: structuredClone(note.document), + tagsAtCapture: meta.tagIds.map((id) => { + const tag = state.tags.find((tag) => tag.id === id)!; + return { id: tag.id, name: tag.name, color: tag.color }; + }), + }; + state.versions.push(version); + return version.id; +} +export function retainHistory( + state: LibraryState, + now: number, + protectedId?: string, +): void { + const sorted = state.versions + .filter( + (version) => + version.id === protectedId || + Date.parse(version.capturedAt) >= now - LIBRARY_LIMITS.historyAgeMs, + ) + .sort( + (a, b) => + a.capturedAt.localeCompare(b.capturedAt) || a.id.localeCompare(b.id), + ); + const perNote = new Map(); + const sizes = new Map( + sorted.map((version) => [version.id, jsonBytes(version)]), + ); + for (const version of sorted) { + const group = perNote.get(version.noteId) ?? { count: 0, bytes: 0 }; + group.count++; + group.bytes += sizes.get(version.id)!; + perNote.set(version.noteId, group); + } + const removed = new Set(); + for (const version of sorted) { + const group = perNote.get(version.noteId)!; + if ( + (group.count > LIBRARY_LIMITS.noteVersions || + group.bytes > LIBRARY_LIMITS.noteHistoryBytes) && + version.id !== protectedId + ) { + removed.add(version.id); + group.count--; + group.bytes -= sizes.get(version.id)!; + } + } + let bytes = + 2 + + sorted + .filter((version) => !removed.has(version.id)) + .reduce((sum, version) => sum + sizes.get(version.id)! + 1, 0); + for (const version of sorted) { + if (bytes <= LIBRARY_LIMITS.historyBytes) break; + if (!removed.has(version.id) && version.id !== protectedId) { + removed.add(version.id); + bytes -= sizes.get(version.id)! + 1; + } + } + if ( + bytes > LIBRARY_LIMITS.historyBytes || + [...perNote.values()].some( + (group) => + group.count > LIBRARY_LIMITS.noteVersions || + group.bytes > LIBRARY_LIMITS.noteHistoryBytes, + ) + ) + throw new LibraryCapacityError(); + state.versions = sorted.filter((version) => !removed.has(version.id)); +} +export function removeNotes(state: LibraryState, ids: Set): void { + state.anchors = state.anchors.filter((anchor) => !ids.has(anchor.noteId)); + state.noteMetadata = state.noteMetadata.filter( + (meta) => !ids.has(meta.noteId), + ); + state.versions = state.versions.filter((version) => !ids.has(version.noteId)); + state.legacyDuplicateTags = state.legacyDuplicateTags.filter( + (item) => !ids.has(item.noteId), + ); + if (state.version === 3) + state.noteLinks = state.noteLinks.filter((row) => !ids.has(row.noteId)); + for (const receipt of state.receipts) + if (receipt.result.type === "note" && ids.has(receipt.result.noteId)) + receipt.result.undo = null; +} +export function retainReceipts( + state: LibraryState, + now: number, + protectedId?: string, +): void { + state.receipts = state.receipts + .filter( + (receipt) => + receipt.kind === "note-links-repair" || + Date.parse(receipt.committedAt) >= now - LIBRARY_LIMITS.receiptAgeMs, + ) + .sort( + (a, b) => + a.committedAt.localeCompare(b.committedAt) || a.id.localeCompare(b.id), + ); + while (state.receipts.length > LIBRARY_LIMITS.receipts) { + state.receipts.splice( + state.receipts.findIndex((receipt) => receipt.id !== protectedId), + 1, + ); + } +} +export function activeTag(state: LibraryState, id: string): Tag { + const tag = state.tags.find((tag) => tag.id === id && tag.state === "active"); + if (!tag) throw new Error("tag_missing: This tag is unavailable."); + return tag; +} +function reviseMemberships( + state: LibraryState, + notes: Note[], + sourceId: string, + targetId: string | null, + now: string, +): void { + for (const meta of state.noteMetadata) { + if (!meta.tagIds.includes(sourceId)) continue; + meta.tagIds = [ + ...new Set( + meta.tagIds.flatMap((id) => + id === sourceId ? (targetId ? [targetId] : []) : [id], + ), + ), + ]; + meta.revision++; + const note = notes.find((note) => note.id === meta.noteId)!; + note.tags = meta.tagIds.map((id) => activeTag(state, id).name); + note.revision++; + note.updatedAt = now; + } +} +export function renameTag( + state: LibraryState, + notes: Note[], + id: string, + name: string, + now: string, +): void { + const tag = activeTag(state, id); + if ( + state.tags.some( + (other) => + other.id !== id && other.state === "active" && other.name === name, + ) + ) + throw new Error( + "tag_name_conflict: That tag exists. Merge the tags explicitly instead.", + ); + tag.name = name; + tag.revision++; + tag.updatedAt = now; + reviseMemberships(state, notes, id, id, now); +} +export function mergeTags( + state: LibraryState, + notes: Note[], + sourceId: string, + targetId: string, + now: string, +): void { + if (sourceId === targetId) + throw new Error("tag_merge_invalid: Choose a different target tag."); + const source = activeTag(state, sourceId), + target = activeTag(state, targetId); + for (const tag of state.tags) + if (tag.state === "merged" && tag.mergedInto === sourceId) { + tag.mergedInto = targetId; + tag.revision++; + tag.updatedAt = now; + } + source.state = "merged"; + source.mergedInto = targetId; + source.revision++; + source.updatedAt = now; + target.revision++; + target.updatedAt = now; + reviseMemberships(state, notes, sourceId, targetId, now); +} +export function deleteTag( + state: LibraryState, + notes: Note[], + id: string, + now: string, +): void { + activeTag(state, id); + for (const tag of state.tags) + if (tag.id === id || (tag.state === "merged" && tag.mergedInto === id)) { + tag.state = "deleted"; + tag.mergedInto = null; + tag.revision++; + tag.updatedAt = now; + } + reviseMemberships(state, notes, id, null, now); +} +function plain(node: RichNode): string { + return [node.text ?? "", ...(node.content ?? []).map(plain)].join(" "); +} +export function projectLibraryNotes( + state: LibrarySnapshot, + options: { + query?: string; + tagId?: string; + trash?: boolean; + pinnedOnly?: boolean; + } = {}, +): Note[] { + const metadata = new Map(state.metadata.map((meta) => [meta.noteId, meta])), + query = options.query?.trim().toLocaleLowerCase() ?? ""; + return state.notes + .filter( + (note) => + Boolean(note.trashedAt) === Boolean(options.trash) && + (!options.pinnedOnly || metadata.get(note.id)?.pinned) && + (!options.tagId || + metadata.get(note.id)?.tagIds.includes(options.tagId)) && + (!query || + `${note.title} ${plain(note.document)} ${note.tags.join(" ")}` + .toLocaleLowerCase() + .includes(query)), + ) + .sort( + (a, b) => + (!options.trash + ? Number(metadata.get(b.id)?.pinned) - + Number(metadata.get(a.id)?.pinned) + : 0) || + (state.preferences.sort === "title" + ? a.title.localeCompare(b.title, "en", { sensitivity: "base" }) + : Date.parse( + state.preferences.sort === "created" ? b.createdAt : b.updatedAt, + ) - + Date.parse( + state.preferences.sort === "created" ? a.createdAt : a.updatedAt, + )) || + a.title.localeCompare(b.title) || + a.id.localeCompare(b.id), + ); +} diff --git a/web/src/library/validation.ts b/web/src/library/validation.ts new file mode 100644 index 0000000..af18804 --- /dev/null +++ b/web/src/library/validation.ts @@ -0,0 +1,480 @@ +import { validateLinkRows } from "../links/scanner"; +import type { Note } from "../domain/contracts"; +import { inspectNoteDocument } from "../domain/validation"; +import type { + LibraryState, + LibraryCommonState, + LegacyLibraryState, + LibraryToken, + NoteSort, + TagColor, + TrashUndo, +} from "./contracts"; +import { LibraryCapacityError } from "./contracts"; +import { + SOURCE_LIMITS, + validateSourceAnchor, + validateSourceReceipt, +} from "../study/sourceValidation"; +export const LEGACY_LIBRARY_BYTES = 64 * 1024 * 1024; +export const SOURCE_LIBRARY_OVERHEAD = new TextEncoder().encode( + ',"anchors":[],"sourceReceipts":[]', +).byteLength; +export const LINK_LIBRARY_OVERHEAD = new TextEncoder().encode( + ',"noteLinks":[]', +).byteLength; +export const SOURCE_LIBRARY_BYTES = + LEGACY_LIBRARY_BYTES + SOURCE_LIBRARY_OVERHEAD; +export const LIBRARY_LIMITS = { + tags: 10000, + notes: 10000, + bytes: SOURCE_LIBRARY_BYTES + LINK_LIBRARY_OVERHEAD, + historyBytes: 32 * 1024 * 1024, + noteHistoryBytes: 8 * 1024 * 1024, + noteVersions: 50, + historyAgeMs: 30 * 86400000, + receipts: 500, + receiptAgeMs: 600000, + undoMs: 5000, +} as const; +export function invalid(): never { + throw new Error( + "library_invalid: Library metadata is unsupported or corrupt. The saved copy was kept.", + ); +} +export function object( + value: unknown, + keys: string[], +): Record { + if ( + !value || + typeof value !== "object" || + Array.isArray(value) || + Object.getPrototypeOf(value) !== Object.prototype + ) + invalid(); + const row = value as Record; + if ( + Object.keys(row).length !== keys.length || + keys.some((key) => !Object.hasOwn(row, key)) + ) + invalid(); + return row; +} +export function uuid(value: unknown): asserts value is string { + if ( + typeof value !== "string" || + !/^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test( + value, + ) + ) + invalid(); +} +export function integer(value: unknown, min = 0): asserts value is number { + if (typeof value !== "number" || !Number.isSafeInteger(value) || value < min) + invalid(); +} +export function text(value: unknown, max: number): asserts value is string { + if (typeof value !== "string" || value.length > max) invalid(); +} +export function noteId(value: unknown): asserts value is string { + if (typeof value !== "string") invalid(); +} +export function timestamp( + value: unknown, + canonical = true, +): asserts value is string { + if ( + typeof value !== "string" || + !Number.isFinite(Date.parse(value)) || + (canonical && new Date(value).toISOString() !== value) + ) + invalid(); +} +export function color(value: unknown): asserts value is TagColor { + if ( + value !== null && + (typeof value !== "string" || !/^#[0-9a-f]{6}$/i.test(value)) + ) + invalid(); +} +export function sort(value: unknown): asserts value is NoteSort { + if (value !== "edited" && value !== "created" && value !== "title") invalid(); +} +export function token(value: unknown): LibraryToken { + const row = object(value, ["epoch", "revision"]); + uuid(row.epoch); + integer(row.revision); + return { epoch: row.epoch, revision: row.revision }; +} +export function tagName(value: unknown): string { + text(value, 100); + const name = value.trim(); + if (!name) throw new Error("tag_name_invalid: Enter a tag name."); + return name; +} +export function jsonBytes(value: unknown): number { + return new TextEncoder().encode(JSON.stringify(value)).byteLength; +} +function array(value: unknown, max: number): asserts value is unknown[] { + if (!Array.isArray(value)) invalid(); + if (value.length > max) throw new LibraryCapacityError(); +} +export function undo(value: unknown): TrashUndo { + const row = object(value, [ + "epoch", + "operationId", + "noteId", + "resultRevision", + "previousTrashedAt", + "resultTrashedAt", + "expiresAt", + ]); + uuid(row.epoch); + uuid(row.operationId); + noteId(row.noteId); + integer(row.resultRevision, 1); + timestamp(row.expiresAt); + if (row.previousTrashedAt !== null) timestamp(row.previousTrashedAt, false); + if (row.resultTrashedAt !== null) timestamp(row.resultTrashedAt, false); + const checked = structuredClone(value as TrashUndo); + return { + epoch: checked.epoch, + operationId: checked.operationId, + noteId: checked.noteId, + resultRevision: checked.resultRevision, + previousTrashedAt: checked.previousTrashedAt, + resultTrashedAt: checked.resultTrashedAt, + expiresAt: checked.expiresAt, + }; +} +function validateLibraryState( + value: unknown, + notes: Note[], + legacy: boolean, + recoverIndex = false, +): unknown { + const linked = + !legacy && (value as { version?: unknown } | null)?.version === 3; + const state = object(value, [ + "version", + "epoch", + "revision", + "tags", + "noteMetadata", + "versions", + "preferences", + "receipts", + "legacyDuplicateTags", + ...(legacy ? [] : ["anchors", "sourceReceipts"]), + ...(linked ? ["noteLinks"] : []), + ]); + if (state.version !== (legacy ? 1 : linked ? 3 : 2)) invalid(); + uuid(state.epoch); + integer(state.revision); + array(state.tags, LIBRARY_LIMITS.tags); + array(state.noteMetadata, LIBRARY_LIMITS.notes); + array(state.versions, LIBRARY_LIMITS.notes * LIBRARY_LIMITS.noteVersions); + array(state.receipts, LIBRARY_LIMITS.receipts); + array(state.legacyDuplicateTags, LIBRARY_LIMITS.notes); + const prefs = object(state.preferences, ["sort"]); + sort(prefs.sort); + const tags = new Map>(), + names = new Set(); + for (const item of state.tags) { + const row = object(item, [ + "id", + "name", + "color", + "revision", + "createdAt", + "updatedAt", + "state", + "mergedInto", + ]); + uuid(row.id); + if (tags.has(row.id)) invalid(); + text(row.name, 100); + color(row.color); + integer(row.revision, 1); + timestamp(row.createdAt); + timestamp(row.updatedAt); + if (Date.parse(row.updatedAt) < Date.parse(row.createdAt)) invalid(); + if (!["active", "deleted", "merged"].includes(row.state as string)) + invalid(); + if (row.state === "merged") uuid(row.mergedInto); + else if (row.mergedInto !== null) invalid(); + if (row.state === "active") { + if (names.has(row.name)) invalid(); + names.add(row.name); + } + tags.set(row.id, row); + } + for (const tag of tags.values()) + if ( + tag.state === "merged" && + (tag.id === tag.mergedInto || + tags.get(tag.mergedInto as string)?.state !== "active") + ) + invalid(); + const current = new Map(notes.map((note) => [note.id, note])), + seen = new Set(); + for (const item of state.noteMetadata) { + const row = object(item, ["noteId", "revision", "pinned", "tagIds"]); + noteId(row.noteId); + integer(row.revision, 1); + if ( + typeof row.pinned !== "boolean" || + seen.has(row.noteId) || + !current.has(row.noteId) + ) + invalid(); + seen.add(row.noteId); + array(row.tagIds, 100); + const ids = new Set(); + const labels: string[] = []; + for (const id of row.tagIds) { + uuid(id); + if (ids.has(id) || tags.get(id)?.state !== "active") invalid(); + ids.add(id); + labels.push(tags.get(id)!.name as string); + } + if ( + JSON.stringify(labels) !== JSON.stringify(current.get(row.noteId)!.tags) + ) + invalid(); + } + if (seen.size !== current.size) invalid(); + const versionIds = new Set(), + revisions = new Set(), + perNote = new Map(); + for (const item of state.versions) { + const row = object(item, [ + "id", + "noteId", + "noteRevision", + "capturedAt", + "reason", + "title", + "document", + "tagsAtCapture", + ]); + uuid(row.id); + noteId(row.noteId); + integer(row.noteRevision, 1); + timestamp(row.capturedAt); + text(row.title, 500); + if ( + versionIds.has(row.id) || + !current.has(row.noteId) || + row.noteRevision > current.get(row.noteId)!.revision || + (row.reason !== "edit" && row.reason !== "history-restore") + ) + invalid(); + versionIds.add(row.id); + const revision = JSON.stringify([row.noteId, row.noteRevision]); + if (revisions.has(revision)) invalid(); + revisions.add(revision); + inspectNoteDocument(row.document, linked ? "notes" : "plain"); + array(row.tagsAtCapture, 100); + const historical = new Set(); + for (const item of row.tagsAtCapture) { + const tag = object(item, ["id", "name", "color"]); + uuid(tag.id); + text(tag.name, 100); + color(tag.color); + if (!tags.has(tag.id) || historical.has(tag.id)) invalid(); + historical.add(tag.id); + } + const size = perNote.get(row.noteId) ?? { count: 0, bytes: 0 }; + size.count++; + size.bytes += jsonBytes(row); + perNote.set(row.noteId, size); + if ( + size.count > LIBRARY_LIMITS.noteVersions || + size.bytes > LIBRARY_LIMITS.noteHistoryBytes + ) + throw new LibraryCapacityError(); + } + if (jsonBytes(state.versions) > LIBRARY_LIMITS.historyBytes) + throw new LibraryCapacityError(); + const receiptIds = new Set(); + for (const item of state.receipts) { + const row = object(item, [ + "id", + "epoch", + "committedAt", + "libraryRevision", + "kind", + "arguments", + "result", + ]); + uuid(row.id); + uuid(row.epoch); + integer(row.libraryRevision, 1); + timestamp(row.committedAt); + text(row.arguments, 2048); + if ( + row.epoch !== state.epoch || + row.libraryRevision > state.revision || + receiptIds.has(row.id) || + ![ + "pin", + "sort", + "tag-create", + "tag-rename", + "tag-color", + "tag-merge", + "tag-delete", + "history-restore", + "trash", + "restore", + "trash-undo", + "empty-trash", + "legacy-erase", + ...(linked ? ["note-links-repair"] : []), + ].includes(row.kind as string) + ) + invalid(); + receiptIds.add(row.id); + try { + JSON.parse(row.arguments); + } catch { + invalid(); + } + const result = row.result as Record | null; + if (!result) invalid(); + if (row.kind === "note-links-repair") { + object(result, ["type", "indexDigest"]); + if ( + result.type !== "note-links-repair" || + typeof result.indexDigest !== "string" || + !/^[a-f0-9]{64}$/.test(result.indexDigest) + ) + invalid(); + const args = object(JSON.parse(row.arguments as string), [ + "revision", + "rawIndexDigest", + "documentsDigest", + "indexDigest", + ]); + integer(args.revision); + if ((args.revision as number) + 1 !== row.libraryRevision) invalid(); + for (const key of ["rawIndexDigest", "documentsDigest", "indexDigest"]) + if ( + typeof args[key] !== "string" || + !/^[a-f0-9]{64}$/.test(args[key] as string) + ) + invalid(); + if (args.indexDigest !== result.indexDigest) invalid(); + } else if (result.type === "metadata") object(result, ["type"]); + else if (result.type === "tag") { + object(result, ["type", "tagId"]); + uuid(result.tagId); + if (!tags.has(result.tagId)) invalid(); + } else if (result.type === "empty-trash") { + object(result, ["type", "removedCount"]); + integer(result.removedCount); + if (result.removedCount > LIBRARY_LIMITS.notes) invalid(); + } else if (result.type === "note") { + object(result, ["type", "noteId", "resultRevision", "undo"]); + noteId(result.noteId); + integer(result.resultRevision, 1); + if (result.undo !== null) { + const checked = undo(result.undo); + if ( + checked.epoch !== state.epoch || + checked.noteId !== result.noteId || + checked.resultRevision !== result.resultRevision || + checked.operationId !== row.id + ) + invalid(); + } + } else invalid(); + } + const legacyIds = new Set(); + for (const item of state.legacyDuplicateTags) { + const row = object(item, ["noteId", "values"]); + noteId(row.noteId); + array(row.values, 100); + if (!current.has(row.noteId) || legacyIds.has(row.noteId)) invalid(); + legacyIds.add(row.noteId); + for (const value of row.values) text(value, 100); + } + if (!legacy) { + array(state.anchors, SOURCE_LIMITS.anchors); + array(state.sourceReceipts, SOURCE_LIMITS.receipts); + const anchors = new Set(), + entries = new Set(), + receipts = new Set(); + for (const raw of state.anchors) { + const anchor = validateSourceAnchor(raw); + const note = current.get(anchor.noteId); + if ( + anchors.has(anchor.id) || + entries.has(anchor.entryId) || + !note || + anchor.noteRevision > note.revision || + (anchor.status === "attached" && anchor.noteRevision !== note.revision) + ) + invalid(); + anchors.add(anchor.id); + entries.add(anchor.entryId); + } + for (const raw of state.sourceReceipts) { + const receipt = validateSourceReceipt(raw); + if (receipts.has(receipt.operationId)) invalid(); + receipts.add(receipt.operationId); + } + } + if (linked && !recoverIndex) validateLinkRows(state.noteLinks, notes); + if ( + jsonBytes(value) > + (legacy + ? LEGACY_LIBRARY_BYTES + : linked + ? LIBRARY_LIMITS.bytes + : SOURCE_LIBRARY_BYTES) + ) + throw new LibraryCapacityError(); + return structuredClone(value); +} +export function validateLegacyLibrary( + value: unknown, + notes: Note[], +): LegacyLibraryState { + return validateLibraryState(value, notes, true) as LegacyLibraryState; +} +export function validateLibrary(value: unknown, notes: Note[]): LibraryState { + return validateLibraryState(value, notes, false) as LibraryState; +} +export function validateLibraryCommon( + value: unknown, + notes: Note[], +): LibraryCommonState { + if (!value || typeof value !== "object" || Array.isArray(value)) invalid(); + const raw = value as Record; + // Every common field is validated above; the untrusted isolated index is + // deliberately omitted rather than cast to a typed LibraryState. + const checked = validateLibraryState( + Object.hasOwn(raw, "noteLinks") ? raw : { ...raw, noteLinks: undefined }, + notes, + false, + true, + ) as LibraryCommonState & { noteLinks: unknown }; + const { noteLinks: _untrustedIndex, ...common } = checked; + return common; +} +export function normalizeLibrary(value: unknown, notes: Note[]): LibraryState { + if ((value as { version?: unknown } | null)?.version === 1) + return validateLibrary( + { + ...validateLegacyLibrary(value, notes), + version: 2, + anchors: [], + sourceReceipts: [], + }, + notes, + ); + return validateLibrary(value, notes); +} diff --git a/web/src/links/model.ts b/web/src/links/model.ts new file mode 100644 index 0000000..26cbd4e --- /dev/null +++ b/web/src/links/model.ts @@ -0,0 +1,60 @@ +import type { Note } from "../domain/contracts"; +import type { LibraryCommonState } from "../library/contracts"; +import type { LinkToken, NoteLinksRow } from "./contracts"; + +export type LinkOwnerSnapshot = { + accountId: string; + generation: number; + rawVersion: 1 | 2 | 3; + notes: Note[]; + library: LibraryCommonState; + index: { status: "valid"; rows: NoteLinksRow[] } | { status: "invalid" }; + rawIndex: string; + recordIdentity: Readonly<{ core: string; library: string }>; + knownMismatch: boolean; +}; +export type LinkCertificate = { + snapshot: LinkOwnerSnapshot; + rows: NoteLinksRow[]; + documentDigests: [string, number, string][]; + snippets: [string, string][]; +}; +export type LinkRepairCommand = { + operationId: string; + arguments: string; + indexDigest: string; + certificate: LinkCertificate; +}; +export interface LinkOwnerAccess { + read(): Promise; + authorizeLegacy( + certificate: LinkCertificate, + fresh: () => void, + ): Promise; + mismatch(snapshot: LinkOwnerSnapshot): void; + repair( + command: LinkRepairCommand, + fresh: () => void, + ): Promise<{ committedToken: LinkToken; replayed: boolean }>; +} +export const linkToken = (state: LinkOwnerSnapshot): LinkToken => ({ + epoch: state.library.epoch, + revision: state.library.revision, +}); +export function sameLinkToken(a: LinkToken, b: LinkToken): boolean { + return a.epoch === b.epoch && a.revision === b.revision; +} +export function sameOwnerState( + a: LinkOwnerSnapshot, + b: LinkOwnerSnapshot, +): boolean { + return ( + a.accountId === b.accountId && + a.generation === b.generation && + a.rawVersion === b.rawVersion && + sameLinkToken(linkToken(a), linkToken(b)) && + a.rawIndex === b.rawIndex && + a.recordIdentity.core === b.recordIdentity.core && + a.recordIdentity.library === b.recordIdentity.library + ); +} diff --git a/web/src/links/scanner.ts b/web/src/links/scanner.ts new file mode 100644 index 0000000..4625965 --- /dev/null +++ b/web/src/links/scanner.ts @@ -0,0 +1,187 @@ +import type { NoteDocument, RichNode } from "../domain/contracts"; +import { inspectNoteDocument } from "../domain/validation"; +import { NoteLinkError, type NoteLinksRow, type NoteTarget } from "./contracts"; +import { validateNoteTarget } from "./reference"; + +export const NOTE_LINK_LIMITS = Object.freeze({ + references: 1000, + totalReferences: 100000, + rows: 10000, + documentBytes: 8 * 1024 * 1024, + documentNodes: 50000, + workBytes: 64 * 1024 * 1024, + workNodes: 250000, + yieldNodes: 2048, + searchUnits: 5000000, +}); +export type LinkWork = { utf8Units: number; nodeVisits: number }; +export const targetKey = (target: NoteTarget): string => + JSON.stringify([target.accountId, target.libraryEpoch, target.noteId]); +export function charge(work: LinkWork, units: number, nodes: number): void { + if ( + work.utf8Units + units > NOTE_LINK_LIMITS.workBytes || + work.nodeVisits + nodes > NOTE_LINK_LIMITS.workNodes + ) + throw new NoteLinkError("note_link_verification_limit"); + work.utf8Units += units; + work.nodeVisits += nodes; +} +export function inspectLinkDocument( + document: unknown, + policy: "plain" | "notes", + work: LinkWork, +): NoteDocument { + const inspected = inspectNoteDocument(document, policy); + if ( + inspected.jsonUnits * 3 > NOTE_LINK_LIMITS.documentBytes || + inspected.nodeVisits > NOTE_LINK_LIMITS.documentNodes + ) + throw new NoteLinkError("note_link_limit"); + charge(work, inspected.jsonUnits * 3, inspected.nodeVisits); + return inspected.document; +} +export function scanLinkDocument( + document: NoteDocument, + work: LinkWork, +): NoteLinksRow["targets"] { + const found = new Map(); + const queue: RichNode[] = [document]; + let count = 0; + while (queue.length) { + const node = queue.pop()!; + charge(work, 0, 1); + if (node.type === "noteLink") { + const target = validateNoteTarget(node.attrs?.target); + if (++count > NOTE_LINK_LIMITS.references) + throw new NoteLinkError("note_link_limit"); + const key = targetKey(target), + prior = found.get(key); + if (prior) prior.occurrences++; + else found.set(key, { target, occurrences: 1 }); + } + if (node.content) + for (let i = node.content.length - 1; i >= 0; i--) + queue.push(node.content[i]!); + } + return [...found.entries()] + .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)) + .map(([, row]) => row); +} +export function validateLinkRows( + input: unknown, + notes: readonly { id: string; revision: number }[], +): NoteLinksRow[] { + if (!Array.isArray(input) || input.length > NOTE_LINK_LIMITS.rows) + throw new NoteLinkError("note_link_index_invalid"); + const current = new Map(notes.map((note) => [note.id, note.revision])), + seen = new Set(); + let total = 0; + const rows: NoteLinksRow[] = []; + for (const value of input) { + if ( + !value || + typeof value !== "object" || + Object.keys(value).sort().join() !== "noteId,noteRevision,targets" || + typeof value.noteId !== "string" || + current.get(value.noteId) !== value.noteRevision || + seen.has(value.noteId) || + !Array.isArray(value.targets) || + value.targets.length > NOTE_LINK_LIMITS.references + ) + throw new NoteLinkError("note_link_index_invalid"); + seen.add(value.noteId); + const targets: NoteLinksRow["targets"] = [], + keys = new Set(); + let perNote = 0; + for (const entry of value.targets) { + if ( + !entry || + typeof entry !== "object" || + Object.keys(entry).sort().join() !== "occurrences,target" || + !Number.isSafeInteger(entry.occurrences) || + entry.occurrences < 1 + ) + throw new NoteLinkError("note_link_index_invalid"); + const target = validateNoteTarget(entry.target), + key = targetKey(target); + if (keys.has(key)) throw new NoteLinkError("note_link_index_invalid"); + keys.add(key); + perNote += entry.occurrences; + total += entry.occurrences; + if ( + perNote > NOTE_LINK_LIMITS.references || + total > NOTE_LINK_LIMITS.totalReferences + ) + throw new NoteLinkError("note_link_limit"); + targets.push({ target, occurrences: entry.occurrences }); + } + rows.push({ + noteId: value.noteId, + noteRevision: value.noteRevision, + targets, + }); + } + // Rows with no references are omitted. Completeness is a document-proof + // property and deliberately not inferred by this structural validator. + if (rows.some((row) => row.targets.length === 0)) + throw new NoteLinkError("note_link_index_invalid"); + return rows; +} +export function sameTargets( + a: NoteLinksRow["targets"], + b: NoteLinksRow["targets"], +): boolean { + if (a.length !== b.length) return false; + const map = new Map(a.map((row) => [targetKey(row.target), row.occurrences])); + return b.every((row) => map.get(targetKey(row.target)) === row.occurrences); +} + +export async function scanLinkDocumentAsync( + document: NoteDocument, + work: LinkWork, + fresh: () => void, + snippet?: { text: string }, +): Promise { + const found = new Map(), + queue: RichNode[] = [document]; + let count = 0, + sinceYield = 0, + started = performance.now(); + while (queue.length) { + fresh(); + const node = queue.pop()!; + charge(work, 0, 1); + if (snippet && snippet.text.length < 160) + snippet.text += ( + node.type === "noteLink" + ? "Note reference" + : node.type === "hardBreak" + ? "\n" + : (node.text ?? "") + ).slice(0, 160 - snippet.text.length); + if (node.type === "noteLink") { + const target = validateNoteTarget(node.attrs?.target), + key = targetKey(target), + prior = found.get(key); + if (++count > NOTE_LINK_LIMITS.references) + throw new NoteLinkError("note_link_limit"); + if (prior) prior.occurrences++; + else found.set(key, { target, occurrences: 1 }); + } + if (node.content) + for (let i = node.content.length - 1; i >= 0; i--) + queue.push(node.content[i]!); + if ( + ++sinceYield >= NOTE_LINK_LIMITS.yieldNodes || + performance.now() - started >= 8 + ) { + await new Promise((resolve) => setTimeout(resolve, 0)); + fresh(); + sinceYield = 0; + started = performance.now(); + } + } + return [...found.entries()] + .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)) + .map(([, row]) => row); +} diff --git a/web/src/study/sourceModel.ts b/web/src/study/sourceModel.ts new file mode 100644 index 0000000..754cb3f --- /dev/null +++ b/web/src/study/sourceModel.ts @@ -0,0 +1,245 @@ +import type { StudyEntry } from "../collections/contracts"; +import { saveEntry, deleteEntry } from "./model"; +import { initialSchedule } from "./scheduler"; +import { canonicalSourceDocument, captureSourceSelection } from "./sourceCodec"; +import { SourceError } from "./sourceContracts"; +import type { + AnchoredStudySource, + SourceAnchor, + SourceCommand, + SourceExpected, + SourceOwnerSnapshot, + SourceReceipt, + SourceResolution, +} from "./sourceContracts"; +import { SOURCE_LIMITS } from "./sourceValidation"; + +export function sourceTokens(state: SourceOwnerSnapshot): SourceExpected { + return { + library: { epoch: state.library.epoch, revision: state.library.revision }, + collections: { + epoch: state.collections.epoch, + revision: state.collections.revision, + }, + }; +} +export function resolveSource( + state: SourceOwnerSnapshot, + entryId: string, + documentDigest: string, +): SourceResolution { + const entry = state.collections.study.entries.find( + (entry) => entry.id === entryId, + ), + source = entry?.source; + if (!entry || !source) return { status: "unavailable", reason: "missing" }; + if (!("version" in source)) + return { status: "unavailable", reason: "legacy" }; + const anchor = state.library.anchors.find( + (anchor) => anchor.id === source.anchorId, + ); + const note = state.notes.find((note) => note.id === source.noteId); + if (!note || !anchor) return { status: "unavailable", reason: "missing" }; + if (note.trashedAt !== null) + return { status: "unavailable", reason: "trashed" }; + if ( + source.libraryEpoch !== state.library.epoch || + anchor.entryId !== entryId || + anchor.noteId !== source.noteId || + anchor.birth.noteRevision !== source.noteRevision || + anchor.quote !== source.quote || + anchor.status !== "attached" || + anchor.noteRevision !== note.revision || + anchor.documentDigest !== documentDigest + ) + return { status: "unavailable", reason: "stale" }; + try { + if ( + captureSourceSelection(note.document, anchor.from, anchor.to).passage + .quote !== anchor.quote + ) + return { status: "unavailable", reason: "stale" }; + } catch { + return { status: "unavailable", reason: "stale" }; + } + return { + status: "attached", + entryId, + anchorId: anchor.id, + anchorRevision: anchor.revision, + noteId: note.id, + noteRevision: note.revision, + documentDigest, + from: anchor.from, + to: anchor.to, + quote: anchor.quote, + }; +} +export function applySourceCommand( + state: SourceOwnerSnapshot, + command: SourceCommand, +): SourceReceipt { + let receipt: SourceReceipt; + if (command.kind === "create") { + const selection = command.selection, + note = state.notes.find((note) => note.id === selection.noteId); + if ( + !note || + note.trashedAt !== null || + note.revision !== selection.noteRevision || + state.library.epoch !== selection.libraryEpoch || + canonicalSourceDocument(note.document).canonicalJSON !== + selection.fingerprint.canonicalJSON || + captureSourceSelection( + note.document, + selection.passage.from, + selection.passage.to, + ).passage.quote !== selection.passage.quote + ) + throw new SourceError( + "source_stale", + "The selected passage changed. Select it again.", + ); + if (state.library.anchors.length >= SOURCE_LIMITS.anchors) + throw new SourceError("source_limit", "Source annotations are full."); + if ( + state.library.anchors.some( + (anchor) => + anchor.id === command.anchorId || anchor.entryId === command.entryId, + ) || + state.collections.study.entries.some( + (entry) => entry.id === command.entryId, + ) + ) + throw new SourceError( + "source_operation_reused", + "This card identity is already in use.", + ); + const source: AnchoredStudySource = { + version: 2, + noteId: note.id, + noteRevision: note.revision, + quote: selection.passage.quote, + libraryEpoch: state.library.epoch, + anchorId: command.anchorId, + }; + const entry: StudyEntry = { + ...structuredClone(command.draft), + id: command.entryId, + revision: 1, + source, + }; + const at = new Date(); + const nextStudy = saveEntry(state.collections.study, entry, at); + const cards = nextStudy.cards.filter((card) => card.entryId === entry.id); + if (cards.length !== command.cardIds.length) + throw new SourceError( + "source_invalid", + "Derived card identities do not match the selected type.", + ); + cards.forEach((card, i) => { + card.id = command.cardIds[i]!; + }); + state.collections.study = nextStudy; + const anchor: SourceAnchor = { + id: command.anchorId, + revision: 1, + noteId: note.id, + entryId: entry.id, + birth: { + noteRevision: note.revision, + documentDigest: selection.fingerprint.digest, + from: selection.passage.from, + to: selection.passage.to, + }, + noteRevision: note.revision, + from: selection.passage.from, + to: selection.passage.to, + quote: selection.passage.quote, + documentDigest: selection.fingerprint.digest, + status: "attached", + staleReason: null, + }; + state.library.anchors.push(anchor); + receipt = { + operationId: command.operationId, + digest: command.digest, + kind: "create", + at: at.toISOString(), + entryId: entry.id, + cardIds: command.cardIds.slice(), + anchorId: anchor.id, + createOperationId: command.operationId, + entryRevision: 1, + cardRevisions: cards.map(() => 1), + anchorRevision: 1, + }; + } else { + const original = state.library.sourceReceipts.find( + (row) => row.operationId === command.original.operationId, + ); + const entry = state.collections.study.entries.find( + (entry) => entry.id === command.entryId, + ), + anchor = state.library.anchors.find( + (anchor) => anchor.id === command.original.anchorId, + ); + const cards = state.collections.study.cards.filter( + (card) => card.entryId === command.entryId, + ), + now = Date.now(); + const schedule = original ? initialSchedule(new Date(original.at)) : null; + const matchingCards = + original && + schedule && + cards.length === original.cardIds.length && + original.cardIds.every((id) => + cards.some( + (card) => + card.id === id && + card.revision === 1 && + (Object.keys(schedule) as (keyof typeof schedule)[]).every( + (key) => card.schedule[key] === schedule[key], + ), + ), + ); + if ( + !original || + original.kind !== "create" || + JSON.stringify(original) !== JSON.stringify(command.original) || + !entry || + entry.revision !== 1 || + !anchor || + anchor.revision !== 1 || + anchor.entryId !== entry.id || + !matchingCards || + resolveSource(state, entry.id, anchor.documentDigest).status !== + "attached" || + state.collections.study.events.some((event) => + original.cardIds.includes(event.cardId), + ) || + now < Date.parse(original.at) || + now > Date.parse(original.at) + 5000 + ) + throw new SourceError( + "source_undo_unavailable", + "This card or source changed, or Undo expired.", + ); + state.collections.study = deleteEntry(state.collections.study, entry.id); + state.library.anchors = state.library.anchors.filter( + (row) => row.id !== anchor.id, + ); + receipt = { + ...structuredClone(original), + operationId: command.operationId, + digest: command.digest, + kind: "undo", + at: new Date(now).toISOString(), + }; + } + state.library.sourceReceipts.push(receipt); + state.library.sourceReceipts = state.library.sourceReceipts.slice( + -SOURCE_LIMITS.receipts, + ); + return receipt; +} diff --git a/web/tests/data/note-links-limits.test.ts b/web/tests/data/note-links-limits.test.ts new file mode 100644 index 0000000..b67ca30 --- /dev/null +++ b/web/tests/data/note-links-limits.test.ts @@ -0,0 +1,121 @@ +// @vitest-environment node +import { expect, it } from "vitest"; +import type { Note } from "../../src/domain/contracts"; +import { emptyLibrary } from "../../src/library/model"; +import { + validateLegacyLibrary, + validateLibrary, + normalizeLibrary, + jsonBytes, + LEGACY_LIBRARY_BYTES, + SOURCE_LIBRARY_BYTES, + SOURCE_LIBRARY_OVERHEAD, + LINK_LIBRARY_OVERHEAD, + LIBRARY_LIMITS, +} from "../../src/library/validation"; +import { scanLinkDocument, validateLinkRows } from "../../src/links/scanner"; +const note: Note = { + id: "id", + title: "", + document: { type: "doc", content: [] }, + tags: [], + createdAt: "2026-11-11T00:00:00.000Z", + updatedAt: "2026-11-11T00:00:00.000Z", + trashedAt: null, + revision: 1, +}; +const target = { + kind: "note" as const, + accountId: "11111111-1111-4111-8111-111111111111", + libraryEpoch: "22222222-2222-4222-8222-222222222222", + noteId: "opaque", +}; +it("keeps exact cumulative raw1/raw2/raw3 overhead and rejects nonempty/digit-growth overflow", () => { + const state = emptyLibrary(); + state.noteMetadata = [{ noteId: "", revision: 1, pinned: false, tagIds: [] }]; + const { anchors, sourceReceipts, ...common } = state; + const raw1 = { ...common, version: 1 }; + const length = LEGACY_LIBRARY_BYTES - jsonBytes(raw1), + id = "x".repeat(length), + notes = [{ ...note, id }]; + raw1.noteMetadata[0]!.noteId = id; + expect(jsonBytes(raw1)).toBe(LEGACY_LIBRARY_BYTES); + expect(validateLegacyLibrary(raw1, notes).version).toBe(1); + const raw2 = normalizeLibrary(raw1, notes); + expect(raw2.version).toBe(2); + expect(jsonBytes(raw2)).toBe(SOURCE_LIBRARY_BYTES); + expect(SOURCE_LIBRARY_BYTES - LEGACY_LIBRARY_BYTES).toBe( + SOURCE_LIBRARY_OVERHEAD, + ); + const raw3 = { ...raw2, version: 3 as const, noteLinks: [] }; + expect(jsonBytes(raw3)).toBe(LIBRARY_LIMITS.bytes); + expect(LIBRARY_LIMITS.bytes - SOURCE_LIBRARY_BYTES).toBe( + LINK_LIBRARY_OVERHEAD, + ); + expect(validateLibrary(raw3, notes).version).toBe(3); + expect(() => validateLegacyLibrary(raw3, notes)).toThrow(); + expect(() => validateLibrary({ ...raw2, revision: 10 }, notes)).toThrow( + expect.objectContaining({ code: "library_capacity" }), + ); + expect(() => + validateLibrary( + { + ...raw3, + noteLinks: [ + { + noteId: id, + noteRevision: 1, + targets: [{ target, occurrences: 1 }], + }, + ], + }, + notes, + ), + ).toThrow(expect.objectContaining({ code: "library_capacity" })); +}, 15000); +it("enforces occurrence and total-graph limits with no silently dropped references", () => { + const document = { + type: "doc" as const, + content: [ + { + type: "paragraph", + content: Array.from({ length: 1001 }, () => ({ + type: "noteLink", + attrs: { version: 1, target }, + })), + }, + ], + }; + expect(() => + scanLinkDocument(document, { utf8Units: 0, nodeVisits: 0 }), + ).toThrow(expect.objectContaining({ code: "note_link_limit" })); + const notes = Array.from({ length: 101 }, (_, i) => ({ + id: String(i), + revision: 1, + })); + expect(() => + validateLinkRows( + notes.map((note) => ({ + noteId: note.id, + noteRevision: 1, + targets: [{ target, occurrences: 1000 }], + })), + notes, + ), + ).toThrow(expect.objectContaining({ code: "note_link_limit" })); + expect(() => + validateLinkRows( + [ + { + noteId: "0", + noteRevision: 1, + targets: [ + { target, occurrences: 1 }, + { target, occurrences: 1 }, + ], + }, + ], + notes, + ), + ).toThrow(expect.objectContaining({ code: "note_link_index_invalid" })); +});