diff --git a/web/src/clips/contracts.ts b/web/src/clips/contracts.ts new file mode 100644 index 0000000..bafda9e --- /dev/null +++ b/web/src/clips/contracts.ts @@ -0,0 +1,115 @@ +import type { PersistenceState } from "../domain/contracts"; +import type { CollectionsRevision } from "../collections/contracts"; + +export type Clip = { + id: string; + revision: number; + originalUrl: string; + canonicalUrl: string; + title: string; + text: string | null; + createdAt: string; + updatedAt: string; + readAt: string | null; + readLater: boolean; + archivedAt: string | null; +}; +export type ClipActionInput = + | { + kind: "create"; + url: string; + title: string; + text: string; + readLater: boolean; + allowDuplicate: boolean; + } + | { kind: "set-read"; clipId: string; targetRevision: number; read: boolean } + | { + kind: "set-read-later"; + clipId: string; + targetRevision: number; + readLater: boolean; + } + | { + kind: "set-archived"; + clipId: string; + targetRevision: number; + archived: boolean; + } + | { kind: "undo-create"; receiptOperationId: string; targetRevision: number }; +export type ClipReceipt = { + operationId: string; + digest: string; + kind: ClipActionInput["kind"]; + clipId: string; + resultRevision: number | null; + at: string; + changed: boolean; +}; +export type ClipState = { items: Clip[]; receipts: ClipReceipt[] }; +export type ClipSnapshot = { + epoch: string; + revision: number; + clips: ClipState; +}; +declare const intentBrand: unique symbol; +export type ClipIntent = Readonly<{ + operationId: string; + clipId: string; + [intentBrand]: true; +}>; +export type ClipMutationResult = { + snapshot: ClipSnapshot; + receipt: ClipReceipt; + changed: boolean; + replayed: boolean; +}; +export interface ClipsStore { + read(): Promise; + prepare( + expected: CollectionsRevision, + input: ClipActionInput, + ): Promise; + commit(intent: ClipIntent): Promise; + subscribe(listener: () => void): () => void; + persistence(): PersistenceState; + onInvalidate(listener: (reason: "epoch" | "closed") => void): () => void; + close(): void; +} +export type ClipErrorCode = + | "clip_invalid" + | "clip_limit" + | "clip_stale" + | "clip_intent_revoked" + | "clip_undo_unavailable" + | "clip_operation_reused"; +export class ClipError extends Error { + constructor( + public readonly code: ClipErrorCode, + message: string, + ) { + super(`${code}: ${message}`); + this.name = "ClipError"; + } +} +export class ClipDuplicateError extends Error { + readonly code = "clip_duplicate"; + readonly existing: Pick< + Clip, + "id" | "title" | "originalUrl" | "createdAt" | "revision" | "archivedAt" + >; + constructor(clip: Clip) { + super("This link is already saved."); + this.name = "ClipDuplicateError"; + const { id, title, originalUrl, createdAt, revision, archivedAt } = clip; + this.existing = { id, title, originalUrl, createdAt, revision, archivedAt }; + } +} +export class ClipConflictError extends Error { + readonly code = "clip_conflict"; + constructor(public readonly current: ClipSnapshot) { + super("Clips changed. Reload before applying this action again."); + this.name = "ClipConflictError"; + } +} +export type ClipExportFile = { filename: string; content: string }; diff --git a/web/src/clips/validation.ts b/web/src/clips/validation.ts new file mode 100644 index 0000000..6d79027 --- /dev/null +++ b/web/src/clips/validation.ts @@ -0,0 +1,237 @@ +import { safeHref } from "../domain/validation"; +import { ClipError } from "./contracts"; +import type { ClipActionInput, ClipState } from "./contracts"; + +export const CLIP_LIMITS = { + items: 5000, + receipts: 500, + url: 8192, + title: 300, + text: 100000, + bytes: 12 * 1024 * 1024, +} as const; +const UUID = + /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; +export function invalid(): never { + throw new ClipError("clip_invalid", "Unsupported or corrupt clip data."); +} +export function object( + value: unknown, + keys: string[], +): Record { + if ( + !value || + typeof value !== "object" || + Array.isArray(value) || + Object.getPrototypeOf(value) !== Object.prototype + ) + invalid(); + const row = value as Record; + if ( + Object.keys(row).length !== keys.length || + keys.some((key) => !Object.hasOwn(row, key)) + ) + invalid(); + return row; +} +export function id(value: unknown): asserts value is string { + if (typeof value !== "string" || !UUID.test(value)) invalid(); +} +export function integer(value: unknown, min = 0): asserts value is number { + if (typeof value !== "number" || !Number.isSafeInteger(value) || value < min) + invalid(); +} +function text(value: unknown, max: number): asserts value is string { + if (typeof value !== "string" || value.length > max) invalid(); +} +function bool(value: unknown): asserts value is boolean { + if (typeof value !== "boolean") invalid(); +} +function date(value: unknown): asserts value is string { + if ( + typeof value !== "string" || + value.length !== 24 || + !Number.isFinite(Date.parse(value)) || + new Date(value).toISOString() !== value + ) + invalid(); +} +export function clipUrl(value: unknown): { + originalUrl: string; + canonicalUrl: string; + hostname: string; +} { + text(value, CLIP_LIMITS.url); + if (!safeHref(value)) invalid(); + const authority = /^https?:\/\/([^/?#]*)/i.exec(value)?.[1]; + if (!authority || authority.includes("@") || value.includes("\\")) invalid(); + let url: URL; + try { + url = new URL(value); + } catch { + invalid(); + } + if ( + !["http:", "https:"].includes(url.protocol) || + !url.hostname || + url.username || + url.password + ) + invalid(); + url.hash = ""; + if (url.toString().length > CLIP_LIMITS.url) invalid(); + return { + originalUrl: value, + canonicalUrl: url.toString(), + hostname: url.hostname, + }; +} +export function validateClipAction(value: unknown): ClipActionInput { + if (!value || typeof value !== "object") invalid(); + const kind = (value as { kind?: unknown }).kind; + if (kind === "create") { + const row = object(value, [ + "kind", + "url", + "title", + "text", + "readLater", + "allowDuplicate", + ]); + const url = clipUrl(row.url); + text(row.title, CLIP_LIMITS.title); + text(row.text, CLIP_LIMITS.text); + bool(row.readLater); + bool(row.allowDuplicate); + const title = row.title.trim() || url.hostname; + text(title, CLIP_LIMITS.title); + return { + kind, + url: url.originalUrl, + title, + text: row.text.trim() ? row.text : "", + readLater: row.readLater, + allowDuplicate: row.allowDuplicate, + }; + } + if (kind === "undo-create") { + const row = object(value, ["kind", "receiptOperationId", "targetRevision"]); + id(row.receiptOperationId); + integer(row.targetRevision, 1); + return { + kind, + receiptOperationId: row.receiptOperationId, + targetRevision: row.targetRevision, + }; + } + if ( + kind === "set-read" || + kind === "set-read-later" || + kind === "set-archived" + ) { + const flag = + kind === "set-read" + ? "read" + : kind === "set-read-later" + ? "readLater" + : "archived"; + const row = object(value, ["kind", "clipId", "targetRevision", flag]); + id(row.clipId); + integer(row.targetRevision, 1); + bool(row[flag]); + return structuredClone(row) as ClipActionInput; + } + invalid(); +} +export function validateClipState(value: unknown): ClipState { + const state = object(value, ["items", "receipts"]); + if (!Array.isArray(state.items) || !Array.isArray(state.receipts)) invalid(); + if ( + state.items.length > CLIP_LIMITS.items || + state.receipts.length > CLIP_LIMITS.receipts + ) + throw new ClipError("clip_limit", "Clips exceed the local storage limits."); + const items = new Set(), + receipts = new Set(); + for (const value of state.items) { + const row = object(value, [ + "id", + "revision", + "originalUrl", + "canonicalUrl", + "title", + "text", + "createdAt", + "updatedAt", + "readAt", + "readLater", + "archivedAt", + ]); + id(row.id); + integer(row.revision, 1); + if (items.has(row.id)) invalid(); + items.add(row.id); + const url = clipUrl(row.originalUrl); + if (url.canonicalUrl !== row.canonicalUrl) invalid(); + text(row.title, CLIP_LIMITS.title); + if (!row.title || row.title.trim() !== row.title) invalid(); + if (row.text !== null) { + text(row.text, CLIP_LIMITS.text); + if (!row.text.trim()) invalid(); + } + date(row.createdAt); + date(row.updatedAt); + if (row.updatedAt < row.createdAt) invalid(); + for (const key of ["readAt", "archivedAt"]) + if (row[key] !== null) { + date(row[key]); + if (row[key] < row.createdAt || row[key] > row.updatedAt) invalid(); + } + bool(row.readLater); + } + for (const value of state.receipts) { + const row = object(value, [ + "operationId", + "digest", + "kind", + "clipId", + "resultRevision", + "at", + "changed", + ]); + id(row.operationId); + id(row.clipId); + date(row.at); + bool(row.changed); + if (receipts.has(row.operationId)) invalid(); + receipts.add(row.operationId); + if (typeof row.digest !== "string" || !/^[a-f0-9]{64}$/.test(row.digest)) + invalid(); + if ( + ![ + "create", + "set-read", + "set-read-later", + "set-archived", + "undo-create", + ].includes(row.kind as string) + ) + invalid(); + if (row.kind === "undo-create") { + if (row.resultRevision !== null || !row.changed) invalid(); + } else { + integer(row.resultRevision, 1); + if (row.kind === "create" && (row.resultRevision !== 1 || !row.changed)) + invalid(); + } + } + if ( + new TextEncoder().encode(JSON.stringify(value)).byteLength > + CLIP_LIMITS.bytes + ) + throw new ClipError("clip_limit", "Clips exceed the local storage limits."); + return structuredClone(value as ClipState); +} +export function emptyClips(): ClipState { + return { items: [], receipts: [] }; +} diff --git a/web/src/collections/contracts.ts b/web/src/collections/contracts.ts new file mode 100644 index 0000000..e086329 --- /dev/null +++ b/web/src/collections/contracts.ts @@ -0,0 +1,117 @@ +import type { PersistenceState } from "../domain/contracts"; +import type { ClipState } from "../clips/contracts"; +import type { AnchoredStudySource } from "../study/sourceContracts"; + +export type Grade = "again" | "hard" | "good" | "easy"; +export type Schedule = { + version: 1; + state: "new" | "learning" | "review"; + due: string; + intervalDays: number; + ease: number; + repetitions: number; + lapses: number; + lastReviewedAt: string | null; +}; +export type Deck = { + id: string; + title: string; + createdAt: string; + updatedAt: string; + revision: number; +}; +export type LegacyStudySource = { + noteId: string; + noteRevision: number; + quote: string; +}; +export type StudySource = LegacyStudySource | AnchoredStudySource; +type EntryBase = { + id: string; + deckId: string; + tags: string[]; + source: StudySource | null; + revision: number; +}; +export type StudyEntry = EntryBase & + ( + | { type: "basic"; front: string; back: string } + | { type: "reversed"; front: string; back: string } + | { type: "cloze"; text: string; blanks: { id: string; answer: string }[] } + ); +export type CardKey = + "basic:forward" | "reversed:forward" | "reversed:reverse" | `cloze:${string}`; +export type ReviewCard = { + id: string; + entryId: string; + key: CardKey; + schedule: Schedule; + revision: number; +}; +export type ReviewSnapshot = { schedule: Schedule; cardRevision: number }; +export type ReviewEvent = { + id: string; + cardId: string; + sessionId: string; + at: string; + grade: Grade; + algorithm: "orbyn-interval-v1"; + before: ReviewSnapshot; + after: ReviewSnapshot; + undoneAt: string | null; +}; +export type StudyPreferences = { + algorithm: "orbyn-interval-v1"; + timezone: string; + dayStartHour: number; + newPerDay: number; + reviewsPerDay: number; + showNextReview: boolean; + showStreak: boolean; +}; +export type StudyState = { + decks: Deck[]; + entries: StudyEntry[]; + cards: ReviewCard[]; + events: ReviewEvent[]; + preferences: StudyPreferences; +}; +/** Binary fields use canonical padded base64. Only JournalVault replaces this envelope. */ +export type JournalEnvelope = { + version: 1; + kdf: { name: "PBKDF2-SHA256"; iterations: 600000; salt: string }; + iv: string; + ciphertext: string; +}; +export type LegacyCollectionsState = { + version: 1; + epoch: string; + revision: number; + study: StudyState; + journal: JournalEnvelope | null; +}; +export type CollectionsStateV2 = Omit & { + version: 2; + clips: ClipState; +}; +export type CollectionsState = Omit & { + version: 3; +}; +export type CollectionsRevision = Pick; +export interface CollectionsStore { + read(): Promise; + update( + expected: CollectionsRevision, + change: (current: CollectionsState) => CollectionsState, + ): Promise; + subscribe(listener: () => void): () => void; + persistence(): PersistenceState; + close(): void; +} +export class CollectionsRevisionConflictError extends Error { + readonly code = "collections_revision_conflict"; + constructor(public readonly current: CollectionsState) { + super("Collections changed in another window. Reload before retrying."); + this.name = "CollectionsRevisionConflictError"; + } +} diff --git a/web/src/collections/validation.ts b/web/src/collections/validation.ts new file mode 100644 index 0000000..e72dc65 --- /dev/null +++ b/web/src/collections/validation.ts @@ -0,0 +1,396 @@ +import type { + CollectionsState, + LegacyCollectionsState, + CollectionsStateV2, + JournalEnvelope, + Schedule, + StudyEntry, + StudyState, +} from "./contracts"; +import { emptyClips, validateClipState } from "../clips/validation"; +import { validateAnchoredSource } from "../study/sourceValidation"; + +export const LEGACY_COLLECTIONS_BYTES = 32 * 1024 * 1024; +export const EMPTY_CLIPS_OVERHEAD = new TextEncoder().encode( + ',"clips":{"items":[],"receipts":[]}', +).byteLength; + +export const COLLECTIONS_LIMITS = { + decks: 1000, + entries: 10000, + cards: 20000, + events: 100000, + text: 100000, + serializedBytes: LEGACY_COLLECTIONS_BYTES + EMPTY_CLIPS_OVERHEAD, + envelopeBytes: 12 * 1024 * 1024, +} as const; +const uuid = + /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; +const blankId = /^[A-Za-z0-9_-]{1,100}$/; +function fail(): never { + throw new Error( + "collections_invalid: Unsupported or corrupt collections data.", + ); +} +function object(value: unknown, keys: string[]): Record { + if ( + !value || + typeof value !== "object" || + Array.isArray(value) || + Object.getPrototypeOf(value) !== Object.prototype + ) + fail(); + const result = value as Record; + if ( + Object.keys(result).length !== keys.length || + keys.some((key) => !Object.hasOwn(result, key)) + ) + fail(); + return result; +} +function text( + value: unknown, + max: number = COLLECTIONS_LIMITS.text, + nonempty = false, +): asserts value is string { + if ( + typeof value !== "string" || + value.length > max || + (nonempty && !value.trim()) + ) + fail(); +} +function id(value: unknown): asserts value is string { + if (typeof value !== "string" || !uuid.test(value)) fail(); +} +function integer( + value: unknown, + min = 0, + max = Number.MAX_SAFE_INTEGER, +): asserts value is number { + if ( + typeof value !== "number" || + !Number.isSafeInteger(value) || + value < min || + value > max + ) + fail(); +} +function date(value: unknown): asserts value is string { + if ( + typeof value !== "string" || + value.length !== 24 || + !Number.isFinite(Date.parse(value)) || + new Date(value).toISOString() !== value + ) + fail(); +} +function array(value: unknown, max: number): asserts value is unknown[] { + if (!Array.isArray(value) || value.length > max) fail(); +} +function bool(value: unknown) { + if (typeof value !== "boolean") fail(); +} +function unique( + rows: unknown[], + check: (value: unknown) => Record, +): Map> { + const seen = new Map>(); + for (const value of rows) { + const row = check(value); + id(row.id); + if (seen.has(row.id)) fail(); + seen.set(row.id, row); + } + return seen; +} +function schedule(value: unknown): asserts value is Schedule { + const row = object(value, [ + "version", + "state", + "due", + "intervalDays", + "ease", + "repetitions", + "lapses", + "lastReviewedAt", + ]); + if ( + row.version !== 1 || + !["new", "learning", "review"].includes(row.state as string) + ) + fail(); + date(row.due); + integer(row.intervalDays, 0, 36500); + integer(row.repetitions); + integer(row.lapses); + if ( + typeof row.ease !== "number" || + !Number.isFinite(row.ease) || + row.ease < 1.3 || + row.ease > 3 + ) + fail(); + if (row.lastReviewedAt !== null) date(row.lastReviewedAt); +} +export function derivedCardKeys(entry: StudyEntry): string[] { + return entry.type === "basic" + ? ["basic:forward"] + : entry.type === "reversed" + ? ["reversed:forward", "reversed:reverse"] + : entry.blanks.map((blank) => `cloze:${blank.id}`); +} +export function emptyStudy(timezone = "UTC"): StudyState { + return { + decks: [], + entries: [], + cards: [], + events: [], + preferences: { + algorithm: "orbyn-interval-v1", + timezone, + dayStartHour: 4, + newPerDay: 10, + reviewsPerDay: 100, + showNextReview: true, + showStreak: true, + }, + }; +} +export function validateJournalEnvelope( + value: unknown, +): JournalEnvelope | null { + if (value === null) return null; + const row = object(value, ["version", "kdf", "iv", "ciphertext"]); + const kdf = object(row.kdf, ["name", "iterations", "salt"]); + if ( + row.version !== 1 || + kdf.name !== "PBKDF2-SHA256" || + kdf.iterations !== 600000 + ) + fail(); + const binary = (value: unknown, min: number, max: number) => { + if ( + typeof value !== "string" || + value.length > Math.ceil(max / 3) * 4 || + value.length % 4 !== 0 || + !/^[A-Za-z0-9+/]*={0,2}$/.test(value) + ) + fail(); + const decoded = atob(value); + if (decoded.length < min || decoded.length > max || btoa(decoded) !== value) + fail(); + }; + binary(kdf.salt, 16, 16); + binary(row.iv, 12, 12); + binary(row.ciphertext, 16, COLLECTIONS_LIMITS.envelopeBytes); + return structuredClone(value as JournalEnvelope); +} +function validateState( + value: unknown, + version: 1 | 2 | 3, +): CollectionsState | LegacyCollectionsState | CollectionsStateV2 { + const legacy = version === 1; + const row = object(value, [ + "version", + "epoch", + "revision", + "study", + "journal", + ...(legacy ? [] : ["clips"]), + ]); + if (row.version !== version) fail(); + id(row.epoch); + integer(row.revision); + const study = object(row.study, [ + "decks", + "entries", + "cards", + "events", + "preferences", + ]); + array(study.decks, COLLECTIONS_LIMITS.decks); + array(study.entries, COLLECTIONS_LIMITS.entries); + array(study.cards, COLLECTIONS_LIMITS.cards); + array(study.events, COLLECTIONS_LIMITS.events); + const decks = unique(study.decks, (value) => { + const d = object(value, [ + "id", + "title", + "createdAt", + "updatedAt", + "revision", + ]); + text(d.title, 1000, true); + date(d.createdAt); + date(d.updatedAt); + if (d.updatedAt < d.createdAt) fail(); + integer(d.revision, 1); + return d; + }); + const entries = unique(study.entries, (value) => { + const candidate = value as Record | null; + const variant = candidate?.type; + const extra = variant === "cloze" ? ["text", "blanks"] : ["front", "back"]; + const e = object(value, [ + "id", + "deckId", + "tags", + "source", + "revision", + "type", + ...extra, + ]); + id(e.deckId); + if (!decks.has(e.deckId)) fail(); + integer(e.revision, 1); + array(e.tags, 100); + const tags = new Set(); + for (const tag of e.tags) { + text(tag, 100, true); + if (tags.has(tag)) fail(); + tags.add(tag); + } + if (e.source !== null) { + if (version === 3 && (e.source as { version?: unknown })?.version === 2) + validateAnchoredSource(e.source); + else { + const s = object(e.source, ["noteId", "noteRevision", "quote"]); + id(s.noteId); + integer(s.noteRevision, 1); + text(s.quote); + } + } + if (variant === "basic" || variant === "reversed") { + text(e.front, COLLECTIONS_LIMITS.text, true); + text(e.back, COLLECTIONS_LIMITS.text, true); + } else if (variant === "cloze") { + text(e.text, COLLECTIONS_LIMITS.text, true); + array(e.blanks, 100); + if (!e.blanks.length) fail(); + const blanks = new Set(); + for (const raw of e.blanks) { + const b = object(raw, ["id", "answer"]); + if (typeof b.id !== "string" || !blankId.test(b.id) || blanks.has(b.id)) + fail(); + text(b.answer, COLLECTIONS_LIMITS.text, true); + blanks.add(b.id); + } + const matches = [...e.text.matchAll(/\{\{([^{}]+)\}\}/g)].map( + (match) => match[1]!, + ); + if ( + matches.length !== blanks.size || + new Set(matches).size !== matches.length || + matches.some((key) => !blanks.has(key)) || + e.text.replace(/\{\{[^{}]+\}\}/g, "").includes("{{") || + e.text.replace(/\{\{[^{}]+\}\}/g, "").includes("}}") + ) + fail(); + } else fail(); + return e; + }); + const seenKeys = new Set(); + unique(study.cards, (value) => { + const c = object(value, ["id", "entryId", "key", "schedule", "revision"]); + id(c.entryId); + integer(c.revision, 1); + schedule(c.schedule); + const entry = entries.get(c.entryId); + if ( + !entry || + typeof c.key !== "string" || + !derivedCardKeys(entry as StudyEntry).includes(c.key) + ) + fail(); + const key = `${c.entryId}/${c.key}`; + if (seenKeys.has(key)) fail(); + seenKeys.add(key); + return c; + }); + for (const entry of entries.values()) + for (const key of derivedCardKeys(entry as StudyEntry)) + if (!seenKeys.has(`${entry.id}/${key}`)) fail(); + unique(study.events, (value) => { + const e = object(value, [ + "id", + "cardId", + "sessionId", + "at", + "grade", + "algorithm", + "before", + "after", + "undoneAt", + ]); + id(e.cardId); + id(e.sessionId); + date(e.at); + if ( + !["again", "hard", "good", "easy"].includes(e.grade as string) || + e.algorithm !== "orbyn-interval-v1" + ) + fail(); + const before = object(e.before, ["schedule", "cardRevision"]); + const after = object(e.after, ["schedule", "cardRevision"]); + schedule(before.schedule); + schedule(after.schedule); + integer(before.cardRevision, 1); + integer(after.cardRevision, 2); + if (after.cardRevision !== before.cardRevision + 1) fail(); + if (e.undoneAt !== null) { + date(e.undoneAt); + if (e.undoneAt < e.at) fail(); + } + return e; + }); + const p = object(study.preferences, [ + "algorithm", + "timezone", + "dayStartHour", + "newPerDay", + "reviewsPerDay", + "showNextReview", + "showStreak", + ]); + if (p.algorithm !== "orbyn-interval-v1") fail(); + text(p.timezone, 100, true); + try { + new Intl.DateTimeFormat("en", { timeZone: p.timezone }); + } catch { + fail(); + } + integer(p.dayStartHour, 0, 23); + integer(p.newPerDay, 0, 10000); + integer(p.reviewsPerDay, 0, 20000); + bool(p.showNextReview); + bool(p.showStreak); + validateJournalEnvelope(row.journal); + if (!legacy) validateClipState(row.clips); + if ( + new TextEncoder().encode(JSON.stringify(value)).byteLength > + (legacy ? LEGACY_COLLECTIONS_BYTES : COLLECTIONS_LIMITS.serializedBytes) + ) + fail(); + return structuredClone(value as CollectionsState); +} +export function validateLegacyCollections( + value: unknown, +): LegacyCollectionsState { + return validateState(value, 1) as LegacyCollectionsState; +} +export function validateCollections(value: unknown): CollectionsState { + return validateState(value, 3) as CollectionsState; +} +export function validateCollectionsV2(value: unknown): CollectionsStateV2 { + return validateState(value, 2) as CollectionsStateV2; +} +export function normalizeCollections(value: unknown): CollectionsState { + if ((value as { version?: unknown } | null)?.version === 1) { + const legacy = validateLegacyCollections(value); + return validateCollections({ ...legacy, version: 3, clips: emptyClips() }); + } + if ((value as { version?: unknown } | null)?.version === 2) + return validateCollections({ ...validateCollectionsV2(value), version: 3 }); + return validateCollections(value); +} diff --git a/web/src/library/contracts.ts b/web/src/library/contracts.ts new file mode 100644 index 0000000..39abd9a --- /dev/null +++ b/web/src/library/contracts.ts @@ -0,0 +1,237 @@ +import type { NoteLinksRow } from "../links/contracts"; +import type { Note, NoteDocument, NotesStore } from "../domain/contracts"; +import type { + SourceAnchor, + SourceReceipt, + SourceAwareNoteUpdate, +} from "../study/sourceContracts"; +export type LibraryToken = { epoch: string; revision: number }; +export type NoteSort = "edited" | "created" | "title"; +export type TagColor = string | null; +export type Tag = { + id: string; + name: string; + color: TagColor; + revision: number; + createdAt: string; + updatedAt: string; + state: "active" | "deleted" | "merged"; + mergedInto: string | null; +}; +export type NoteMetadata = { + noteId: string; + revision: number; + pinned: boolean; + tagIds: string[]; +}; +export type NoteVersion = { + id: string; + noteId: string; + noteRevision: number; + capturedAt: string; + reason: "edit" | "history-restore"; + title: string; + document: NoteDocument; + tagsAtCapture: { id: string; name: string; color: TagColor }[]; +}; +export type TrashUndo = { + epoch: string; + operationId: string; + noteId: string; + resultRevision: number; + previousTrashedAt: string | null; + resultTrashedAt: string | null; + expiresAt: string; +}; +export type LibraryAction = + | "pin" + | "sort" + | "tag-create" + | "tag-rename" + | "tag-color" + | "tag-merge" + | "tag-delete" + | "history-restore" + | "trash" + | "restore" + | "trash-undo" + | "empty-trash" + | "legacy-erase" + | "note-links-repair"; +export type LibraryOperationReceipt = { + id: string; + epoch: string; + committedAt: string; + libraryRevision: number; + kind: LibraryAction; + arguments: string; + result: + | { type: "metadata" } + | { type: "tag"; tagId: string } + | { + type: "note"; + noteId: string; + resultRevision: number; + undo: TrashUndo | null; + } + | { type: "empty-trash"; removedCount: number } + | { type: "note-links-repair"; indexDigest: string }; +}; +export type LegacyLibraryState = { + version: 1; + epoch: string; + revision: number; + tags: Tag[]; + noteMetadata: NoteMetadata[]; + versions: NoteVersion[]; + preferences: { sort: NoteSort }; + receipts: LibraryOperationReceipt[]; + legacyDuplicateTags: { noteId: string; values: string[] }[]; +}; +export type SourceLibraryState = Omit & { + version: 2; + anchors: SourceAnchor[]; + sourceReceipts: SourceReceipt[]; +}; +export type LinkedLibraryState = Omit & { + version: 3; + noteLinks: NoteLinksRow[]; +}; +export type LibraryState = SourceLibraryState | LinkedLibraryState; +export type LibraryCommonState = Omit & { + version: 2 | 3; +}; +export type LibrarySnapshot = { + token: LibraryToken; + notes: Note[]; + tags: Tag[]; + metadata: NoteMetadata[]; + preferences: { sort: NoteSort }; +}; +export type LibraryReadResult = + | { mode: "ready"; snapshot: LibrarySnapshot } + | { + mode: "legacy-read-only"; + notes: Note[]; + reason: "metadata-limit"; + recovery: { kind: "export-or-confirm-erase"; deviceGeneration: number }; + }; +export type LibraryMutationResult = { + snapshot: LibrarySnapshot; + note: Note | null; + tag: Tag | null; + undo: TrashUndo | null; + removedCount: number; + replayed: boolean; +}; +export interface NotesLibrary { + replaceDocument( + noteId: string, + input: Pick, + expectedRevision: number, + ): Promise; + updateWithSourceSteps: SourceAwareNoteUpdate; + read(): Promise; + setPinned( + noteId: string, + pinned: boolean, + expected: LibraryToken, + operationId: string, + ): Promise; + setSort( + sort: NoteSort, + expected: LibraryToken, + operationId: string, + ): Promise; + createTag( + input: { name: string; color: TagColor }, + expected: LibraryToken, + operationId: string, + ): Promise; + renameTag( + id: string, + name: string, + expected: LibraryToken, + operationId: string, + ): Promise; + recolorTag( + id: string, + color: TagColor, + expected: LibraryToken, + operationId: string, + ): Promise; + mergeTag( + sourceId: string, + targetId: string, + expected: LibraryToken, + operationId: string, + ): Promise; + deleteTag( + id: string, + expected: LibraryToken, + operationId: string, + ): Promise; + history(noteId: string): Promise; + version(noteId: string, versionId: string): Promise; + restoreVersion( + noteId: string, + versionId: string, + expectedNoteRevision: number, + operationId: string, + ): Promise; + changeTrash( + noteId: string, + action: "trash" | "restore", + expectedNoteRevision: number, + operationId: string, + ): Promise; + undoTrash( + receipt: TrashUndo, + operationId: string, + ): Promise; + emptyTrash( + expected: LibraryToken, + operationId: string, + ): Promise; + eraseLegacy( + input: { deviceGeneration: number; confirmation: "ERASE" }, + operationId: string, + ): Promise; +} +export interface NotesLibraryStore extends NotesStore { + library: NotesLibrary; +} +export class LibraryReadOnlyError extends Error { + readonly code = "library_read_only"; + constructor() { + super( + "library_read_only: Library metadata exceeds its limit. Export your notes or confirm erasing this workspace.", + ); + this.name = "LibraryReadOnlyError"; + } +} +export class LibraryConflictError extends Error { + readonly code = "library_conflict"; + constructor(public readonly current: LibrarySnapshot) { + super("library_conflict: The library changed. Reload before trying again."); + this.name = "LibraryConflictError"; + } +} +export class LibraryEpochError extends Error { + readonly code = "library_epoch_changed"; + constructor() { + super( + "library_epoch_changed: This workspace was erased. Reopen it before making changes.", + ); + this.name = "LibraryEpochError"; + } +} +export class LibraryCapacityError extends Error { + readonly code = "library_capacity"; + constructor() { + super( + "library_capacity: Library metadata or history exceeds its supported size.", + ); + this.name = "LibraryCapacityError"; + } +} diff --git a/web/src/search/identity.ts b/web/src/search/identity.ts new file mode 100644 index 0000000..82c2872 --- /dev/null +++ b/web/src/search/identity.ts @@ -0,0 +1,96 @@ +import type { WorkEvent } from "./work"; + +/** Native Map string hashing/comparison can consume a legal opaque identity in + * one uninterruptible operation. Hash it in admitted chunks instead, bucket by + * a fixed-size integer, then verify every collision against exact UTF16 units. + * FNV is only an index accelerator, never an identity or authorization proof. */ +export class SearchIdentityMap { + private readonly buckets = new Map(); + private generation = 0; + + clear(): void { + this.generation++; + this.buckets.clear(); + } + + private guard(generation: number): void { + if (generation !== this.generation) throw new Error("Search identities were released."); + } + + private *hash(key: string, generation: number): Generator { + let hash = 0x811c9dc5; + for (let from = 0; from < key.length; ) { + const to = Math.min(key.length, from + 4096); + const units = to - from; + yield { type: "reserve", bytes: units * 3, visits: 1, scalars: units }; + this.guard(generation); + for (let i = from; i < to; i++) hash = Math.imul(hash ^ key.charCodeAt(i), 0x01000193); + from = to; + yield { type: "settle", bytes: units * 3, visits: 1, scalars: units }; + } + return hash >>> 0; + } + + private *equal(a: string, b: string, generation: number): Generator { + yield { type: "reserve", bytes: 8, visits: 1, scalars: 0 }; + this.guard(generation); + const sameLength = a.length === b.length; + yield { type: "settle", bytes: 8, visits: 1, scalars: 0 }; + if (!sameLength) return false; + for (let from = 0; from < a.length; ) { + const to = Math.min(a.length, from + 4096); + const units = to - from; + yield { type: "reserve", bytes: units * 6, visits: 1, scalars: units }; + this.guard(generation); + let checked = 0, equal = true; + for (let i = from; i < to; i++) { + checked++; + if (a.charCodeAt(i) !== b.charCodeAt(i)) { equal = false; break; } + } + yield { type: "settle", bytes: checked * 6, visits: 1, scalars: checked }; + if (!equal) return false; + from = to; + } + return true; + } + + *get(key: string): Generator { + const generation = this.generation; + const hash = yield* this.hash(key, generation); + yield { type: "reserve", bytes: 32, visits: 1, scalars: 0 }; + this.guard(generation); + const bucket = this.buckets.get(hash); + yield { type: "settle", bytes: 32, visits: 1, scalars: 0 }; + if (!bucket) return undefined; + for (const entry of bucket) { + if (yield* this.equal(entry.key, key, generation)) { + this.guard(generation); + return entry.value; + } + } + return undefined; + } + + *set(key: string, value: T): Generator { + const generation = this.generation; + const hash = yield* this.hash(key, generation); + yield { type: "reserve", bytes: 32, visits: 1, scalars: 0 }; + this.guard(generation); + const existing = this.buckets.get(hash); + yield { type: "settle", bytes: 32, visits: 1, scalars: 0 }; + if (existing) { + for (const entry of existing) { + if (yield* this.equal(entry.key, key, generation)) { + this.guard(generation); + entry.value = value; + return; + } + } + } + yield { type: "reserve", bytes: 48, visits: 1, scalars: 0 }; + this.guard(generation); + if (existing) existing.push({ key, value }); + else this.buckets.set(hash, [{ key, value }]); + yield { type: "settle", bytes: 48, visits: 1, scalars: 0 }; + } +} diff --git a/web/src/search/work.ts b/web/src/search/work.ts new file mode 100644 index 0000000..d4f66f2 --- /dev/null +++ b/web/src/search/work.ts @@ -0,0 +1,121 @@ +/** A producer must reserve before touching a source chunk, then settle its + * actual work before requesting another chunk. Limits apply to one deliberate + * round; the iterator itself survives a pause. */ +export type WorkCost = Readonly<{ bytes: number; visits: number; scalars: number }>; +export type WorkEvent = WorkCost & Readonly<{ type: "reserve" | "settle" }>; +export type WorkLimit = Readonly<{ bytes: number; visits: number }>; +export type WorkResult = + | Readonly<{ status: "paused"; used: WorkCost }> + | Readonly<{ status: "done"; value: T; used: WorkCost }>; + +const defaultLimit: WorkLimit = Object.freeze({ bytes: 64 * 1024 * 1024, visits: 250000 }); +const zero = (): WorkCost => ({ bytes: 0, visits: 0, scalars: 0 }); +const validCost = (cost: WorkCost): boolean => + [cost.bytes, cost.visits, cost.scalars].every(value => Number.isSafeInteger(value) && value >= 0); + +/** A task boundary, rather than a resolved Promise/microtask: native input and + * cancellation can run between chunks. Ports are closed after each yield. */ +export function searchTaskBoundary(): Promise { + if (typeof MessageChannel === "undefined") + return new Promise(resolve => setTimeout(resolve, 0)); + return new Promise(resolve => { + const channel = new MessageChannel(); + channel.port1.onmessage = () => { + channel.port1.close(); + channel.port2.close(); + resolve(); + }; + channel.port2.postMessage(null); + }); +} + +export class SearchWork { + private iterator: Generator | null; + private event: IteratorResult | null = null; + private running = false; + private closed = false; + private result: { value: T } | null = null; + + constructor( + iterator: Generator, + private readonly limit: WorkLimit = defaultLimit, + private readonly yieldTask: () => Promise = searchTaskBoundary, + private readonly assertCurrent: () => void = () => {}, + ) { + if (!Number.isSafeInteger(limit.bytes) || limit.bytes <= 0 || + !Number.isSafeInteger(limit.visits) || limit.visits <= 0) + throw new Error("Invalid search work limits."); + this.iterator = iterator; + } + + close(): void { + this.closed = true; + const iterator = this.iterator; + this.iterator = null; + this.event = null; + this.result = null; + iterator?.return(undefined as T); + } + + async round(signal?: AbortSignal): Promise> { + if (this.running) throw new Error("Search work is already running."); + const assert = () => { + if (signal?.aborted || this.closed) + throw new DOMException("Search stopped.", "AbortError"); + this.assertCurrent(); + }; + assert(); + if (this.result) return { status: "done", value: this.result.value, used: zero() }; + this.running = true; + let bytes = 0, visits = 0, scalars = 0; + let batchVisits = 0, batchScalars = 0, started = performance.now(); + const used = (): WorkCost => Object.freeze({ bytes, visits, scalars }); + try { + this.event ??= this.iterator!.next(); + while (!this.event.done) { + assert(); + const reserve = this.event.value; + if (reserve.type !== "reserve" || !validCost(reserve)) + throw new Error("Search work requires a valid reservation."); + if (reserve.bytes > this.limit.bytes || reserve.visits > this.limit.visits || + reserve.scalars > 4096 || reserve.visits > 2048) + throw new Error("Search work has an oversized indivisible reservation."); + if (reserve.bytes > this.limit.bytes - bytes || reserve.visits > this.limit.visits - visits) + return { status: "paused", used: used() }; + + // Grant only this reservation. next() runs one bounded synchronous + // producer operation and must immediately report its actual cost. + const settled = this.iterator!.next(); + if (settled.done || settled.value.type !== "settle" || !validCost(settled.value) || + settled.value.bytes > reserve.bytes || settled.value.visits > reserve.visits || + settled.value.scalars > reserve.scalars) + throw new Error("Search work exceeded or omitted its reservation settlement."); + bytes += settled.value.bytes; + visits += settled.value.visits; + scalars += settled.value.scalars; + batchVisits += settled.value.visits; + batchScalars += settled.value.scalars; + if (batchScalars >= 4096 || batchVisits >= 2048 || performance.now() - started >= 8) { + await this.yieldTask(); + assert(); + batchVisits = 0; + batchScalars = 0; + started = performance.now(); + } + assert(); + this.event = this.iterator!.next(); + } + assert(); + const value = this.event.value; + this.result = { value }; + this.iterator = null; + this.event = null; + return { status: "done", value, used: used() }; + } catch (error) { + this.close(); + throw error; + } finally { + this.running = false; + } + } +} diff --git a/web/src/study/cardContent.ts b/web/src/study/cardContent.ts new file mode 100644 index 0000000..4d72861 --- /dev/null +++ b/web/src/study/cardContent.ts @@ -0,0 +1,457 @@ +import type { CardKey } from "../collections/contracts"; +import { SearchIdentityMap } from "../search/identity"; +import type { WorkEvent } from "../search/work"; +export type CardContentInput = + | Readonly<{ + type: "basic"; + key: "basic:forward"; + front: string; + back: string; + }> + | Readonly<{ + type: "reversed"; + key: "reversed:forward" | "reversed:reverse"; + front: string; + back: string; + }> + | Readonly<{ + type: "cloze"; + key: Extract; + text: string; + blanks: readonly Readonly<{ id: string; answer: string }>[]; + }>; +export type CardContentSide = "front" | "back"; +declare const readerBrand: unique symbol; +declare const cursorBrand: unique symbol; +export type CardContentReader = Readonly<{ [readerBrand]: true }>; +export type CardContentCursor = Readonly<{ [cursorBrand]: true }>; +export type CardContentPage = Readonly<{ + text: string; + start: number; + end: number; + total: number; + previous: CardContentCursor | null; + next: CardContentCursor | null; +}>; +type Segment = { text: string; from: number; to: number; end: number }; +type Field = { segments: Segment[]; total: number }; +type State = { + front: Field; + back: Field; + disposed: boolean; + pages: Set>; +}; +type Position = { + reader: CardContentReader; + side: CardContentSide; + start: number; + previous: CardContentCursor | null; +}; +const readers = new WeakMap(), + cursors = new WeakMap(); +const empty = (): Field => ({ segments: [], total: 0 }); +function invalid(): never { + throw new Error("Invalid card content."); +} +function stateOf(reader: CardContentReader): State { + const state = readers.get(reader); + if (!state || state.disposed) + throw new Error("Card content was released or is invalid."); + return state; +} +function* work( + bytes: number, + visits: number, + operation: () => T, + scalars = 0, +): Generator { + yield { type: "reserve", bytes, visits, scalars }; + const result = operation(); + yield { type: "settle", bytes, visits, scalars }; + return result; +} +function release(state: State) { + state.disposed = true; + for (const page of state.pages) page.return(undefined!); + state.pages.clear(); + state.front.segments.length = 0; + state.back.segments.length = 0; + state.front.total = 0; + state.back.total = 0; +} +export function disposeCardContent(reader: CardContentReader): void { + const state = readers.get(reader); + if (!state) invalid(); + release(state); +} +function checkedText(value: unknown): string { + if (typeof value !== "string" || value.length > 100000) invalid(); + return value; +} +function capture(input: CardContentInput): CardContentInput { + if (!input || typeof input !== "object") invalid(); + if (input.type === "basic" || input.type === "reversed") { + if ( + input.type === "basic" + ? input.key !== "basic:forward" + : input.key !== "reversed:forward" && input.key !== "reversed:reverse" + ) + invalid(); + return Object.freeze({ + type: input.type, + key: input.key, + front: checkedText(input.front), + back: checkedText(input.back), + }) as CardContentInput; + } + if ( + input.type !== "cloze" || + typeof input.key !== "string" || + input.key.length > 106 || + !input.key.startsWith("cloze:") || + !Array.isArray(input.blanks) || + input.blanks.length < 1 || + input.blanks.length > 100 + ) + invalid(); + const blanks = input.blanks.map((blank) => { + if ( + !blank || + typeof blank !== "object" || + typeof blank.id !== "string" || + blank.id.length < 1 || + blank.id.length > 100 + ) + invalid(); + return Object.freeze({ id: blank.id, answer: checkedText(blank.answer) }); + }); + return Object.freeze({ + type: "cloze", + key: input.key, + text: checkedText(input.text), + blanks: Object.freeze(blanks), + }); +} +function* nonempty(text: string): Generator { + for (let from = 0; from < text.length;) { + const to = Math.min(text.length, from + 4096), + found = yield* work( + (to - from) * 6 + 64, + 1, + () => text.slice(from, to).trim().length > 0, + to - from, + ); + if (found) return; + from = to; + } + invalid(); +} +function* append( + field: Field, + text: string, + from = 0, + to = text.length, +): Generator { + if (from === to) return; + yield* work(64, 1, () => { + field.total += to - from; + if (!Number.isSafeInteger(field.total)) invalid(); + field.segments.push({ text, from, to, end: field.total }); + }); +} +function* parse( + input: Extract, + state: State, +): Generator { + const blanks = new SearchIdentityMap<{ answer: string; seen: number }>(), + literals: { from: number; to: number }[] = []; + try { + for (const blank of input.blanks) { + yield* work( + blank.id.length * 6 + 64, + 1, + () => { + if (!/^[A-Za-z0-9_-]{1,100}$/.test(blank.id)) invalid(); + }, + blank.id.length, + ); + yield* nonempty(blank.answer); + if (yield* blanks.get(blank.id)) invalid(); + yield* blanks.set(blank.id, { answer: blank.answer, seen: 0 }); + } + const selected = input.key.slice(6); + if (!(yield* blanks.get(selected))) invalid(); + let literal = 0, + open = -1, + stage = 0, + length = 0; + for (let from = 0; from < input.text.length;) { + const to = Math.min(input.text.length, from + 4096), + found: { from: number; to: number; id: string }[] = []; + yield* work( + (to - from) * 40 + 128, + 1, + () => { + for (let p = from; p < to; p++) { + const unit = input.text[p]!; + if (stage === 0) { + if (unit === "{") { + open = p; + stage = 1; + } + } else if (stage === 1) { + if (unit === "{") { + stage = 2; + length = 0; + } else { + stage = 0; + open = -1; + } + } else if (stage === 2) { + if (unit === "}") stage = 3; + else if (unit === "{") { + open = length === 0 ? p - 1 : p; + stage = length === 0 ? 2 : 1; + length = 0; + } else if (length < 101) length++; + else { + stage = 0; + open = -1; + } + } else if (unit === "}" && length) { + found.push({ + from: open, + to: p + 1, + id: input.text.slice(open + 2, p - 1), + }); + stage = 0; + open = -1; + } else if (unit === "{") { + open = p; + stage = 1; + length = 0; + } else { + stage = 0; + open = -1; + } + } + }, + to - from, + ); + for (const marker of found) { + const blank = yield* blanks.get(marker.id); + if (!blank || blank.seen++) invalid(); + yield* work(32, 1, () => + literals.push({ from: literal, to: marker.from }), + ); + yield* append(state.front, input.text, literal, marker.from); + yield* append(state.back, input.text, literal, marker.from); + yield* append( + state.front, + marker.id === selected ? "[...]" : blank.answer, + ); + yield* append(state.back, blank.answer); + literal = marker.to; + } + from = to; + } + yield* work(32, 1, () => + literals.push({ from: literal, to: input.text.length }), + ); + yield* append(state.front, input.text, literal); + yield* append(state.back, input.text, literal); + // Match producer admission after removing complete markers, without joining + // the literal residue or scanning answers as if they were marker syntax. + let previous = ""; + for (const span of literals) + for (let from = span.from; from < span.to;) { + const to = Math.min(span.to, from + 4096); + yield* work( + (to - from) * 6 + 32, + 1, + () => { + for (let p = from; p < to; p++) { + const unit = input.text[p]!; + if ( + (unit === "{" && previous === "{") || + (unit === "}" && previous === "}") + ) + invalid(); + previous = unit; + } + }, + to - from, + ); + from = to; + } + for (const blank of input.blanks) + if ((yield* blanks.get(blank.id))!.seen !== 1) invalid(); + } finally { + blanks.clear(); + literals.length = 0; + } +} +export function* buildCardContent( + input: CardContentInput, +): Generator { + const state: State = { + front: empty(), + back: empty(), + disposed: false, + pages: new Set(), + }; + let completed = false; + try { + // One bounded structural copy occurs before the first possible suspension + // after admission; no actual StudyEntry/input object is kept in reader state. + const stable = yield* work(16384, 102, () => { + const captured = capture(input); + input = undefined!; + return captured; + }); + if (stable.type === "cloze") { + yield* nonempty(stable.text); + yield* parse(stable, state); + } else { + yield* nonempty(stable.front); + yield* nonempty(stable.back); + yield* append( + state.front, + stable.key === "reversed:reverse" ? stable.back : stable.front, + ); + yield* append( + state.back, + stable.key === "reversed:reverse" ? stable.front : stable.back, + ); + } + const reader = Object.freeze({}) as CardContentReader; + readers.set(reader, state); + completed = true; + return reader; + } finally { + input = undefined!; + if (!completed) release(state); + } +} +function position( + reader: CardContentReader, + side: CardContentSide, + start: number, + previous: CardContentCursor | null, +): CardContentCursor { + const value = Object.freeze({}) as CardContentCursor; + cursors.set(value, { reader, side, start, previous }); + return value; +} +function* locate( + field: Field, + offset: number, +): Generator { + let low = 0, + high = field.segments.length; + while (low < high) { + const middle = Math.floor((low + high) / 2), + end = yield* work(32, 1, () => field.segments[middle]!.end); + if (offset < end) high = middle; + else low = middle + 1; + } + return low; +} +function* unitAt( + field: Field, + offset: number, +): Generator { + const i = yield* locate(field, offset), + segment = field.segments[i]; + if (!segment) return NaN; + return yield* work( + 8, + 1, + () => + segment.text.charCodeAt( + segment.from + offset - (segment.end - (segment.to - segment.from)), + ), + 1, + ); +} +function* page( + reader: CardContentReader, + side: CardContentSide, + cursor?: CardContentCursor, +): Generator { + const state = stateOf(reader); + if (side !== "front" && side !== "back") invalid(); + const field = state[side]; + let current: CardContentCursor; + if (cursor !== undefined) { + const at = cursors.get(cursor); + if (!at || at.reader !== reader || at.side !== side) invalid(); + current = cursor; + } else current = yield* work(64, 1, () => position(reader, side, 0, null)); + const at = cursors.get(current)!, + start = at.start; + let end = Math.min(field.total, start + 16384); + if (end < field.total) { + const before = yield* unitAt(field, end - 1), + after = yield* unitAt(field, end); + if ( + before >= 0xd800 && + before <= 0xdbff && + after >= 0xdc00 && + after <= 0xdfff + ) + end--; + } + const chunks: string[] = []; + let index = yield* locate(field, start), + offset = start; + while (offset < end) { + stateOf(reader); + const segment = field.segments[index]!; + const begin = + segment.from + offset - (segment.end - (segment.to - segment.from)), + to = Math.min(segment.to, begin + 4096, begin + end - offset); + yield* work( + (to - begin) * 7 + 64, + 1, + () => chunks.push(segment.text.slice(begin, to)), + to - begin, + ); + offset += to - begin; + if (offset === segment.end) index++; + } + stateOf(reader); + const text = yield* work((end - start) * 2 + 128, chunks.length + 1, () => + chunks.join(""), + ); + stateOf(reader); + const next = + end < field.total + ? yield* work(64, 1, () => position(reader, side, end, current)) + : null; + return Object.freeze({ + text, + start, + end, + total: field.total, + previous: at.previous, + next, + }); +} +export function readCardContentPage( + reader: CardContentReader, + side: CardContentSide, + cursor?: CardContentCursor, +): Generator { + const state = stateOf(reader); + let iterator!: Generator; + iterator = (function* () { + try { + return yield* page(reader, side, cursor); + } finally { + state.pages.delete(iterator); + } + })(); + state.pages.add(iterator); + return iterator; +} diff --git a/web/src/study/csv.ts b/web/src/study/csv.ts new file mode 100644 index 0000000..0ad22c6 --- /dev/null +++ b/web/src/study/csv.ts @@ -0,0 +1,82 @@ +import type { StudyEntry, StudyState } from "../collections/contracts"; +import { cardText } from "./model"; +import { initialSchedule } from "./scheduler"; + +const MAX_BYTES = 10_000_000; +/** RFC-style quoted CSV. Parse and validate the whole input before creating cards. */ +export function parseStudyCsv(source: string): { front: string; back: string; tags: string[] }[] { + if (new TextEncoder().encode(source).length > MAX_BYTES || source.includes("\0")) throw new Error("invalid_csv: Choose a text CSV file up to 10 MB."); + source = source.replace(/^\uFEFF/, ""); + const rows: string[][] = []; + let row: string[] = [], field = "", quoted = false, closed = false; + const finishField = () => { row.push(field); field = ""; closed = false; }; + const finishRow = () => { + finishField(); + if (row.some(value => value.trim())) rows.push(row); + row = []; + if (rows.length > 10001) throw new Error("csv_limit: Import at most 10,000 rows."); + }; + for (let i = 0; i < source.length; i++) { + const char = source[i]!; + if (quoted) { + if (char === '"') { + if (source[i + 1] === '"') { field += '"'; i++; } + else { quoted = false; closed = true; } + } else field += char; + } else if (char === '"' && !field && !closed) quoted = true; + else if (char === ",") finishField(); + else if (char === "\n" || char === "\r") { + if (char === "\r" && source[i + 1] === "\n") i++; + finishRow(); + } else { + if (closed || char === '"') throw new Error("invalid_csv: Check the quoted fields in this file."); + field += char; + } + if (field.length > 200_000 || row.length > 3) throw new Error("invalid_csv: Expected Front, Back and optional Tags columns."); + } + if (quoted) throw new Error("invalid_csv: A quoted field was not closed."); + if (field || row.length || closed) finishRow(); + const header = rows.shift()?.map(value => value.trim().toLowerCase()); + if (!header || header[0] !== "front" || header[1] !== "back" || (header.length !== 2 && !(header.length === 3 && header[2] === "tags"))) + throw new Error("invalid_csv: Start with Front,Back or Front,Back,Tags headers."); + if (!rows.length) throw new Error("empty_csv: This file has no cards to import."); + return rows.map((values, index) => { + if (values.length !== header.length || !values[0]?.trim() || !values[1]?.trim() || values[0].length > 100_000 || values[1].length > 100_000) + throw new Error(`invalid_csv: Row ${index + 2} needs a question and an answer, each up to 100,000 characters.`); + let tags: unknown = []; + if (values[2]?.trim()) { + try { tags = JSON.parse(values[2]); } catch { throw new Error(`invalid_csv: Row ${index + 2} Tags must be a JSON list, such as ["biology"].`); } + } + if (!Array.isArray(tags) || tags.length > 100 || tags.some(tag => typeof tag !== "string" || !tag.trim() || tag.length > 100)) + throw new Error(`invalid_csv: Check the tags on row ${index + 2}.`); + return { front: values[0], back: values[1], tags: [...new Set(tags as string[])] }; + }); +} + +export function importStudyCsv(state: StudyState, deckId: string, source: string, at: Date): StudyState { + const rows = parseStudyCsv(source); + if (state.entries.length + rows.length > 10_000 || state.cards.length + rows.length > 20_000) throw new Error("card_limit: This import would exceed the card limit."); + // Only return a new state after all rows have succeeded. No external writes here. + if (!state.decks.some(deck => deck.id === deckId)) throw new Error("deck_missing: Choose an existing deck."); + const next = structuredClone(state); + for (const row of rows) { + const entry: StudyEntry = { ...row, id: crypto.randomUUID(), deckId, revision: 1, source: null, type: "basic" }; + next.entries.push(entry); + next.cards.push({ id: crypto.randomUUID(), entryId: entry.id, key: "basic:forward", revision: 1, schedule: initialSchedule(at) }); + } + return next; +} + +export function exportStudyCsv(state: StudyState, deckId: string): string { + if (!state.decks.some(deck => deck.id === deckId)) throw new Error("deck_missing: This deck no longer exists."); + const cell = (value: string) => `"${value.replaceAll('"', '""')}"`; + const rows = ["Front,Back,Tags"]; + const entries = new Map(state.entries.filter(entry => entry.deckId === deckId).map(entry => [entry.id, entry])); + for (const card of state.cards) { + const entry = entries.get(card.entryId); + if (!entry) continue; + const rendered = cardText(entry, card.key); + rows.push([rendered.front, rendered.back, JSON.stringify(entry.tags)].map(cell).join(",")); + } + return `${rows.join("\r\n")}\r\n`; +} diff --git a/web/src/study/model.ts b/web/src/study/model.ts new file mode 100644 index 0000000..dbea1c5 --- /dev/null +++ b/web/src/study/model.ts @@ -0,0 +1,151 @@ +import type { CardKey, Grade, ReviewCard, StudyEntry, StudyState } from "../collections/contracts"; +import { initialSchedule, nextSchedule, studyDay } from "./scheduler"; + +function text(value: string, label: string, limit = 100_000) { + if (typeof value !== "string" || !value.trim() || value.length > limit) + throw new Error(`invalid_${label}: Enter ${label.replaceAll("_", " ")} within ${limit} characters.`); +} +export function createDeck(state: StudyState, title: string, at: Date): StudyState { + text(title, "deck_name", 500); + if (state.decks.length >= 1000) throw new Error("deck_limit: You can create up to 1,000 decks."); + const next = structuredClone(state); + next.decks.push({ id: crypto.randomUUID(), title: title.trim(), createdAt: at.toISOString(), updatedAt: at.toISOString(), revision: 1 }); + return next; +} + +function keys(entry: StudyEntry): CardKey[] { + switch (entry.type) { + case "basic": case "reversed": + text(entry.front, "question"); text(entry.back, "answer"); + return entry.type === "basic" ? ["basic:forward"] : ["reversed:forward", "reversed:reverse"]; + case "cloze": { + text(entry.text, "cloze_text"); + if (!entry.blanks.length || entry.blanks.length > 100) throw new Error("invalid_cloze: Add between 1 and 100 blanks."); + const ids = new Set(); + for (const blank of entry.blanks) { + if (!/^[A-Za-z0-9_-]{1,100}$/.test(blank.id) || ids.has(blank.id)) throw new Error("invalid_cloze: Each blank needs a unique identifier."); + ids.add(blank.id); text(blank.answer, "cloze_answer"); + if (entry.text.split(`{{${blank.id}}}`).length !== 2) throw new Error("invalid_cloze: Include each blank exactly once in the text."); + } + if ([...entry.text.matchAll(/\{\{([^{}]+)\}\}/g)].some(match => !ids.has(match[1]!))) + throw new Error("invalid_cloze: Give every placeholder an answer."); + return entry.blanks.map(blank => `cloze:${blank.id}` as const); + } + default: throw new Error("invalid_card_type: Choose Basic, Reversed or Fill in the blank."); + } +} + +export function saveEntry(state: StudyState, entry: StudyEntry, at: Date): StudyState { + if (!state.decks.some(deck => deck.id === entry.deckId)) throw new Error("deck_missing: Choose an existing deck."); + const nextKeys = keys(entry); + const current = state.entries.find(item => item.id === entry.id); + if (entry.revision !== (current?.revision ?? 0) + 1) throw new Error("entry_conflict: Reload this card before saving."); + if (!current && state.entries.length >= 10_000) throw new Error("entry_limit: You can store up to 10,000 card entries."); + const next = structuredClone(state); + next.entries = next.entries.filter(item => item.id !== entry.id); + next.entries.push(structuredClone(entry)); + const existing = state.cards.filter(card => card.entryId === entry.id); + next.cards = next.cards.filter(card => card.entryId !== entry.id); + for (const key of nextKeys) { + const card = existing.find(card => card.key === key); + next.cards.push(card ? { ...structuredClone(card), revision: card.revision + 1 } : { id: crypto.randomUUID(), entryId: entry.id, key, revision: 1, schedule: initialSchedule(at) }); + } + if (next.cards.length > 20_000) throw new Error("card_limit: You can store up to 20,000 review cards."); + return next; +} + +export function deleteEntry(state: StudyState, entryId: string): StudyState { + const next = structuredClone(state); + next.entries = next.entries.filter(entry => entry.id !== entryId); + next.cards = next.cards.filter(card => card.entryId !== entryId); + return next; +} + +export function cardText(entry: StudyEntry, key: CardKey): { front: string; back: string } { + if (entry.type === "cloze") { + if (!key.startsWith("cloze:") || !entry.blanks.some(blank => `cloze:${blank.id}` === key)) throw new Error("card_missing: This blank no longer exists."); + const render = (question: boolean) => entry.text.replace(/\{\{([^{}]+)\}\}/g, (_, id: string) => question && key === `cloze:${id}` ? "[...]" : entry.blanks.find(blank => blank.id === id)?.answer ?? "[unavailable]"); + return { front: render(true), back: render(false) }; + } + if (!keys(entry).includes(key)) throw new Error("card_missing: This card direction no longer exists."); + return key === "reversed:reverse" ? { front: entry.back, back: entry.front } : { front: entry.front, back: entry.back }; +} + +export function gradeCard(state: StudyState, cardId: string, grade: Grade, sessionId: string, operationId: string, at: Date): StudyState { + const previous = state.events.find(event => event.id === operationId); + if (previous) { + if (previous.cardId !== cardId || previous.grade !== grade || previous.sessionId !== sessionId || previous.at !== at.toISOString() || previous.undoneAt) + throw new Error("operation_conflict: This review operation was already used."); + return structuredClone(state); + } + const card = state.cards.find(card => card.id === cardId); + if (!card) throw new Error("card_missing: This card no longer exists."); + if (state.events.length >= 100_000) throw new Error("review_limit: Review history is full. Export your data before continuing."); + const schedule = nextSchedule(card.schedule, grade, at); + const next = structuredClone(state); + const result = next.cards.find(item => item.id === cardId)!; + result.schedule = schedule; + result.revision++; + next.events.push({ id: operationId, cardId, sessionId, at: at.toISOString(), grade, algorithm: "orbyn-interval-v1", before: { schedule: structuredClone(card.schedule), cardRevision: card.revision }, after: { schedule: structuredClone(schedule), cardRevision: result.revision }, undoneAt: null }); + return next; +} + +export function undoReview(state: StudyState, eventId: string, at: Date): StudyState { + const event = state.events.find(event => event.id === eventId); + const card = state.cards.find(card => card.id === event?.cardId); + if (!event || event.undoneAt || !card || card.revision !== event.after.cardRevision || + JSON.stringify(card.schedule) !== JSON.stringify(event.after.schedule) || + state.events.filter(item => item.cardId === card.id && !item.undoneAt).at(-1)?.id !== event.id) + throw new Error("undo_unavailable: This card changed after that review. Its latest schedule was kept."); + const next = structuredClone(state); + const target = next.cards.find(item => item.id === card.id)!; + target.schedule = structuredClone(event.before.schedule); + target.revision++; + next.events.find(item => item.id === eventId)!.undoneAt = at.toISOString(); + return next; +} + +export function reviewQueue(state: StudyState, at: Date, deckId?: string): ReviewCard[] { + const prefs = state.preferences; + const day = studyDay(at, prefs.timezone, prefs.dayStartHour); + const today = state.events.filter(event => !event.undoneAt && studyDay(new Date(event.at), prefs.timezone, prefs.dayStartHour) === day); + const admitted = new Set(today.map(event => event.cardId)); + const newAdmissions = new Set(today.filter(event => event.before.schedule.state === "new").map(event => event.cardId)); + const reviewAdmissions = new Set(today.filter(event => event.before.schedule.state !== "new" && !newAdmissions.has(event.cardId)).map(event => event.cardId)); + let fresh = Math.max(0, prefs.newPerDay - newAdmissions.size), reviews = Math.max(0, prefs.reviewsPerDay - reviewAdmissions.size); + const deckEntries = deckId ? new Set(state.entries.filter(entry => entry.deckId === deckId).map(entry => entry.id)) : null; + const candidates = state.cards.filter(card => Date.parse(card.schedule.due) <= +at && (!deckEntries || deckEntries.has(card.entryId))) + .sort((a, b) => Number(a.schedule.state === "new") - Number(b.schedule.state === "new") || Date.parse(a.schedule.due) - Date.parse(b.schedule.due) || a.id.localeCompare(b.id)); + return structuredClone(candidates.filter(card => { + if (admitted.has(card.id)) return true; + if (card.schedule.state === "new") return fresh-- > 0; + return reviews-- > 0; + })); +} + +function shiftDay(day: string, offset: number): string { + const date = new Date(`${day}T12:00:00Z`); + date.setUTCDate(date.getUTCDate() + offset); + return date.toISOString().slice(0, 10); +} +export function progress(state: StudyState, at: Date) { + const prefs = state.preferences; + const today = studyDay(at, prefs.timezone, prefs.dayStartHour); + const active = state.events.filter(event => !event.undoneAt && Date.parse(event.at) <= +at); + const dated = active.map(event => ({ event, day: studyDay(new Date(event.at), prefs.timezone, prefs.dayStartHour) })); + const recent = dated.filter(item => item.day >= shiftDay(today, -29)); + const dayOfWeek = new Date(`${today}T12:00:00Z`).getUTCDay(); + const monday = shiftDay(today, -(dayOfWeek + 6) % 7); + const days = new Set(dated.map(item => item.day)); + let streak = 0, cursor = days.has(today) ? today : shiftDay(today, -1); + while (days.has(cursor)) { streak++; cursor = shiftDay(cursor, -1); } + const dueNext14Days = Array.from({ length: 14 }, (_, i) => ({ day: shiftDay(today, i), count: 0 })); + for (const card of state.cards) { + const due = studyDay(new Date(card.schedule.due), prefs.timezone, prefs.dayStartHour); + const bucket = due <= today ? dueNext14Days[0] : dueNext14Days.find(item => item.day === due); + if (bucket) bucket.count++; + } + return { reviewedThisWeek: dated.filter(item => item.day >= monday).length, reviewedLast30Days: recent.length, + remembered: recent.length ? recent.filter(item => item.event.grade === "good" || item.event.grade === "easy").length / recent.length : null, + streak, dueNext14Days, cardCount: state.cards.length }; +} diff --git a/web/src/study/scheduler.ts b/web/src/study/scheduler.ts new file mode 100644 index 0000000..ddb0ab1 --- /dev/null +++ b/web/src/study/scheduler.ts @@ -0,0 +1,77 @@ +import type { Grade, Schedule } from "../collections/contracts"; + +const DAY = 86_400_000; +const dayFormatters = new Map(); +function instant(at: Date): number { + const time = at.getTime(); + if (!Number.isFinite(time)) throw new Error("invalid_date: Choose a valid review time."); + return time; +} + +/** Local interval policy v1. Preview and commit must call this same function. */ +export function initialSchedule(at: Date): Schedule { + return { version: 1, state: "new", due: new Date(instant(at)).toISOString(), intervalDays: 0, ease: 2.5, repetitions: 0, lapses: 0, lastReviewedAt: null }; +} + +export function nextSchedule(previous: Schedule, grade: Grade, at: Date): Schedule { + const time = instant(at); + if (previous.version !== 1 || !["new", "learning", "review"].includes(previous.state) || + !Number.isFinite(previous.intervalDays) || previous.intervalDays < 0 || previous.intervalDays > 36500 || + !Number.isFinite(previous.ease) || previous.ease < 1.3 || previous.ease > 3 || + !Number.isSafeInteger(previous.repetitions) || previous.repetitions < 0 || + !Number.isSafeInteger(previous.lapses) || previous.lapses < 0) + throw new Error("invalid_schedule: This card's review schedule could not be read."); + const next = { ...previous, lastReviewedAt: at.toISOString() }; + let delay: number; + switch (grade) { + case "again": + next.state = "learning"; + next.intervalDays = 0; + next.repetitions = 0; + next.lapses++; + delay = 60_000; + break; + case "hard": + next.ease = Math.max(1.3, next.ease - 0.15); + if (previous.state === "review") { + next.intervalDays = Math.min(36500, Math.max(1, Math.ceil(previous.intervalDays * 1.2))); + delay = next.intervalDays * DAY; + } else { + next.state = "learning"; + next.intervalDays = 0; + delay = 600_000; + } + break; + case "good": + case "easy": + next.state = "review"; + next.repetitions++; + next.intervalDays = Math.min(36500, previous.state === "review" + ? Math.max(grade === "easy" ? 4 : 1, Math.ceil(previous.intervalDays * previous.ease * (grade === "easy" ? 1.3 : 1))) + : grade === "easy" ? 4 : 1); + if (grade === "easy") next.ease = Math.min(3, next.ease + 0.15); + delay = next.intervalDays * DAY; + break; + default: + throw new Error("invalid_grade: Choose Again, Hard, Good or Easy."); + } + next.due = new Date(time + delay).toISOString(); + return next; +} + +/** Calendar grouping, distinct from elapsed schedule intervals. Handles DST. */ +export function studyDay(at: Date, timeZone: string, dayStartHour: number): string { + instant(at); + if (!Number.isInteger(dayStartHour) || dayStartHour < 0 || dayStartHour > 23) + throw new Error("invalid_day_start: Choose a whole hour from 0 to 23."); + let formatter = dayFormatters.get(timeZone); + if (!formatter) { + formatter = new Intl.DateTimeFormat("en-US", { timeZone, year: "numeric", month: "2-digit", day: "2-digit", hour: "2-digit", hourCycle: "h23" }); + if (dayFormatters.size >= 16) dayFormatters.delete(dayFormatters.keys().next().value!); + dayFormatters.set(timeZone, formatter); + } + const parts = formatter.formatToParts(at); + const value = (type: Intl.DateTimeFormatPartTypes) => Number(parts.find(part => part.type === type)?.value); + const calendar = new Date(Date.UTC(value("year"), value("month") - 1, value("day") - (value("hour") < dayStartHour ? 1 : 0))); + return calendar.toISOString().slice(0, 10); +} diff --git a/web/src/study/sourceCodec.ts b/web/src/study/sourceCodec.ts new file mode 100644 index 0000000..5c58152 --- /dev/null +++ b/web/src/study/sourceCodec.ts @@ -0,0 +1,564 @@ +import { Node as PMNode, type Schema } from "@tiptap/pm/model"; +import { + Step, + AddMarkStep, + RemoveMarkStep, + AddNodeMarkStep, + RemoveNodeMarkStep, + AttrStep, + DocAttrStep, + ReplaceStep, + ReplaceAroundStep, +} from "@tiptap/pm/transform"; +import type { NoteDocument } from "../domain/contracts"; +import { validateNoteDocument, safeHref } from "../domain/validation"; +import { createNoteSchema } from "../editor/schema"; + +export type SourcePassage = Readonly<{ + from: number; + to: number; + quote: string; +}>; +export type SourceSelection = Readonly<{ + passage: SourcePassage; + context: Readonly<{ + text: string; + selectionFrom: number; + selectionTo: number; + }>; +}>; +export type SourceFingerprint = Readonly<{ + document: NoteDocument; + canonicalJSON: string; + digest: string; +}>; +export type SourceEditProof = Readonly<{ + baseDigest: string; + finalDigest: string; + steps: readonly unknown[]; +}>; +declare const verifiedBrand: unique symbol; +export type VerifiedSourceEdit = Readonly<{ [verifiedBrand]: true }>; +export type SourceMapping = + | { status: "attached"; passage: SourcePassage } + | { status: "stale"; reason: "passage-edited" }; +export class SourceCodecError extends Error { + constructor( + public readonly code: "source_codec_invalid" | "source_codec_limit", + ) { + super( + code === "source_codec_limit" + ? "This source selection or edit exceeds the supported verification limits." + : "This source passage or edit could not be verified.", + ); + this.name = "SourceCodecError"; + } +} +export const SOURCE_CODEC_LIMITS = Object.freeze({ + steps: 1000, + proofBytes: 2 * 1024 * 1024, + text: 100000, + replayUnits: 64 * 1024 * 1024, +}); +type Transition = { before: PMNode; after: PMNode; step: Step }; +type Verified = { + base: PMNode; + final: PMNode; + transitions: Transition[]; + cost: { replayUnits: number; stepCount: number; maxDocumentUnits: number }; +}; +const verified = new WeakMap(); +function invalid(): never { + throw new SourceCodecError("source_codec_invalid"); +} +function limit(): never { + throw new SourceCodecError("source_codec_limit"); +} +function record(value: unknown): Record { + if ( + !value || + typeof value !== "object" || + Array.isArray(value) || + ![Object.prototype, null].includes(Object.getPrototypeOf(value)) + ) + invalid(); + return value as Record; +} +function stable(value: unknown, depth = 0): unknown { + if (depth > 64) invalid(); + if (value === null || typeof value === "string" || typeof value === "boolean") + return value; + if (typeof value === "number") { + if (!Number.isFinite(value)) invalid(); + return value; + } + if (Array.isArray(value)) return value.map((item) => stable(item, depth + 1)); + const row = record(value), + output: Record = Object.create(null); + for (const key of Object.keys(row).sort()) + output[key] = stable(row[key], depth + 1); + return output; +} +function rawMark(value: unknown, schema: Schema): void { + const mark = record(value); + if ( + typeof mark.type !== "string" || + !Object.hasOwn(schema.marks, mark.type) || + Object.keys(mark).some((key) => !["type", "attrs"].includes(key)) + ) + invalid(); + const definition = schema.marks[mark.type]!; + if (mark.attrs !== undefined) { + const attrs = record(mark.attrs); + if ( + Object.keys(attrs).some( + (key) => !Object.hasOwn(definition.spec.attrs ?? {}, key), + ) + ) + invalid(); + stable(attrs); + } + if (mark.type === "link") { + const href = (mark.attrs as Record | undefined)?.href; + if (!safeHref(href)) invalid(); + try { + if (!["http:", "https:", "mailto:"].includes(new URL(href).protocol)) + invalid(); + } catch { + invalid(); + } + } +} +function rawNode(value: unknown, schema: Schema): void { + const row = record(value); + if ( + Object.keys(row).some( + (key) => !["type", "attrs", "text", "marks", "content"].includes(key), + ) + ) + invalid(); + if (typeof row.type !== "string" || !Object.hasOwn(schema.nodes, row.type)) + invalid(); + if (row.text !== undefined && row.type !== "text") invalid(); + const type = schema.nodes[row.type]!; + if (row.attrs !== undefined) { + const attrs = record(row.attrs); + if ( + Object.keys(attrs).some( + (key) => !Object.hasOwn(type.spec.attrs ?? {}, key), + ) + ) + invalid(); + stable(attrs); + } + if (row.marks !== undefined) { + if (!Array.isArray(row.marks)) invalid(); + for (const value of row.marks) rawMark(value, schema); + } + if (row.type === "text" && row.content !== undefined) invalid(); + if (row.content !== undefined) { + if (!Array.isArray(row.content)) invalid(); + for (const child of row.content) rawNode(child, schema); + } +} +function canonical( + input: unknown, + schema: Schema, +): { document: NoteDocument; canonicalJSON: string; node: PMNode } { + try { + const row = record(input); + const emptyEditorJSON = row.type === "doc" && Object.keys(row).length === 1; + const checked = validateNoteDocument( + emptyEditorJSON ? { type: "doc", content: [] } : input, + ); + rawNode(checked, schema); + // The existing editor admits a stored legacy empty doc until its first insertion. + // Keep that exact base for transaction replay; never invent an unrecorded paragraph. + const node = schema.nodeFromJSON(checked); + if (checked.content.length) node.check(); + const serialized = node.toJSON(); + const document = validateNoteDocument( + node.childCount ? serialized : { ...serialized, content: [] }, + ); + rawNode(document, schema); + return { document, canonicalJSON: JSON.stringify(stable(document)), node }; + } catch (error) { + if (error instanceof SourceCodecError) throw error; + invalid(); + } +} +export function canonicalSourceDocument(input: unknown): { + document: NoteDocument; + canonicalJSON: string; +} { + const result = canonical(input, createNoteSchema()); + return { document: result.document, canonicalJSON: result.canonicalJSON }; +} +function integer(value: unknown, max: number): asserts value is number { + if ( + !Number.isSafeInteger(value) || + (value as number) < 0 || + (value as number) > max + ) + invalid(); +} +function scalarBoundary(doc: PMNode, position: number): void { + const resolved = doc.resolve(position), + before = resolved.nodeBefore, + after = resolved.nodeAfter; + const left = before?.isText + ? before.text?.charCodeAt(before.text.length - 1) + : undefined; + const right = after?.isText ? after.text?.charCodeAt(0) : undefined; + if ( + left !== undefined && + right !== undefined && + left >= 0xd800 && + left <= 0xdbff && + right >= 0xdc00 && + right <= 0xdfff + ) + invalid(); +} +const leafText = (node: PMNode) => + node.type.name === "hardBreak" ? "\n" : "\ufffc"; +function projection(doc: PMNode, from: number, to: number): string { + return doc.textBetween(from, to, "\n", leafText); +} +function checkedPassage( + doc: PMNode, + from: number, + to: number, + expected?: string, +): SourcePassage { + integer(from, doc.content.size); + integer(to, doc.content.size); + if ( + from >= to || + !doc.resolve(from).parent.isTextblock || + !doc.resolve(to).parent.isTextblock + ) + invalid(); + scalarBoundary(doc, from); + scalarBoundary(doc, to); + doc.nodesBetween(from, to, (node) => { + if (node.isLeaf && !node.isText && node.type.name !== "hardBreak") + invalid(); + }); + const quote = projection(doc, from, to); + if (!quote.trim()) invalid(); + if (quote.length > SOURCE_CODEC_LIMITS.text) limit(); + if (expected !== undefined && quote !== expected) invalid(); + return Object.freeze({ from, to, quote }); +} +export function captureSourceSelection( + document: NoteDocument, + from: number, + to: number, +): SourceSelection { + const { node } = canonical(document, createNoteSchema()), + passage = checkedPassage(node, from, to); + let contextFrom = node.resolve(from).start(), + contextTo = node.resolve(to).end(); + // A reference's authorized display title is not persisted prose. Bound the + // real text context at adjacent chips, never insert an object marker/title + // into a Cloze answer or search for an equal quote somewhere else. + node.nodesBetween(contextFrom, contextTo, (child, position) => { + if (child.type.name !== "noteLink") return; + if (position + child.nodeSize <= from) + contextFrom = Math.max(contextFrom, position + child.nodeSize); + else if (position >= to) contextTo = Math.min(contextTo, position); + }); + const text = projection(node, contextFrom, contextTo), + selectionFrom = projection(node, contextFrom, from).length; + if (text.length > SOURCE_CODEC_LIMITS.text) limit(); + const selectionTo = selectionFrom + passage.quote.length; + if (text.slice(selectionFrom, selectionTo) !== passage.quote) invalid(); + return Object.freeze({ + passage, + context: Object.freeze({ text, selectionFrom, selectionTo }), + }); +} +export async function fingerprintSourceDocument( + document: NoteDocument, +): Promise { + const result = canonicalSourceDocument(document), + bytes = new TextEncoder().encode(result.canonicalJSON); + try { + const hash = new Uint8Array(await crypto.subtle.digest("SHA-256", bytes)); + try { + return Object.freeze({ + ...result, + digest: Array.from(hash, (byte) => + byte.toString(16).padStart(2, "0"), + ).join(""), + }); + } finally { + hash.fill(0); + } + } finally { + bytes.fill(0); + } +} +function stepRows(input: readonly unknown[]): Record[] { + if (!Array.isArray(input)) invalid(); + if (input.length > SOURCE_CODEC_LIMITS.steps) limit(); + let copy: unknown; + try { + copy = structuredClone(input); + } catch { + invalid(); + } + const rows = copy as unknown[], + bytes = new TextEncoder().encode(JSON.stringify(stable(rows))); + if (bytes.byteLength > SOURCE_CODEC_LIMITS.proofBytes) limit(); + const result: Record[] = []; + let count = 0; + for (const value of rows) { + const row = record(value); + if (++count > SOURCE_CODEC_LIMITS.steps) limit(); + if (row.stepType === "markSet") { + if ( + Object.keys(row).some((key) => !["stepType", "steps"].includes(key)) || + !Array.isArray(row.steps) + ) + invalid(); + for (const child of row.steps) { + const next = record(child); + if (++count > SOURCE_CODEC_LIMITS.steps) limit(); + if (next.stepType !== "addMark" && next.stepType !== "removeMark") + invalid(); + result.push(next); + } + } else result.push(row); + } + return result; +} +function checkedStep( + row: Record, + doc: PMNode, + schema: Schema, +): Step { + const keys: Record = { + addMark: ["stepType", "from", "to", "mark"], + removeMark: ["stepType", "from", "to", "mark"], + addNodeMark: ["stepType", "pos", "mark"], + removeNodeMark: ["stepType", "pos", "mark"], + attr: ["stepType", "pos", "attr", "value"], + docAttr: ["stepType", "attr", "value"], + replace: ["stepType", "from", "to", "slice", "structure"], + replaceAround: [ + "stepType", + "from", + "to", + "gapFrom", + "gapTo", + "insert", + "slice", + "structure", + ], + }; + if ( + typeof row.stepType !== "string" || + !Object.hasOwn(keys, row.stepType) || + Object.keys(row).some((key) => !keys[row.stepType as string]!.includes(key)) + ) + invalid(); + const required = keys[row.stepType]!.filter( + (key) => !["slice", "structure"].includes(key), + ); + if (required.some((key) => !Object.hasOwn(row, key))) invalid(); + if (Object.hasOwn(row, "mark")) rawMark(row.mark, schema); + if (Object.hasOwn(row, "slice") && row.slice !== null) { + const slice = record(row.slice); + if ( + Object.keys(slice).some( + (key) => !["content", "openStart", "openEnd"].includes(key), + ) + ) + invalid(); + for (const key of ["openStart", "openEnd"]) + if (Object.hasOwn(slice, key)) integer(slice[key], 32); + if (slice.content !== undefined) { + if (!Array.isArray(slice.content)) invalid(); + for (const node of slice.content) rawNode(node, schema); + } + } + for (const key of ["from", "to", "pos", "gapFrom", "gapTo"]) + if (Object.hasOwn(row, key)) integer(row[key], doc.content.size); + if (Object.hasOwn(row, "insert")) integer(row.insert, 2000000); + if (Object.hasOwn(row, "structure") && typeof row.structure !== "boolean") + invalid(); + if (row.stepType === "attr" || row.stepType === "docAttr") { + const target = + row.stepType === "docAttr" ? doc : doc.nodeAt(row.pos as number); + if ( + !target || + typeof row.attr !== "string" || + !Object.hasOwn(target.type.spec.attrs ?? {}, row.attr) + ) + invalid(); + } + try { + const step = Step.fromJSON(schema, row); + if (!( + step instanceof AddMarkStep || + step instanceof RemoveMarkStep || + step instanceof AddNodeMarkStep || + step instanceof RemoveNodeMarkStep || + step instanceof AttrStep || + step instanceof DocAttrStep || + step instanceof ReplaceStep || + step instanceof ReplaceAroundStep + )) + invalid(); + if (step instanceof ReplaceStep && step.from > step.to) invalid(); + if ( + step instanceof ReplaceAroundStep && + !( + step.from <= step.gapFrom && + step.gapFrom <= step.gapTo && + step.gapTo <= step.to && + step.insert <= step.slice.size + ) + ) + invalid(); + return step; + } catch (error) { + if (error instanceof SourceCodecError) throw error; + invalid(); + } +} +export function verifySourceEdit( + base: NoteDocument, + final: NoteDocument, + steps: readonly unknown[], +): VerifiedSourceEdit { + const schema = createNoteSchema(), + start = canonical(base, schema), + end = canonical(final, schema), + rows = stepRows(steps); + let node = start.node; + let beforeUnits = Math.max(start.canonicalJSON.length, node.content.size + 1); + let maxDocumentUnits = Math.max( + beforeUnits, + end.canonicalJSON.length, + end.node.content.size + 1, + ); + let work = + beforeUnits + Math.max(end.canonicalJSON.length, end.node.content.size + 1); + const transitions: Transition[] = []; + for (const row of rows) { + work += beforeUnits; + if (work > SOURCE_CODEC_LIMITS.replayUnits) limit(); + const step = checkedStep(row, node, schema); + let after: PMNode; + try { + const result = step.apply(node); + if (result.failed || !result.doc) invalid(); + after = result.doc; + after.check(); + const json = validateNoteDocument(after.toJSON()); + rawNode(json, schema); + const afterUnits = Math.max( + JSON.stringify(json).length, + after.content.size + 1, + ); + work += afterUnits; + if (work > SOURCE_CODEC_LIMITS.replayUnits) limit(); + maxDocumentUnits = Math.max(maxDocumentUnits, afterUnits); + beforeUnits = afterUnits; + } catch (error) { + if (error instanceof SourceCodecError) throw error; + invalid(); + } + if ( + !(step instanceof ReplaceStep || step instanceof ReplaceAroundStep) && + (node.content.size !== after.content.size || + projection(node, 0, node.content.size) !== + projection(after, 0, after.content.size)) + ) + invalid(); + if (step instanceof ReplaceAroundStep) { + const original = node.slice(step.gapFrom, step.gapTo), + moved = after.slice( + step.from + step.insert, + step.from + step.insert + (step.gapTo - step.gapFrom), + ); + if (!original.eq(moved)) invalid(); + } + transitions.push({ before: node, after, step }); + node = after; + } + if (!node.eq(end.node)) invalid(); + const handle = Object.freeze({}) as VerifiedSourceEdit; + verified.set(handle, { + base: start.node, + final: end.node, + transitions, + cost: { + replayUnits: work, + stepCount: transitions.length, + maxDocumentUnits, + }, + }); + return handle; +} +/** Conservative serialized/position traversal cost; includes zero-step base/final validation. */ +export function sourceEditCost(edit: VerifiedSourceEdit): Readonly<{ + replayUnits: number; + stepCount: number; + maxDocumentUnits: number; +}> { + const data = edit && verified.get(edit); + if (!data) invalid(); + return Object.freeze({ ...data.cost }); +} +export function mapSourcePassage( + edit: VerifiedSourceEdit, + passage: SourcePassage, +): SourceMapping { + const data = edit && verified.get(edit); + if (!data) invalid(); + if (typeof record(passage).quote !== "string") invalid(); + let current = checkedPassage( + data.base, + passage.from, + passage.to, + passage.quote, + ); + if ( + data.transitions.length * current.quote.length > + SOURCE_CODEC_LIMITS.replayUnits + ) + limit(); + for (const { step, after } of data.transitions) { + let stale = false; + if (step instanceof ReplaceStep) { + stale = + step.from === step.to + ? current.from < step.from && step.from < current.to + : step.from < current.to && step.to > current.from; + } else if (step instanceof ReplaceAroundStep) { + const intersects = step.from < current.to && step.to > current.from; + stale = + intersects && + !(current.from >= step.gapFrom && current.to <= step.gapTo); + } + if (stale) return { status: "stale", reason: "passage-edited" }; + const map = step.getMap(), + from = map.map(current.from, 1), + to = map.map(current.to, -1); + try { + current = checkedPassage(after, from, to, current.quote); + } catch (error) { + if ( + error instanceof SourceCodecError && + error.code === "source_codec_invalid" + ) + return { status: "stale", reason: "passage-edited" }; + throw error; + } + } + return { status: "attached", passage: current }; +} diff --git a/web/src/study/sourceContracts.ts b/web/src/study/sourceContracts.ts new file mode 100644 index 0000000..d2aac78 --- /dev/null +++ b/web/src/study/sourceContracts.ts @@ -0,0 +1,193 @@ +import type { Note, NoteDocument, PersistenceState } from "../domain/contracts"; +import type { + CollectionsRevision, + CollectionsState, + StudyEntry, +} from "../collections/contracts"; +import type { + LibraryState, + LibraryToken, + NotesLibraryStore, +} from "../library/contracts"; +import type { + SourceEditProof, + SourceFingerprint, + SourcePassage, +} from "./sourceCodec"; + +export type AnchoredStudySource = { + version: 2; + noteId: string; + noteRevision: number; + quote: string; + libraryEpoch: string; + anchorId: string; +}; +export type SourceAnchor = { + id: string; + revision: number; + noteId: string; + entryId: string; + birth: { + noteRevision: number; + documentDigest: string; + from: number; + to: number; + }; + noteRevision: number; + from: number; + to: number; + quote: string; + documentDigest: string; + status: "attached" | "stale"; + staleReason: null | "passage-edited" | "document-replaced"; +}; +export type SavedNoteSelection = { + noteId: string; + noteRevision: number; + libraryEpoch: string; + fingerprint: SourceFingerprint; + passage: SourcePassage; +}; +export type MakeCardDraft = { deckId: string; tags: string[] } & ( + | { type: "basic" | "reversed"; front: string; back: string } + | { type: "cloze"; text: string; blanks: { id: string; answer: string }[] } +); +export type SourceExpected = { + library: LibraryToken; + collections: CollectionsRevision; +}; +export type SourceReceipt = { + operationId: string; + digest: string; + kind: "create" | "undo"; + at: string; + entryId: string; + cardIds: string[]; + anchorId: string; + createOperationId: string; + entryRevision: 1; + cardRevisions: number[]; + anchorRevision: 1; +}; +declare const sourceIntentBrand: unique symbol; +export type MakeCardIntent = Readonly<{ + kind: "create"; + operationId: string; + entryId: string; + [sourceIntentBrand]: true; +}>; +export type UndoMakeCardIntent = Readonly<{ + kind: "undo"; + operationId: string; + entryId: string; + [sourceIntentBrand]: true; +}>; +export type SourceResolution = + | { + status: "attached"; + entryId: string; + anchorId: string; + anchorRevision: number; + noteId: string; + noteRevision: number; + documentDigest: string; + from: number; + to: number; + quote: string; + } + | { + status: "unavailable"; + reason: "legacy" | "stale" | "missing" | "trashed" | "locked"; + }; +export type SourceAnnotation = Extract< + SourceResolution, + { status: "attached" } +>; +export type MakeCardResult = { + entryId: string; + cardIds: string[]; + source: SourceResolution; + receipt: SourceReceipt; + changed: boolean; + replayed: boolean; + tokens: SourceExpected; +}; +export interface StudySourceService { + prepareCreate( + selection: SavedNoteSelection, + expected: SourceExpected, + draft: MakeCardDraft, + ): Promise; + prepareUndo( + expected: SourceExpected, + receiptOperationId: string, + ): Promise; + commit(intent: MakeCardIntent | UndoMakeCardIntent): Promise; + resolve(entryId: string): Promise; + annotations(noteId: string): Promise; + subscribe(listener: () => void): () => void; + persistence(): PersistenceState; + onInvalidate(listener: (reason: "epoch" | "closed") => void): () => void; + close(): void; +} +export type SourceErrorCode = + | "source_invalid" + | "source_limit" + | "source_stale" + | "source_revoked" + | "source_undo_unavailable" + | "source_operation_reused"; +export class SourceError extends Error { + constructor( + public readonly code: SourceErrorCode, + message: string, + ) { + super(`${code}: ${message}`); + this.name = "SourceError"; + } +} +export class SourceConflictError extends Error { + readonly code = "source_conflict"; + constructor(public readonly current: SourceExpected) { + super( + "The source or Study collection changed. Reload before trying again.", + ); + this.name = "SourceConflictError"; + } +} +/** Private owner bridge: all calls use the existing library snapshot writer. */ +export type SourceOwnerSnapshot = { + notes: Note[]; + library: LibraryState; + collections: CollectionsState; +}; +export interface SourceOwnerAccess { + read(): Promise; + commit( + command: SourceCommand, + guard: () => void, + ): Promise<{ snapshot: SourceOwnerSnapshot; replayed: boolean }>; +} +export type SourceAwareNoteUpdate = ( + noteId: string, + patch: Partial>, + expectedRevision: number, + proof: SourceEditProof, +) => Promise; +export type SourceCommand = { + operationId: string; + digest: string; + expected: SourceExpected; + preparedAt: string; + entryId: string; +} & ( + | { + kind: "create"; + selection: SavedNoteSelection; + draft: MakeCardDraft; + anchorId: string; + cardIds: string[]; + } + | { kind: "undo"; original: SourceReceipt } +); diff --git a/web/src/study/sourceValidation.ts b/web/src/study/sourceValidation.ts new file mode 100644 index 0000000..7057733 --- /dev/null +++ b/web/src/study/sourceValidation.ts @@ -0,0 +1,260 @@ +import { SourceError } from "./sourceContracts"; +import type { + AnchoredStudySource, + MakeCardDraft, + SavedNoteSelection, + SourceAnchor, + SourceExpected, + SourceReceipt, +} from "./sourceContracts"; +export const SOURCE_LIMITS = { + anchors: 10000, + receipts: 500, + text: 100000, +} as const; +const UUID = + /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; +export function invalid(): never { + throw new SourceError( + "source_invalid", + "Unsupported or corrupt source data.", + ); +} +export function object( + value: unknown, + keys: string[], +): Record { + if ( + !value || + typeof value !== "object" || + Array.isArray(value) || + Object.getPrototypeOf(value) !== Object.prototype + ) + invalid(); + const row = value as Record; + if ( + Object.keys(row).length !== keys.length || + keys.some((key) => !Object.hasOwn(row, key)) + ) + invalid(); + return row; +} +export function id(value: unknown): asserts value is string { + if (typeof value !== "string" || !UUID.test(value)) invalid(); +} +export function integer(value: unknown, min = 0): asserts value is number { + if (typeof value !== "number" || !Number.isSafeInteger(value) || value < min) + invalid(); +} +export function text( + value: unknown, + nonempty = false, + max: number = SOURCE_LIMITS.text, +): asserts value is string { + if ( + typeof value !== "string" || + value.length > max || + (nonempty && !value.trim()) + ) + invalid(); +} +export function digest(value: unknown): asserts value is string { + if (typeof value !== "string" || !/^[a-f0-9]{64}$/.test(value)) invalid(); +} +function date(value: unknown) { + if ( + typeof value !== "string" || + value.length !== 24 || + !Number.isFinite(Date.parse(value)) || + new Date(value).toISOString() !== value + ) + invalid(); +} +function range(row: Record) { + integer(row.from, 1); + integer(row.to, 1); + if (row.from >= row.to || row.to > 2_000_000) invalid(); +} +export function validateAnchoredSource(value: unknown): AnchoredStudySource { + const row = object(value, [ + "version", + "noteId", + "noteRevision", + "quote", + "libraryEpoch", + "anchorId", + ]); + if (row.version !== 2) invalid(); + text(row.noteId, false, 2_000_000); + integer(row.noteRevision, 1); + text(row.quote, true); + id(row.libraryEpoch); + id(row.anchorId); + return structuredClone(value as AnchoredStudySource); +} +export function validateSourceAnchor(value: unknown): SourceAnchor { + const row = object(value, [ + "id", + "revision", + "noteId", + "entryId", + "birth", + "noteRevision", + "from", + "to", + "quote", + "documentDigest", + "status", + "staleReason", + ]); + id(row.id); + id(row.entryId); + text(row.noteId, false, 2_000_000); + integer(row.revision, 1); + integer(row.noteRevision, 1); + range(row); + text(row.quote, true); + digest(row.documentDigest); + const birth = object(row.birth, [ + "noteRevision", + "documentDigest", + "from", + "to", + ]); + integer(birth.noteRevision, 1); + digest(birth.documentDigest); + range(birth); + if ( + birth.noteRevision > row.noteRevision || + (row.status === "attached" + ? row.staleReason !== null + : row.status !== "stale" || + !["passage-edited", "document-replaced"].includes( + row.staleReason as string, + )) + ) + invalid(); + return structuredClone(value as SourceAnchor); +} +export function validateSourceReceipt(value: unknown): SourceReceipt { + const row = object(value, [ + "operationId", + "digest", + "kind", + "at", + "entryId", + "cardIds", + "anchorId", + "createOperationId", + "entryRevision", + "cardRevisions", + "anchorRevision", + ]); + for (const key of ["operationId", "entryId", "anchorId", "createOperationId"]) + id(row[key]); + digest(row.digest); + date(row.at); + if ( + !["create", "undo"].includes(row.kind as string) || + row.entryRevision !== 1 || + row.anchorRevision !== 1 || + !Array.isArray(row.cardIds) || + !Array.isArray(row.cardRevisions) || + !row.cardIds.length || + row.cardIds.length > 100 || + row.cardIds.length !== row.cardRevisions.length || + new Set(row.cardIds).size !== row.cardIds.length + ) + invalid(); + for (const value of row.cardIds) id(value); + for (const value of row.cardRevisions) if (value !== 1) invalid(); + if (row.kind === "create" && row.operationId !== row.createOperationId) + invalid(); + return structuredClone(value as SourceReceipt); +} +export function validateSourceExpected(value: unknown): SourceExpected { + const row = object(value, ["library", "collections"]); + for (const key of ["library", "collections"]) { + const token = object(row[key], ["epoch", "revision"]); + id(token.epoch); + integer(token.revision); + } + return structuredClone(value as SourceExpected); +} +export function validateMakeCardDraft(value: unknown): MakeCardDraft { + if (!value || typeof value !== "object") invalid(); + const kind = (value as { type?: unknown }).type; + const row = object(value, [ + "deckId", + "tags", + "type", + ...(kind === "cloze" ? ["text", "blanks"] : ["front", "back"]), + ]); + id(row.deckId); + if ( + !Array.isArray(row.tags) || + row.tags.length > 100 || + new Set(row.tags).size !== row.tags.length + ) + invalid(); + for (const tag of row.tags) text(tag, true, 100); + if (kind === "basic" || kind === "reversed") { + text(row.front, true); + text(row.back, true); + } else if (kind === "cloze") { + text(row.text, true); + if ( + !Array.isArray(row.blanks) || + !row.blanks.length || + row.blanks.length > 100 + ) + invalid(); + const ids = new Set(); + for (const blank of row.blanks) { + const b = object(blank, ["id", "answer"]); + if ( + typeof b.id !== "string" || + !/^[A-Za-z0-9_-]{1,100}$/.test(b.id) || + ids.has(b.id) + ) + invalid(); + ids.add(b.id); + text(b.answer, true); + } + const matches = [...row.text.matchAll(/\{\{([^{}]+)\}\}/g)].map( + (match) => match[1]!, + ); + if ( + matches.length !== ids.size || + new Set(matches).size !== matches.length || + matches.some((key) => !ids.has(key)) || + row.text.replace(/\{\{[^{}]+\}\}/g, "").includes("{{") || + row.text.replace(/\{\{[^{}]+\}\}/g, "").includes("}}") + ) + invalid(); + } else invalid(); + return structuredClone(value as MakeCardDraft); +} +export function validateSavedSelection(value: unknown): SavedNoteSelection { + const row = object(value, [ + "noteId", + "noteRevision", + "libraryEpoch", + "fingerprint", + "passage", + ]); + text(row.noteId, false, 2_000_000); + integer(row.noteRevision, 1); + id(row.libraryEpoch); + const passage = object(row.passage, ["from", "to", "quote"]); + range(passage); + text(passage.quote, true); + const fingerprint = object(row.fingerprint, [ + "document", + "canonicalJSON", + "digest", + ]); + text(fingerprint.canonicalJSON, true, 2_000_000); + digest(fingerprint.digest); + return structuredClone(value as SavedNoteSelection); +} diff --git a/web/tests/data/card-content.test.ts b/web/tests/data/card-content.test.ts new file mode 100644 index 0000000..2cd9e8d --- /dev/null +++ b/web/tests/data/card-content.test.ts @@ -0,0 +1,409 @@ +// @vitest-environment node +import { it, expect } from "vitest"; +import { + buildCardContent, + readCardContentPage, + disposeCardContent, + type CardContentInput, + type CardContentReader, + type CardContentSide, +} from "../../src/study/cardContent"; +import { SearchWork, type WorkEvent } from "../../src/search/work"; +import { cardText } from "../../src/study/model"; +import type { StudyEntry } from "../../src/collections/contracts"; +async function done(iterator: Generator) { + const job = new SearchWork(iterator, undefined, async () => {}); + for (let round = 0; round < 100; round++) { + const result = await job.round(); + if (result.status === "done") return result.value; + } + throw new Error("Unexpected noncompletion."); +} +async function all(reader: CardContentReader, side: CardContentSide) { + let page = await done(readCardContentPage(reader, side)), + text = page.text, + pages = 1; + expect(page.text.length).toBeLessThanOrEqual(16384); + while (page.next) { + const next = await done(readCardContentPage(reader, side, page.next)); + expect(next.start).toBe(page.end); + expect(next.text.length).toBeLessThanOrEqual(16384); + expect(next.end - next.start).toBe(next.text.length); + if (page.text.length && next.text.length) + expect( + !( + page.text.charCodeAt(page.text.length - 1) >= 0xd800 && + page.text.charCodeAt(page.text.length - 1) <= 0xdbff && + next.text.charCodeAt(0) >= 0xdc00 && + next.text.charCodeAt(0) <= 0xdfff + ), + ).toBe(true); + text += next.text; + page = next; + pages++; + } + expect(text.length).toBe(page.total); + return { text, pages }; +} +function golden(input: CardContentInput) { + const entry = { + id: "11111111-1111-4111-8111-111111111111", + deckId: "22222222-2222-4222-8222-222222222222", + revision: 1, + tags: [], + source: null, + ...input, + } as StudyEntry; + return cardText(entry, input.key); +} +it.each([ + { + type: "basic", + key: "basic:forward", + front: "actual\n question", + back: "Actual\tanswer", + }, + { + type: "reversed", + key: "reversed:forward", + front: "Original question", + back: "Original answer", + }, + { + type: "reversed", + key: "reversed:reverse", + front: "Original question", + back: "Original answer", + }, + { + type: "cloze", + key: "cloze:one", + text: "{literal} {{{one}}} then {other} {{two}}", + blanks: [ + { id: "one", answer: "FIRST" }, + { id: "two", answer: "SECOND" }, + ], + }, + { + type: "cloze", + key: "cloze:two", + text: "{literal} {{{one}}} then {other} {{two}}", + blanks: [ + { id: "one", answer: "FIRST" }, + { id: "two", answer: "SECOND" }, + ], + }, +])( + "matches actual cardText direction without mutation: $type/$key", + async (input) => { + const before = JSON.stringify(input), + reader = await done(buildCardContent(input)); + try { + const expected = golden(input); + expect((await all(reader, "front")).text).toBe(expected.front); + expect((await all(reader, "back")).text).toBe(expected.back); + expect(JSON.stringify(input)).toBe(before); + } finally { + disposeCardContent(reader); + } + }, +); +it("makes the full legal100answer×100000 expansion reachable through bounded current pages", async () => { + const input: CardContentInput = { + type: "cloze", + key: "cloze:b50", + text: Array.from({ length: 100 }, (_, i) => "{{b" + i + "}}").join(" / "), + blanks: Array.from({ length: 100 }, (_, i) => ({ + id: "b" + i, + answer: String(i % 10).repeat(100000), + })), + }, + reader = await done(buildCardContent(input)); + try { + const expected = golden(input), + front = await all(reader, "front"), + back = await all(reader, "back"); + expect(front.text).toBe(expected.front); + expect(back.text).toBe(expected.back); + expect(back.pages).toBeGreaterThan(600); + expect(front.pages).toBeGreaterThan(600); + } finally { + disposeCardContent(reader); + } +}, 15000); +it("preserves exact Previous/Next ranges with independent front/back and reader cursor affinity", async () => { + const input: CardContentInput = { + type: "basic", + key: "basic:forward", + front: "f".repeat(50000), + back: "b".repeat(40000), + }, + a = await done(buildCardContent(input)), + b = await done(buildCardContent(input)); + try { + const first = await done(readCardContentPage(a, "front")), + second = await done(readCardContentPage(a, "front", first.next!)), + again = await done(readCardContentPage(a, "front", second.previous!)); + expect(again).toMatchObject({ + text: first.text, + start: first.start, + end: first.end, + total: first.total, + }); + expect((await done(readCardContentPage(a, "back"))).start).toBe(0); + await expect( + done(readCardContentPage(a, "back", first.next!)), + ).rejects.toThrow(); + await expect( + done(readCardContentPage(b, "front", first.next!)), + ).rejects.toThrow(); + await expect( + done(readCardContentPage(a, "front", {} as never)), + ).rejects.toThrow(); + await expect( + done(readCardContentPage(a, "front", null as never)), + ).rejects.toThrow(); + } finally { + disposeCardContent(a); + disposeCardContent(b); + } +}); +it("does not split valid surrogate pairs at pages or literal/answer joins, and keeps actual lone units", async () => { + const inputs: CardContentInput[] = [ + { + type: "basic", + key: "basic:forward", + front: "x".repeat(16383) + "😀tail", + back: "lone\uD800x\uDC00literal", + }, + { + type: "cloze", + key: "cloze:two", + text: "x".repeat(16383) + "\uD83D{{one}} then {{two}}", + blanks: [ + { id: "one", answer: "\uDE00tail" }, + { id: "two", answer: "actual" }, + ], + }, + ]; + for (const input of inputs) { + const reader = await done(buildCardContent(input)); + try { + const expected = golden(input); + expect((await all(reader, "front")).text).toBe(expected.front); + expect((await all(reader, "back")).text).toBe(expected.back); + const back = await done(readCardContentPage(reader, "back")); + if (input.type === "cloze") expect(back.end).toBe(16383); + } finally { + disposeCardContent(reader); + } + } +}); +it("captures only whitelisted scalars before suspension, without retaining caller entry/blank objects", async () => { + const input = { + type: "cloze" as const, + key: "cloze:one" as const, + text: "{{one}} then {{two}}", + blanks: [ + { id: "one", answer: "ORIGINAL" }, + { id: "two", answer: "SECOND" }, + ], + get source(): never { + throw new Error("source quote must never be read"); + }, + schedule: { nextReviewAt: "never" }, + }; + const iterator = buildCardContent(input); + expect(iterator.next().value).toMatchObject({ type: "reserve" }); + expect(iterator.next().value).toMatchObject({ type: "settle" }); + input.blanks[0]!.answer = "MUTATED"; + input.text = "MUTATED"; + const reader = await done(iterator); + try { + expect((await all(reader, "back")).text).toBe("ORIGINAL then SECOND"); + expect(input.schedule.nextReviewAt).toBe("never"); + } finally { + disposeCardContent(reader); + } +}); +it("rejects mismatched directions, corrupt markers/memberships and producer-bound overflow", async () => { + const cases: unknown[] = [ + { type: "basic", key: "reversed:reverse", front: "f", back: "b" }, + { type: "reversed", key: "basic:forward", front: "f", back: "b" }, + { + type: "basic", + key: "basic:forward", + front: "x".repeat(100001), + back: "b", + }, + ...[ + { + text: "{{one}}", + key: "cloze:missing", + blanks: [{ id: "one", answer: "A" }], + }, + { + text: "{{one}} {{one}}", + key: "cloze:one", + blanks: [{ id: "one", answer: "A" }], + }, + { + text: "{{one}} {{other}}", + key: "cloze:one", + blanks: [{ id: "one", answer: "A" }], + }, + { + text: "{{one}} {{", + key: "cloze:one", + blanks: [{ id: "one", answer: "A" }], + }, + { + text: "{{{one}}{", + key: "cloze:one", + blanks: [{ id: "one", answer: "A" }], + }, + { + text: "{{one}}", + key: "cloze:one", + blanks: [ + { id: "one", answer: "A" }, + { id: "one", answer: "B" }, + ], + }, + { + text: "{{one}}", + key: "cloze:one", + blanks: [{ id: "one", answer: " " }], + }, + { + text: "{{one}}", + key: "cloze:one", + blanks: [ + { id: "one", answer: "A" }, + { id: "two", answer: "B" }, + ], + }, + ].map((value) => ({ type: "cloze", ...value })), + ]; + for (const input of cases) + await expect( + done(buildCardContent(input as CardContentInput)), + ).rejects.toThrow(); +}); +it("resumes deliberate low-budget build/page rounds, supports cancel, and retires held page on disposal", async () => { + const input: CardContentInput = { + type: "cloze", + key: "cloze:one", + text: "x".repeat(90000) + "{{one}}", + blanks: [{ id: "one", answer: "a".repeat(100000) }], + }, + job = new SearchWork( + buildCardContent(input), + { bytes: 200000, visits: 1000 }, + async () => {}, + ); + expect((await job.round()).status).toBe("paused"); + let reader: CardContentReader | undefined; + for (let i = 0; i < 100; i++) { + const result = await job.round(); + if (result.status === "done") { + reader = result.value; + break; + } + } + expect(reader).toBeDefined(); + try { + const work = new SearchWork( + readCardContentPage(reader!, "back"), + { bytes: 40000, visits: 1000 }, + async () => {}, + ); + expect((await work.round()).status).toBe("paused"); + let text: string | undefined; + for (let i = 0; i < 100; i++) { + const result = await work.round(); + if (result.status === "done") { + text = result.value.text; + break; + } + } + expect(text).toBe("x".repeat(16384)); + const held = readCardContentPage(reader!, "back"); + expect(held.next().done).toBe(false); + disposeCardContent(reader!); + expect(held.next()).toMatchObject({ done: true }); + expect(() => readCardContentPage(reader!, "back")).toThrow(); + disposeCardContent(reader!); + const cancel = new SearchWork( + buildCardContent(input), + { bytes: 200000, visits: 1000 }, + async () => {}, + ); + expect((await cancel.round()).status).toBe("paused"); + cancel.close(); + await expect(cancel.round()).rejects.toMatchObject({ name: "AbortError" }); + expect(() => disposeCardContent({} as never)).toThrow(); + } finally { + disposeCardContent(reader!); + } +}); + +it("aborts held generation work before any reader/page publication", async () => { + const input: CardContentInput = { + type: "basic", + key: "basic:forward", + front: "f".repeat(100000), + back: "b".repeat(100000), + }, + signal = new AbortController(); + const build = new SearchWork(buildCardContent(input), undefined, async () => + signal.abort(), + ); + await expect(build.round(signal.signal)).rejects.toMatchObject({ + name: "AbortError", + }); + const reader = await done(buildCardContent(input)); + let current = true; + try { + const page = new SearchWork( + readCardContentPage(reader, "back"), + undefined, + async () => { + current = false; + }, + () => { + if (!current) throw new Error("retired generation"); + }, + ); + await expect(page.round()).rejects.toThrow("retired generation"); + expect((await done(readCardContentPage(reader, "back"))).text.length).toBe( + 16384, + ); + } finally { + disposeCardContent(reader); + } +}); +it("renders every actual cloze direction with the selected answer hidden only on front", async () => { + const text = "{{a}} then {{b}} then {{c}}", + blanks = [ + { id: "a", answer: "ONE" }, + { id: "b", answer: "TWO" }, + { id: "c", answer: "THREE" }, + ]; + for (const blank of blanks) { + const input: CardContentInput = { + type: "cloze", + key: `cloze:${blank.id}`, + text, + blanks, + }, + reader = await done(buildCardContent(input)); + try { + const expected = golden(input); + expect((await all(reader, "front")).text).toBe(expected.front); + expect((await all(reader, "back")).text).toBe(expected.back); + } finally { + disposeCardContent(reader); + } + } +}); diff --git a/web/tests/data/note-link-schema-source.test.ts b/web/tests/data/note-link-schema-source.test.ts new file mode 100644 index 0000000..9cb633f --- /dev/null +++ b/web/tests/data/note-link-schema-source.test.ts @@ -0,0 +1,164 @@ +import { expect, it, vi } from "vitest"; +import { getSchema } from "@tiptap/core"; +import { Transform } from "@tiptap/pm/transform"; +import type { Node as PMNode } from "@tiptap/pm/model"; +import type { NoteDocument } from "../../src/domain/contracts"; +import { + createNoteExtensions, + createNoteSchema, +} from "../../src/editor/schema"; +import { + captureSourceSelection, + canonicalSourceDocument, + mapSourcePassage, + verifySourceEdit, +} from "../../src/study/sourceCodec"; + +const target = { + kind: "note", + accountId: "11111111-1111-4111-8111-111111111111", + libraryEpoch: "22222222-2222-4222-8222-222222222222", + noteId: "opaque target", +} as const; +const json = (node: PMNode) => node.toJSON() as NoteDocument; +function fixture() { + const schema = createNoteSchema(); + const chip = () => + schema.nodes.noteLink!.create({ version: 1, target: { ...target } }); + const paragraph = (content: PMNode[]) => + schema.nodes.doc!.create( + null, + schema.nodes.paragraph!.create(null, content), + ); + return { schema, chip, paragraph }; +} + +it("uses the same identity-atom schema with or without a browser NodeView", () => { + const renderer = vi.fn(() => { + throw new Error("NodeView must not run in headless schema construction"); + }); + const base = createNoteSchema(); + const viewed = getSchema( + createNoteExtensions({ noteLinkNodeView: renderer }), + ); + const document = { + type: "doc", + content: [ + { + type: "paragraph", + content: [{ type: "noteLink", attrs: { version: 1, target } }], + }, + ], + }; + for (const schema of [base, viewed]) { + const node = schema.nodeFromJSON(document); + node.check(); + expect(node.toJSON()).toEqual(document); + const chip = node.firstChild!.firstChild!; + expect(chip.isInline).toBe(true); + expect(chip.isAtom).toBe(true); + expect(chip.nodeSize).toBe(1); + expect(chip.type.spec.parseDOM).toEqual([]); + const html = chip.type.spec.toDOM!(chip); + expect(JSON.stringify(html)).not.toContain("opaque target"); + expect(JSON.stringify(html)).not.toContain(target.accountId); + expect(JSON.stringify(html)).toContain("Note reference"); + } + expect(renderer).not.toHaveBeenCalled(); +}); + +it("requires a typed target and rejects malformed intermediate target attributes", () => { + const { schema, chip, paragraph } = fixture(); + expect(() => schema.nodes.noteLink!.createChecked({ version: 1 })).toThrow(); + const before = paragraph([chip(), schema.text(" chosen passage")]); + expect(() => + new Transform(before).setNodeAttribute(1, "target", { + ...target, + kind: "task", + }), + ).toThrow(); + expect(() => + verifySourceEdit(json(before), json(before), [ + { + stepType: "attr", + pos: 1, + attr: "target", + value: { ...target, kind: "task" }, + }, + { stepType: "attr", pos: 1, attr: "target", value: { ...target } }, + ]), + ).toThrow(); +}); + +it("bounds exact Cloze context at adjacent real chips without replacing literal text", () => { + const { schema, chip, paragraph } = fixture(); + const before = "middle \ufffc "; + const doc = paragraph([ + schema.text("unrelated left"), + chip(), + schema.text(`${before}chosen tail`), + chip(), + schema.text("unrelated right"), + ]); + let start = 0; + doc.descendants((node, pos) => { + if (node.isText && node.text!.startsWith(before)) + start = pos + before.length; + }); + const selection = captureSourceSelection( + json(doc), + start, + start + "chosen".length, + ); + expect(selection.passage.quote).toBe("chosen"); + expect(selection.context).toEqual({ + text: `${before}chosen tail`, + selectionFrom: before.length, + selectionTo: before.length + "chosen".length, + }); + expect(() => + captureSourceSelection(json(doc), 1, doc.content.size - 1), + ).toThrow(); + expect(canonicalSourceDocument(json(doc)).document).toEqual(json(doc)); +}); + +it("maps an outside chip insertion and stales an insertion inside the selected passage", () => { + const { schema, chip, paragraph } = fixture(); + const doc = paragraph([schema.text("alpha selected omega")]); + const passage = captureSourceSelection(json(doc), 7, 15).passage; + for (const [position, outside] of [ + [1, true], + [10, false], + ] as const) { + const edited = new Transform(doc).insert(position, chip()); + const proof = verifySourceEdit( + json(doc), + json(edited.doc), + edited.steps.map((step) => step.toJSON()), + ); + expect(mapSourcePassage(proof, passage)).toEqual( + outside + ? { status: "attached", passage: { ...passage, from: 8, to: 16 } } + : { status: "stale", reason: "passage-edited" }, + ); + } +}); + +it("retains an unrelated passage when an outside chip is retargeted", () => { + const { schema, chip, paragraph } = fixture(); + const doc = paragraph([chip(), schema.text(" selected")]); + const passage = captureSourceSelection(json(doc), 3, 11).passage; + const edited = new Transform(doc).setNodeAttribute(1, "target", { + ...target, + noteId: "another identity", + }); + const proof = verifySourceEdit( + json(doc), + json(edited.doc), + edited.steps.map((step) => step.toJSON()), + ); + expect(mapSourcePassage(proof, passage)).toEqual({ + status: "attached", + passage, + }); +}); diff --git a/web/tests/data/search-identity.test.ts b/web/tests/data/search-identity.test.ts new file mode 100644 index 0000000..3f1cce0 --- /dev/null +++ b/web/tests/data/search-identity.test.ts @@ -0,0 +1,54 @@ +import { describe, expect, it } from "vitest"; +import { SearchIdentityMap } from "../../src/search/identity"; +import { SearchWork, type WorkEvent } from "../../src/search/work"; + +async function complete(generator: Generator): Promise { + const work = new SearchWork(generator, undefined, async () => {}); + let result = await work.round(); + while (result.status === "paused") result = await work.round(); + return result.value; +} +describe("budgeted opaque search identities", () => { + it("charges every collision candidate even when its length differs", async () => { + const map = new SearchIdentityMap(); + await complete(map.set("costarring", 1)); + await complete(map.set("liquid", 2)); + const work = new SearchWork(map.get("liquid"), { bytes: 100000, visits: 3 }, async () => {}); + expect(await work.round()).toMatchObject({ status: "paused", used: { visits: 3 } }); + expect(await work.round()).toMatchObject({ status: "done", value: 2 }); + }); + it("cannot return a retained value after release at the final comparison settlement", async () => { + const map = new SearchIdentityMap(); + await complete(map.set("foo", 42)); + const read = map.get("foo"); + let event = read.next(); + while (!event.done && !(event.value.type === "settle" && event.value.bytes === 18)) event = read.next(); + expect(event.done).toBe(false); + map.clear(); + expect(() => read.next()).toThrow(/released/); + }); + it("uses exact original identity after hash collision, including lone UTF16 and punctuation", async () => { + const map = new SearchIdentityMap(); + // These distinct UTF16 strings collide under FNV1a32. A hash is never an ID. + await complete(map.set("costarring", 1)); + await complete(map.set("liquid", 2)); + await complete(map.set("\ud800/[x]", 3)); + expect(await complete(map.get("costarring"))).toBe(1); + expect(await complete(map.get("liquid"))).toBe(2); + expect(await complete(map.get("\ud800/[x]"))).toBe(3); + expect(await complete(map.get("\ufffd/[x]"))).toBeUndefined(); + }); + it("pauses inside an enormous identity rather than hashing it before admission", async () => { + const map = new SearchIdentityMap(); + const key = "a".repeat(50000); + const work = new SearchWork(map.set(key, 1), { bytes: 16000, visits: 10 }, async () => {}); + expect(await work.round()).toMatchObject({ status: "paused" }); + // No partially admitted value is visible after cancelling the unfinished set. + work.close(); + expect(await complete(map.get(key))).toBeUndefined(); + await complete(map.set(key, 2)); + expect(await complete(map.get(key))).toBe(2); + map.clear(); + expect(await complete(map.get(key))).toBeUndefined(); + }); +}); diff --git a/web/tests/data/search-work.test.ts b/web/tests/data/search-work.test.ts new file mode 100644 index 0000000..da3479c --- /dev/null +++ b/web/tests/data/search-work.test.ts @@ -0,0 +1,64 @@ +import { describe, expect, it } from "vitest"; +import { SearchWork, type WorkEvent } from "../../src/search/work"; + +describe("resumable search work accounting", () => { + it("reserves before running, refunds unused cost and resumes the exact iterator", async () => { + const executed: number[] = []; + function* source(): Generator { + for (let i = 0; i < 3; i++) { + yield { type: "reserve", bytes: 8, visits: 1, scalars: 2 }; + executed.push(i); + yield { type: "settle", bytes: 4, visits: 1, scalars: 2 }; + } + return 42; + } + const work = new SearchWork(source(), { bytes: 12, visits: 5 }); + expect(await work.round()).toMatchObject({ status: "paused", used: { bytes: 8, visits: 2 } }); + expect(executed).toEqual([0, 1]); + expect(await work.round()).toMatchObject({ status: "done", value: 42, used: { bytes: 4, visits: 1 } }); + expect(executed).toEqual([0, 1, 2]); + expect(await work.round()).toMatchObject({ status: "done", value: 42 }); + }); + it("yields at the scalar bound, rejects overlapping Continue, and fences abort after await", async () => { + let release!: () => void; + const controller = new AbortController(); + let afterYield = false; + function* source(): Generator { + yield { type: "reserve", bytes: 4096, visits: 1, scalars: 4096 }; + yield { type: "settle", bytes: 4096, visits: 1, scalars: 4096 }; + afterYield = true; + } + const work = new SearchWork(source(), undefined, () => new Promise(resolve => { release = resolve; })); + const pending = work.round(controller.signal); + await expect(work.round()).rejects.toThrow(/already running/i); + controller.abort(); release(); + await expect(pending).rejects.toMatchObject({ name: "AbortError" }); + expect(afterYield).toBe(false); + }); + it("treats scalars separately from the structural budget and closes released iterators", async () => { + let released = false; + function* source(): Generator { + try { + for (let i = 0; i < 4; i++) { + yield { type: "reserve", bytes: 4096, visits: 1, scalars: 4096 }; + yield { type: "settle", bytes: 4096, visits: 1, scalars: 4096 }; + } + return "done"; + } finally { released = true; } + } + const work = new SearchWork(source(), { bytes: 16384, visits: 4 }, async () => {}); + expect(await work.round()).toMatchObject({ status: "done", value: "done" }); + expect(released).toBe(true); + }); + it("rejects under-accounting and an impossible indivisible reservation", async () => { + function* bad(): Generator { + yield { type: "reserve", bytes: 1, visits: 1, scalars: 1 }; + yield { type: "settle", bytes: 2, visits: 1, scalars: 1 }; + } + await expect(new SearchWork(bad()).round()).rejects.toThrow(/reservation/i); + function* oversized(): Generator { + yield { type: "reserve", bytes: 3, visits: 0, scalars: 1 }; + } + await expect(new SearchWork(oversized(), { bytes: 2, visits: 1 }).round()).rejects.toThrow(/indivisible/i); + }); +}); diff --git a/web/tests/data/source-codec.test.ts b/web/tests/data/source-codec.test.ts new file mode 100644 index 0000000..427ddce --- /dev/null +++ b/web/tests/data/source-codec.test.ts @@ -0,0 +1,432 @@ +// @vitest-environment node +import { describe, expect, it } from "vitest"; +import { + AddMarkStep, + RemoveMarkStep, + AddNodeMarkStep, + RemoveNodeMarkStep, + AttrStep, + DocAttrStep, + ReplaceAroundStep, + Step, + StepMap, + StepResult, + Transform, + findWrapping, +} from "@tiptap/pm/transform"; +import { Slice, Fragment } from "@tiptap/pm/model"; +import { createNoteSchema } from "../../src/editor/schema"; +import { + canonicalSourceDocument, + captureSourceSelection, + fingerprintSourceDocument, + verifySourceEdit, + mapSourcePassage, + sourceEditCost, +} from "../../src/study/sourceCodec"; +import type { NoteDocument } from "../../src/domain/contracts"; +const document = (text = "alpha target omega"): NoteDocument => ({ + type: "doc", + content: [ + { type: "paragraph", content: text ? [{ type: "text", text }] : [] }, + ], +}); +const passage = { from: 7, to: 13, quote: "target" }; +const schema = () => createNoteSchema(); +describe("writer-verified source passage codec", () => { + it("uses the actual headless editor schema and stable canonical JSON", () => { + const first = canonicalSourceDocument(document()); + expect(first).toEqual( + canonicalSourceDocument(JSON.parse(first.canonicalJSON)), + ); + expect(schema().nodeFromJSON(first.document).textContent).toBe( + "alpha target omega", + ); + expect(() => + canonicalSourceDocument({ + type: "doc", + content: [{ type: "image", attrs: { src: "https://example.test" } }], + }), + ).toThrow(); + }); + it("captures exact multiblock context and prevents split-surrogate or atom selections", () => { + const value: NoteDocument = { + type: "doc", + content: [ + { type: "paragraph", content: [{ type: "text", text: "ab" }] }, + { type: "paragraph", content: [{ type: "text", text: "cd" }] }, + ], + }; + const selected = captureSourceSelection(value, 2, 6); + expect(selected.passage.quote).toBe("b\nc"); + expect( + selected.context.text.slice( + selected.context.selectionFrom, + selected.context.selectionTo, + ), + ).toBe(selected.passage.quote); + expect(() => captureSourceSelection(document("a📝b"), 2, 3)).toThrow(); + expect(captureSourceSelection(document("a📝b"), 2, 4).passage.quote).toBe( + "📝", + ); + expect(() => + captureSourceSelection( + { + type: "doc", + content: [ + { type: "paragraph", content: [{ type: "text", text: "ab" }] }, + { type: "horizontalRule" }, + { type: "paragraph", content: [{ type: "text", text: "cd" }] }, + ], + }, + 1, + 8, + ), + ).toThrow(); + }); + it("maps surrounding edits and supported marks without searching for duplicate quotes", () => { + const s = schema(), + base = s.nodeFromJSON(document()); + const tx = new Transform(base) + .insert(1, s.text("prefix ")) + .addMark(14, 20, s.marks.bold!.create()); + const proof = verifySourceEdit( + document(), + tx.doc.toJSON(), + tx.steps.map((step) => step.toJSON()), + ); + expect(mapSourcePassage(proof, passage)).toEqual({ + status: "attached", + passage: { from: 14, to: 20, quote: "target" }, + }); + expect(() => + mapSourcePassage(proof, { ...passage, quote: "invented" }), + ).toThrow(); + }); + it("stales replacement with identical bytes, insertion inside and delete/cut-paste", () => { + const s = schema(), + base = s.nodeFromJSON(document()); + for (const tx of [ + new Transform(base).replaceWith(7, 13, s.text("target")), + new Transform(base).insert(9, s.text("x")), + new Transform(base).delete(7, 13).insert(1, s.text("target")), + ]) { + expect( + mapSourcePassage( + verifySourceEdit( + document(), + tx.doc.toJSON(), + tx.steps.map((step) => step.toJSON()), + ), + passage, + ), + ).toEqual({ status: "stale", reason: "passage-edited" }); + } + }); + it("preserves retained-gap structural formatting and rejects mismatched final documents", () => { + const s = schema(), + base = s.nodeFromJSON(document()), + range = base.resolve(0).blockRange(base.resolve(base.content.size))!; + const tx = new Transform(base).wrap( + range, + findWrapping(range, s.nodes.blockquote!)!, + ); + expect(tx.steps[0]!.toJSON().stepType).toBe("replaceAround"); + expect( + mapSourcePassage( + verifySourceEdit( + document(), + tx.doc.toJSON(), + tx.steps.map((step) => step.toJSON()), + ), + passage, + ), + ).toEqual({ + status: "attached", + passage: { from: 8, to: 14, quote: "target" }, + }); + expect(() => + verifySourceEdit(document(), document("different"), []), + ).toThrow(); + }); + it("excludes insertions at both selection boundaries", () => { + const s = schema(), + base = s.nodeFromJSON(document()); + const tx = new Transform(base) + .insert(13, s.text(" after")) + .insert(7, s.text("before ")); + expect( + mapSourcePassage( + verifySourceEdit( + document(), + tx.doc.toJSON(), + tx.steps.map((step) => step.toJSON()), + ), + passage, + ), + ).toEqual({ + status: "attached", + passage: { from: 14, to: 20, quote: "target" }, + }); + }); + it("snapshots its fingerprint before asynchronous hashing", async () => { + const value = document(); + const pending = fingerprintSourceDocument(value); + value.content[0]!.content![0]!.text = "later mutation"; + const result = await pending; + expect(result.digest).toMatch(/^[0-9a-f]{64}$/); + expect(result.document).toEqual(document()); + expect(result.canonicalJSON).not.toContain("later mutation"); + expect(result.digest).toBe( + (await fingerprintSourceDocument(document())).digest, + ); + }); + it("rejects unknown steps even when their map is empty and cannot forge verified handles", () => { + class UnknownStep extends Step { + apply(doc: ReturnType["nodeFromJSON"]>) { + return StepResult.ok(doc); + } + getMap() { + return StepMap.empty; + } + invert() { + return this; + } + map() { + return this; + } + toJSON() { + return { stepType: "source-codec-test-unknown" }; + } + static fromJSON() { + return new UnknownStep(); + } + } + Step.jsonID("source-codec-test-unknown", UnknownStep); + expect(() => + verifySourceEdit(document(), document(), [new UnknownStep().toJSON()]), + ).toThrow(); + expect(() => + mapSourcePassage({} as ReturnType, passage), + ).toThrow(); + }); + it("bounds step payload and refuses an unsupported attribute instead of dropping it", () => { + expect(() => + verifySourceEdit( + document(), + document(), + Array.from({ length: 1001 }, () => ({ stepType: "addMark" })), + ), + ).toThrow(); + expect(() => + canonicalSourceDocument({ + type: "doc", + content: [ + { + type: "paragraph", + attrs: { unknown: "do not drop" }, + content: [{ type: "text", text: "x" }], + }, + ], + }), + ).toThrow(); + }); + it("replays the actual first insertion from a stored empty document without inventing a base paragraph", () => { + const empty: NoteDocument = { type: "doc", content: [] }, + s = schema(); + const tx = new Transform(s.nodeFromJSON(empty)).insert( + 0, + s.nodes.paragraph!.create(null, s.text("target")), + ); + expect(canonicalSourceDocument(empty).document).toEqual(empty); + expect(() => + verifySourceEdit( + empty, + tx.doc.toJSON(), + tx.steps.map((step) => step.toJSON()), + ), + ).not.toThrow(); + }); + it("round-trips the installed restricted markSet, rejects unknown or nested children", () => { + const s = schema(), + add = new AddMarkStep(7, 13, s.marks.bold!.create()).toJSON(), + remove = new RemoveMarkStep(7, 13, s.marks.bold!.create()).toJSON(); + const batch = { stepType: "markSet", steps: [add, remove] }; + expect( + Step.fromJSON(s, batch).apply(s.nodeFromJSON(document())).doc!.toJSON(), + ).toEqual(document()); + expect( + mapSourcePassage( + verifySourceEdit(document(), document(), [batch]), + passage, + ), + ).toEqual({ status: "attached", passage }); + expect(() => + verifySourceEdit(document(), document(), [ + { stepType: "markSet", steps: [batch] }, + ]), + ).toThrow(); + expect(() => + verifySourceEdit(document(), document(), [ + { + stepType: "markSet", + steps: [{ stepType: "attr", pos: 0, attr: "unknown", value: 1 }], + }, + ]), + ).toThrow(); + }); + it("supports admitted node marks and attributes but rejects unsupported doc attributes", () => { + const s = schema(), + value: NoteDocument = { + type: "doc", + content: [ + { + type: "heading", + attrs: { level: 2 }, + content: [{ type: "text", text: "alpha target omega" }], + }, + ], + }; + const tx = new Transform(s.nodeFromJSON(value)).step( + new AttrStep(0, "level", 3), + ); + expect( + mapSourcePassage( + verifySourceEdit( + value, + tx.doc.toJSON(), + tx.steps.map((step) => step.toJSON()), + ), + passage, + ), + ).toEqual({ status: "attached", passage }); + expect(() => + verifySourceEdit(value, value, [ + new DocAttrStep("unknown", true).toJSON(), + ]), + ).toThrow(); + const withBreak: NoteDocument = { + type: "doc", + content: [ + { + type: "paragraph", + content: [ + { type: "text", text: "a" }, + { type: "hardBreak" }, + { type: "text", text: "b" }, + ], + }, + ], + }; + const marked = new Transform(s.nodeFromJSON(withBreak)) + .step(new AddNodeMarkStep(2, s.marks.bold!.create())) + .step(new RemoveNodeMarkStep(2, s.marks.bold!.create())); + expect( + mapSourcePassage( + verifySourceEdit( + withBreak, + marked.doc.toJSON(), + marked.steps.map((step) => step.toJSON()), + ), + { from: 1, to: 4, quote: "a\nb" }, + ), + ).toEqual({ + status: "attached", + passage: { from: 1, to: 4, quote: "a\nb" }, + }); + }); + it("stales the removed side of a valid replaceAround while preserving its exact retained gap", () => { + const s = schema(), + base = s.nodeFromJSON(document()), + replacement = new Slice( + Fragment.from(s.nodes.paragraph!.create(null, s.text("replacement "))), + 0, + 0, + ); + const tx = new Transform(base).step( + new ReplaceAroundStep(0, base.content.size, 7, 13, replacement, 13), + ); + const proof = verifySourceEdit( + document(), + tx.doc.toJSON(), + tx.steps.map((step) => step.toJSON()), + ); + expect(mapSourcePassage(proof, passage)).toEqual({ + status: "attached", + passage: { from: 13, to: 19, quote: "target" }, + }); + expect(mapSourcePassage(proof, { from: 1, to: 6, quote: "alpha" })).toEqual( + { status: "stale", reason: "passage-edited" }, + ); + }); + it("preserves literal replacement characters in text but rejects malformed or ignored proof fields", () => { + expect( + captureSourceSelection(document("a\ufffcb"), 1, 4).passage.quote, + ).toBe("a\ufffcb"); + expect(() => + verifySourceEdit(document(), document(), [ + { + stepType: "addMark", + from: 0, + to: 0, + mark: { type: "link", attrs: { href: "javascript:bad" } }, + }, + ]), + ).toThrow(); + expect(() => + verifySourceEdit(document(), document(), [ + { stepType: "attr", pos: 0, attr: "level" }, + ]), + ).toThrow(); + expect(() => + verifySourceEdit(document(), document(), [ + { + stepType: "replace", + from: 1, + to: 1, + slice: { + content: [{ type: "text", text: "", attrs: { unknown: true } }], + }, + }, + ]), + ).toThrow(); + expect(() => + mapSourcePassage( + verifySourceEdit(document(), document(), []), + null as unknown as typeof passage, + ), + ).toThrow(/could not be verified/); + }); + it("charges zero-step validation and serialized attributes in batch mapping metrics", () => { + const value: NoteDocument = { + type: "doc", + content: [ + { + type: "paragraph", + content: [ + { + type: "text", + text: "x", + marks: [ + { + type: "link", + attrs: { href: `https://example.test/${"a".repeat(20000)}` }, + }, + ], + }, + ], + }, + ], + }; + const edit = verifySourceEdit(value, value, []), + cost = sourceEditCost(edit); + expect(cost.stepCount).toBe(0); + expect(cost.maxDocumentUnits).toBeGreaterThan(20000); + expect(cost.replayUnits).toBeGreaterThan(40000); + expect(() => + sourceEditCost({} as ReturnType), + ).toThrow(); + expect(sourceEditCost(edit)).toEqual(cost); + expect(Object.isFrozen(cost)).toBe(true); + }); +}); diff --git a/web/tests/study/csv.test.ts b/web/tests/study/csv.test.ts new file mode 100644 index 0000000..566f26b --- /dev/null +++ b/web/tests/study/csv.test.ts @@ -0,0 +1,22 @@ +import { expect, it } from "vitest"; +import { parseStudyCsv, importStudyCsv, exportStudyCsv } from "../../src/study/csv"; +import { createDeck } from "../../src/study/model"; +import { emptyStudy } from "../../src/collections/validation"; +it("imports quoted multiline Unicode cards and exports without schedules", () => { + const state = createDeck(emptyStudy(), "Languages", new Date()); + const source = 'Front,Back,Tags\r\n"Question, one","Answer ""two""\n日本語","[""tag with space""]"\r\n'; + const rows = parseStudyCsv(source); + expect(rows).toEqual([{ front: "Question, one", back: 'Answer "two"\n日本語', tags: ["tag with space"] }]); + const imported = importStudyCsv(state, state.decks[0]!.id, source, new Date()); + expect(imported.cards).toHaveLength(1); + const exported = exportStudyCsv(imported, state.decks[0]!.id); + expect(parseStudyCsv(exported)).toEqual(rows); + expect(exported).not.toContain("schedule"); + expect(state.cards).toHaveLength(0); +}); +it.each(['Front,Back\n"unclosed,answer', 'Wrong,Back\nx,y', 'Front,Back\nQ,A\nMissing,', 'Front,Back,Tags\nQ,A,not-json', 'Front,Back\n"Q"bad,A'])('rejects invalid CSV without mutating input state: %s', source => { + const state = createDeck(emptyStudy(), "Deck", new Date()); + const before = structuredClone(state); + expect(() => importStudyCsv(state, state.decks[0]!.id, source, new Date())).toThrow(); + expect(state).toEqual(before); +}); diff --git a/web/tests/study/model.test.ts b/web/tests/study/model.test.ts new file mode 100644 index 0000000..f224ece --- /dev/null +++ b/web/tests/study/model.test.ts @@ -0,0 +1,56 @@ +import { describe, expect, it } from "vitest"; +import type { StudyEntry, StudyState } from "../../src/collections/contracts"; +import { createDeck, saveEntry, gradeCard, undoReview, cardText, reviewQueue, progress } from "../../src/study/model"; +const at = new Date("2026-11-11T12:00:00Z"); +const empty = (): StudyState => ({ decks: [], entries: [], cards: [], events: [], preferences: { algorithm: "orbyn-interval-v1", timezone: "UTC", dayStartHour: 4, newPerDay: 10, reviewsPerDay: 100, showNextReview: true, showStreak: true } }); +function seed() { + const state = createDeck(empty(), "Real deck", at); + const entry: StudyEntry = { id: crypto.randomUUID(), deckId: state.decks[0]!.id, revision: 1, tags: [], source: null, type: "reversed", front: "Question", back: "Answer" }; + return { state: saveEntry(state, entry, at), entry }; +} +describe("real Study state transitions", () => { + it("creates independently scheduled reverse cards and reconciles edits by namespaced identity", () => { + const { state, entry } = seed(); + expect(state.cards).toHaveLength(2); + expect(cardText(entry, "reversed:reverse")).toEqual({ front: "Answer", back: "Question" }); + const reviewed = gradeCard(state, state.cards[0]!.id, "good", crypto.randomUUID(), crypto.randomUUID(), at); + expect(reviewed.cards[1]!.schedule.state).toBe("new"); + const edited = saveEntry(reviewed, { ...entry, revision: 2, front: "Changed" }, at); + expect(edited.cards[0]!.id).toBe(state.cards[0]!.id); + expect(edited.cards[0]!.schedule).toEqual(reviewed.cards[0]!.schedule); + const changedType = saveEntry(edited, { ...entry, revision: 3, type: "cloze", text: "{{forward}}", blanks: [{ id: "forward", answer: "Cloze answer" }] }, at); + expect(changedType.cards).toHaveLength(1); + expect(changedType.cards[0]!.id).not.toBe(state.cards[0]!.id); + expect(changedType.cards[0]!.schedule.state).toBe("new"); + }); + it("deduplicates retry by operation ID and safely undoes only latest unchanged grade", () => { + const { state } = seed(); + const card = state.cards[0]!; + const session = crypto.randomUUID(), op = crypto.randomUUID(); + const next = gradeCard(state, card.id, "easy", session, op, at); + expect(gradeCard(next, card.id, "easy", session, op, at)).toEqual(next); + expect(() => gradeCard(next, card.id, "again", session, op, at)).toThrow("operation"); + const undone = undoReview(next, op, at); + expect(undone.cards[0]!.schedule).toEqual(card.schedule); + expect(undone.events[0]!.undoneAt).toBe(at.toISOString()); + const twice = gradeCard(next, card.id, "good", session, crypto.randomUUID(), new Date(+at + 1000)); + expect(() => undoReview(twice, op, at)).toThrow("undo"); + }); + it("renders each cloze with context and rejects missing or duplicate placeholders", () => { + const { state, entry } = seed(); + const cloze: StudyEntry = { id: crypto.randomUUID(), deckId: entry.deckId, revision: 1, tags: [], source: null, type: "cloze", text: "{{city}} is in {{country}}", blanks: [{ id: "city", answer: "Paris" }, { id: "country", answer: "France" }] }; + expect(cardText(cloze, "cloze:city")).toEqual({ front: "[...] is in France", back: "Paris is in France" }); + expect(saveEntry(state, cloze, at).cards).toHaveLength(4); + expect(() => saveEntry(state, { ...cloze, text: "{{city}} {{city}}" }, at)).toThrow("cloze"); + }); + it("uses committed events for progress and enforces real due/new caps", () => { + const { state } = seed(); + state.preferences.newPerDay = 1; + expect(reviewQueue(state, at)).toHaveLength(1); + expect(progress(state, at).remembered).toBeNull(); + const next = gradeCard(state, state.cards[0]!.id, "good", crypto.randomUUID(), crypto.randomUUID(), at); + expect(reviewQueue(next, at)).toHaveLength(0); + expect(progress(next, at)).toMatchObject({ reviewedThisWeek: 1, reviewedLast30Days: 1, remembered: 1, streak: 1, cardCount: 2 }); + expect(progress(undoReview(next, next.events[0]!.id, at), at).remembered).toBeNull(); + }); +}); diff --git a/web/tests/study/scheduler.test.ts b/web/tests/study/scheduler.test.ts new file mode 100644 index 0000000..48aff4f --- /dev/null +++ b/web/tests/study/scheduler.test.ts @@ -0,0 +1,37 @@ +import { describe, expect, it } from "vitest"; +import { initialSchedule, nextSchedule, studyDay } from "../../src/study/scheduler"; + +const now = new Date("2026-11-11T12:00:00Z"); +describe("versioned Study scheduling", () => { + it("uses the same deterministic grade intervals for preview and commit", () => { + const initial = initialSchedule(now); + expect(initial.state).toBe("new"); + for (const [grade, minutes] of [["again", 1], ["hard", 10], ["good", 1440], ["easy", 5760]] as const) { + const result = nextSchedule(initial, grade, now); + expect(Date.parse(result.due) - +now).toBe(minutes * 60_000); + expect(result).toEqual(nextSchedule(initial, grade, now)); + expect(initial).toEqual(initialSchedule(now)); + } + expect(nextSchedule(initial, "hard", now).state).toBe("learning"); + }); + it("repeated lapses reset intervals and never create invalid ease or past due times", () => { + let schedule = nextSchedule(initialSchedule(now), "easy", now); + for (let i = 0; i < 30; i++) schedule = nextSchedule(schedule, "again", now); + expect(schedule.lapses).toBe(30); + expect(schedule.intervalDays).toBe(0); + expect(schedule.state).toBe("learning"); + expect(schedule.ease).toBeGreaterThanOrEqual(1.3); + expect(Date.parse(schedule.due)).toBeGreaterThan(+now); + }); + it("groups by local four AM through midnight and daylight-saving transitions", () => { + expect(studyDay(new Date("2026-10-03T16:30:00Z"), "Australia/Sydney", 4)).toBe("2026-10-03"); + expect(studyDay(new Date("2026-10-03T17:00:00Z"), "Australia/Sydney", 4)).toBe("2026-10-04"); + expect(studyDay(new Date("2026-11-11T02:59:00Z"), "UTC", 4)).toBe("2026-11-10"); + expect(studyDay(new Date("2026-11-11T04:00:00Z"), "UTC", 4)).toBe("2026-11-11"); + }); + it("rejects invalid inputs instead of storing NaN or unsupported schedules", () => { + expect(() => nextSchedule(initialSchedule(now), "wrong" as never, now)).toThrow(); + expect(() => initialSchedule(new Date("invalid"))).toThrow(); + expect(() => studyDay(now, "UTC", 24)).toThrow(); + }); +});