feat(search): add Unicode matching and local search services

web/public/licenses/Unicode-3.0.txt

@@ -0,0 +1,39 @@
+UNICODE LICENSE V3
+
+COPYRIGHT AND PERMISSION NOTICE
+
+Copyright © 1991-2026 Unicode, Inc.
+
+NOTICE TO USER: Carefully read the following legal agreement. BY
+DOWNLOADING, INSTALLING, COPYING OR OTHERWISE USING DATA FILES, AND/OR
+SOFTWARE, YOU UNEQUIVOCALLY ACCEPT, AND AGREE TO BE BOUND BY, ALL OF THE
+TERMS AND CONDITIONS OF THIS AGREEMENT. IF YOU DO NOT AGREE, DO NOT
+DOWNLOAD, INSTALL, COPY, DISTRIBUTE OR USE THE DATA FILES OR SOFTWARE.
+
+Permission is hereby granted, free of charge, to any person obtaining a
+copy of data files and any associated documentation (the "Data Files") or
+software and any associated documentation (the "Software") to deal in the
+Data Files or Software without restriction, including without limitation
+the rights to use, copy, modify, merge, publish, distribute, and/or sell
+copies of the Data Files or Software, and to permit persons to whom the
+Data Files or Software are furnished to do so, provided that either (a)
+this copyright and permission notice appear with all copies of the Data
+Files or Software, or (b) this copyright and permission notice appear in
+associated Documentation.
+
+THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
+KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF
+THIRD PARTY RIGHTS.
+
+IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS NOTICE
+BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES,
+OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS,
+WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION,
+ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THE DATA
+FILES OR SOFTWARE.
+
+Except as contained in this notice, the name of a copyright holder shall
+not be used in advertising or otherwise to promote the sale, use or other
+dealings in these Data Files or Software without prior written
+authorization of the copyright holder.

web/scripts/generate-search-folding.mjs

@@ -0,0 +1,61 @@
+// Rebuild the reviewed Unicode data adaptation from explicit local inputs.
+// This script never downloads or executes upstream content.
+import { readFile, writeFile, mkdir } from "node:fs/promises";
+import { createHash } from "node:crypto";
+
+const dataSha = "ff8d8fefbf123574205085d6714c36149eb946d717a0c585c27f0f4ef58c4183";
+const licenseSha = "e7a93b009565cfce55919a381437ac4db883e9da2126fa28b91d12732bc53d96";
+const inputs = process.argv.slice(2);
+if (inputs.length !== 2) throw new Error("Pass the reviewed CaseFolding.txt and LICENSE.txt paths.");
+const [data, license] = await Promise.all(inputs.map(path => readFile(path)));
+const digest = bytes => createHash("sha256").update(bytes).digest("hex");
+if (digest(data) !== dataSha || digest(license) !== licenseSha) throw new Error("Unicode source differs from the reviewed pin.");
+const rows = new Map();
+for (const line of data.toString("utf8").split(/\r?\n/)) {
+  const body = line.split("#")[0].trim();
+  if (!body) continue;
+  const fields = body.split(";").map(value => value.trim());
+  if (!/^[0-9A-F]{4,6}$/.test(fields[0]) || !["C", "F", "S", "T"].includes(fields[1])) throw new Error("Unexpected Unicode record.");
+  if (fields[1] !== "C" && fields[1] !== "F") continue;
+  const point = Number.parseInt(fields[0], 16);
+  const targets = fields[2].split(" ").map(value => {
+    if (!/^[0-9A-F]{4,6}$/.test(value)) throw new Error("Unexpected mapping.");
+    const result = Number.parseInt(value, 16);
+    if (result > 0x10ffff || (result >= 0xd800 && result <= 0xdfff)) throw new Error("Invalid mapping scalar.");
+    return result;
+  });
+  if (point > 0x10ffff || rows.has(point) || targets.length > 3) throw new Error("Unexpected mapping bounds.");
+  rows.set(point, String.fromCodePoint(...targets));
+}
+let maxUnits = 2;
+for (const value of rows.values()) maxUnits = Math.max(maxUnits, value.length);
+const table = [...rows].sort((a, b) => a[0] - b[0]).map(([point, value]) => `  0x${point.toString(16)}: ${JSON.stringify(value)},`).join("\n");
+const output = `/* Generated from Unicode CaseFolding-17.0.0, statuses C+F (default full folding).
+ * Copyright © 2025 Unicode, Inc. Unicode License V3; accompanying notice:
+ * public/licenses/Unicode-3.0.txt. Source https://www.unicode.org/Public/17.0.0/ucd/CaseFolding.txt
+ * SHA256 ${dataSha}. Rebuild with scripts/generate-search-folding.mjs.
+ */
+export const UNICODE_CASE_FOLDING_VERSION = "17.0.0";
+export const MAX_CASE_FOLD_UTF16_UNITS = ${maxUnits};
+const mapping: Readonly<Record<number, string>> = Object.freeze({
+${table}
+});
+/** One code point only. Unknown values, including stored lone UTF16 units,
+ * retain their exact identity. Query well-formedness is checked separately. */
+export function foldCodePoint(point: number): string {
+  if (!Number.isInteger(point) || point < 0 || point > 0x10ffff) throw new RangeError("Invalid code point.");
+  return mapping[point] ?? String.fromCodePoint(point);
+}
+`;
+await mkdir(new URL("../src/search/", import.meta.url), { recursive: true });
+await mkdir(new URL("../public/licenses/", import.meta.url), { recursive: true });
+await writeFile(new URL("../src/search/unicodeCaseFolding.ts", import.meta.url), output);
+await writeFile(new URL("../public/licenses/Unicode-3.0.txt", import.meta.url), license);
+await writeFile(new URL("../src/search/unicode-provenance.json", import.meta.url), JSON.stringify({
+  version: "17.0.0", kind: "default full case folding, C+F; no Turkic override",
+  source: "https://www.unicode.org/Public/17.0.0/ucd/CaseFolding.txt", sourceSha256: dataSha,
+  license: "Unicode-3.0", licenseSource: "https://www.unicode.org/license.txt", licenseSha256: licenseSha,
+  notice: "public/licenses/Unicode-3.0.txt", generatedMappings: rows.size,
+  generatedSha256: digest(Buffer.from(output)), generator: "scripts/generate-search-folding.mjs",
+}, null, 2) + "\n");
+console.log(`Generated ${rows.size} reviewed Unicode17 mappings; maximum ${maxUnits} UTF16 units.`);

web/src/search/contracts.ts

@@ -0,0 +1,98 @@
+import type { CardKey } from "../collections/contracts";
+import type { NoteTarget } from "../links/contracts";
+
+declare const leaseBrand: unique symbol;
+declare const nodeBrand: unique symbol;
+declare const entryBrand: unique symbol;
+export type SearchLease = Readonly<{ [leaseBrand]: true }>;
+export type SearchNodeCursor = Readonly<{ [nodeBrand]: true }>;
+export type SearchEntryCursor = Readonly<{ [entryBrand]: true }>;
+export type SearchInvalidation = "changed" | "epoch" | "closed" | "unverified";
+export type SearchVersions = Readonly<{
+  accountId: string;
+  library: Readonly<{ epoch: string; revision: number }>;
+  collections: Readonly<{ epoch: string; revision: number }>;
+}>;
+export type SearchNoteView = Readonly<{
+  id: string;
+  revision: number;
+  title: string;
+  tags: readonly string[];
+  updatedAt: string;
+  root: SearchNodeCursor;
+}>;
+export type SearchNodeView = Readonly<{
+  type: string;
+  text: string | null;
+  target: NoteTarget | null;
+  children: number;
+}>;
+export type SearchClipView = Readonly<{
+  id: string;
+  revision: number;
+  title: string;
+  originalUrl: string;
+  canonicalUrl: string;
+  text: string | null;
+  updatedAt: string;
+  archivedAt: string | null;
+}>;
+export type SearchDeckView = Readonly<{
+  id: string;
+  revision: number;
+  title: string;
+}>;
+export type SearchEntryView = Readonly<{
+  id: string;
+  revision: number;
+  deckId: string;
+  tags: readonly string[];
+}> &
+  (
+    | Readonly<{ type: "basic" | "reversed"; front: string; back: string }>
+    | Readonly<{
+        type: "cloze";
+        text: string;
+        blankCount: number;
+        cursor: SearchEntryCursor;
+      }>
+  );
+export type SearchCardView = Readonly<{
+  id: string;
+  revision: number;
+  entryId: string;
+  key: CardKey;
+}>;
+export type SearchSourceReader = Readonly<{
+  versions: SearchVersions;
+  counts: Readonly<{
+    notes: number;
+    clips: number;
+    decks: number;
+    entries: number;
+    cards: number;
+  }>;
+  note(index: number): SearchNoteView;
+  node(cursor: SearchNodeCursor): SearchNodeView;
+  child(cursor: SearchNodeCursor, index: number): SearchNodeCursor;
+  clip(index: number): SearchClipView;
+  deck(index: number): SearchDeckView;
+  entry(index: number): SearchEntryView;
+  blank(
+    cursor: SearchEntryCursor,
+    index: number,
+  ): Readonly<{ id: string; answer: string }>;
+  card(index: number): SearchCardView;
+}>;
+export type SearchSourceErrorCode =
+  | "search_source_invalid"
+  | "search_source_unavailable"
+  | "search_source_corrupt"
+  | "search_source_changed"
+  | "search_source_revoked";
+export class SearchSourceError extends Error {
+  constructor(public readonly code: SearchSourceErrorCode) {
+    super("The local search source is unavailable or changed.");
+    this.name = "SearchSourceError";
+  }
+}

web/src/search/matching.ts

@@ -0,0 +1,182 @@
+import { foldCodePoint, MAX_CASE_FOLD_UTF16_UNITS } from "./unicodeCaseFolding";
+import type { SearchQuery } from "./text";
+import type { WorkEvent } from "./work";
+
+export type SearchRange = Readonly<{ from: number; to: number }>;
+export type SearchExcerpt = Readonly<{
+  text: string;
+  matches: readonly SearchRange[];
+  before: boolean;
+  after: boolean;
+}>;
+export type TextSlice = Readonly<{ source: string; from: number; to: number }>;
+export type ChunkMatch = Readonly<{ mask: number; excerpt: SearchExcerpt | null }>;
+
+/** Descriptors only: the consumer reserves work before slicing/allocating. */
+export function* splitSearchText(source: string): Generator<TextSlice, void, void> {
+  for (let from = 0; from < source.length; ) {
+    let to = Math.min(from + 4095, source.length);
+    if (to < source.length && high(source.charCodeAt(to - 1)) && low(source.charCodeAt(to))) to--;
+    yield { source, from, to };
+    from = to;
+  }
+}
+const high = (unit: number) => unit >= 0xd800 && unit <= 0xdbff;
+const low = (unit: number) => unit >= 0xdc00 && unit <= 0xdfff;
+const utf8 = (point: number) => point <= 0x7f ? 1 : point <= 0x7ff ? 2 : point <= 0xffff ? 3 : 4;
+function bytes(text: string): number {
+  let result = 0;
+  for (const scalar of text) result += utf8(scalar.codePointAt(0)!);
+  return result;
+}
+function prefix(term: string): readonly number[] {
+  const table = new Array<number>(term.length).fill(0);
+  for (let i = 1, size = 0; i < term.length; i++) {
+    while (size > 0 && term[i] !== term[size]) size = table[size - 1]!;
+    if (term[i] === term[size]) size++;
+    table[i] = size;
+  }
+  return table;
+}
+
+/** KMP states span contiguous text nodes/chunks. A semantic atom that is not
+ * searchable MUST call boundary(): it cannot make its two neighbours match. */
+export class LiteralMatcher {
+  private readonly terms: readonly string[];
+  private readonly prefixes: readonly (readonly number[])[];
+  private readonly carryUnits: number;
+  private states: number[];
+  private tailFolded = "";
+  private tailText = "";
+  private tailFrom: number[] = [];
+  private tailTo: number[] = [];
+  private consumed = 0;
+  private pendingHigh = "";
+
+  constructor(query: SearchQuery) {
+    this.terms = query.terms;
+    this.prefixes = this.terms.map(prefix);
+    this.states = this.terms.map(() => 0);
+    this.carryUnits = Math.max(...this.terms.map(term => term.length)) - 1;
+  }
+
+  boundary(): void {
+    this.states.fill(0);
+    this.tailFolded = "";
+    this.tailText = "";
+    this.tailFrom = [];
+    this.tailTo = [];
+    this.consumed = 0;
+    this.pendingHigh = "";
+  }
+
+  *push(input: string | TextSlice): Generator<WorkEvent, ChunkMatch, void> {
+    return yield* this.scan(input, false);
+  }
+
+  /** Finish one contiguous field. A final unmatched high unit remains literal;
+   * the query compiler never admits a lone unit as a valid search term. */
+  *finish(): Generator<WorkEvent, ChunkMatch, void> {
+    if (!this.pendingHigh) return { mask: 0, excerpt: null };
+    return yield* this.scan("", true);
+  }
+
+  private *scan(input: string | TextSlice, final: boolean): Generator<WorkEvent, ChunkMatch, void> {
+    const part = typeof input === "string" ? { source: input, from: 0, to: input.length } : input;
+    const units = part.to - part.from;
+    if (!Number.isSafeInteger(part.from) || !Number.isSafeInteger(part.to) || part.from < 0 ||
+        part.to > part.source.length || units < 0 || units > 4096)
+      throw new Error("Search text requires a bounded valid chunk.");
+    // Leave room for the high unit buffered from a preceding marked node.
+    // Recursion is at most one level for a caller's single 4096-unit chunk.
+    if (units === 4096) {
+      const first = yield* this.scan({ source: part.source, from: part.from, to: part.from + 4095 }, false);
+      const second = yield* this.scan({ source: part.source, from: part.from + 4095, to: part.to }, final);
+      return { mask: first.mask | second.mask, excerpt: first.excerpt ?? second.excerpt };
+    }
+    const admittedUnits = units + this.pendingHigh.length;
+    const maximumFolded = admittedUnits * MAX_CASE_FOLD_UTF16_UNITS;
+    const combinedBound = maximumFolded + this.tailFolded.length;
+    // Includes slicing, scalar folding, map construction, each linear KMP pass,
+    // bounded excerpt/range sorting and carry compaction. Actual settlement
+    // refunds the conservative Unicode/occurrence upper bound.
+    const reserve = admittedUnits * 12 + combinedBound * (40 + 12 * this.terms.length) + 100000;
+    yield { type: "reserve", bytes: reserve, visits: 1, scalars: admittedUnits };
+    let text = this.pendingHigh + part.source.slice(part.from, part.to);
+    this.pendingHigh = "";
+    if (!final && high(text.charCodeAt(text.length - 1))) {
+      this.pendingHigh = text.slice(-1);
+      text = text.slice(0, -1);
+    }
+    let folded = "", inputBytes = 0, foldedBytes = 0, scalars = 0;
+    const from = this.tailFrom.slice(), to = this.tailTo.slice();
+    for (let offset = 0; offset < text.length; ) {
+      const point = text.codePointAt(offset)!;
+      const size = point > 0xffff ? 2 : 1;
+      const mapped = foldCodePoint(point);
+      folded += mapped;
+      inputBytes += utf8(point);
+      foldedBytes += bytes(mapped);
+      scalars++;
+      for (let i = 0; i < mapped.length; i++) {
+        from.push(this.tailText.length + offset);
+        to.push(this.tailText.length + offset + size);
+      }
+      offset += size;
+    }
+    const combined = this.tailText + text;
+    const foldedOffset = this.tailFolded.length;
+    let mask = 0, begin = -1, end = -1, comparisons = 0;
+    const ranges: SearchRange[] = [];
+    for (let i = 0; i < folded.length; i++) {
+      for (let termIndex = 0; termIndex < this.terms.length; termIndex++) {
+        const term = this.terms[termIndex]!;
+        const table = this.prefixes[termIndex]!;
+        let size = this.states[termIndex]!;
+        while (size > 0 && folded[i] !== term[size]) { comparisons++; size = table[size - 1]!; }
+        comparisons++;
+        if (folded[i] === term[size]) size++;
+        if (size === term.length) {
+          mask |= 1 << termIndex;
+          const hitFrom = from[foldedOffset + i - term.length + 1]!;
+          const hitTo = to[foldedOffset + i]!;
+          if (begin < 0) {
+            begin = Math.max(0, hitFrom - 80);
+            if (low(combined.charCodeAt(begin)) && high(combined.charCodeAt(begin - 1))) begin--;
+            end = Math.min(combined.length, begin + 240);
+            if (high(combined.charCodeAt(end - 1)) && low(combined.charCodeAt(end))) end--;
+          }
+          if (hitFrom < end && hitTo > begin)
+            ranges.push({ from: Math.max(begin, hitFrom) - begin, to: Math.min(end, hitTo) - begin });
+          size = table[size - 1]!;
+        }
+        this.states[termIndex] = size;
+      }
+    }
+    ranges.sort((a, b) => a.from - b.from || a.to - b.to);
+    const merged: SearchRange[] = [];
+    for (const range of ranges) {
+      const previous = merged.at(-1);
+      if (previous && range.from <= previous.to)
+        merged[merged.length - 1] = Object.freeze({ from: previous.from, to: Math.max(previous.to, range.to) });
+      else merged.push(Object.freeze(range));
+    }
+    const excerpt = begin < 0 ? null : Object.freeze({
+      text: combined.slice(begin, end), matches: Object.freeze(merged),
+      before: this.consumed - this.tailText.length + begin > 0,
+      after: end < combined.length || part.to < part.source.length || this.pendingHigh.length > 0,
+    });
+    const combinedFolded = this.tailFolded + folded;
+    const keepFrom = Math.max(0, combinedFolded.length - this.carryUnits);
+    const keepOriginalFrom = from[keepFrom] ?? combined.length;
+    this.tailFolded = combinedFolded.slice(keepFrom);
+    this.tailText = combined.slice(keepOriginalFrom);
+    this.tailFrom = from.slice(keepFrom).map(value => value - keepOriginalFrom);
+    this.tailTo = to.slice(keepFrom).map(value => value - keepOriginalFrom);
+    this.consumed += text.length;
+    const actual = inputBytes * 2 + foldedBytes + from.length * 8 + comparisons * 2 +
+      combined.length * 2 + this.tailText.length * 2 + ranges.length * 32 + 512;
+    yield { type: "settle", bytes: actual, visits: 1, scalars };
+    return Object.freeze({ mask, excerpt });
+  }
+}