From d832beb1000c90bad65774c469efd8fa9a2fdcbe Mon Sep 17 00:00:00 2001 From: Luan Nguyen Date: Wed, 11 Nov 2026 21:27:43 +1100 Subject: [PATCH] feat(search): add Unicode matching and local search services --- web/public/licenses/Unicode-3.0.txt | 39 + web/scripts/generate-search-folding.mjs | 61 + web/src/search/contracts.ts | 98 ++ web/src/search/matching.ts | 182 +++ web/src/search/model.ts | 847 ++++++++++++ web/src/search/results.ts | 111 ++ web/src/search/text.ts | 43 + web/src/search/unicode-provenance.json | 13 + web/src/search/unicodeCaseFolding.ts | 1600 +++++++++++++++++++++++ web/tests/data/search-text.test.ts | 80 ++ 10 files changed, 3074 insertions(+) create mode 100644 web/public/licenses/Unicode-3.0.txt create mode 100644 web/scripts/generate-search-folding.mjs create mode 100644 web/src/search/contracts.ts create mode 100644 web/src/search/matching.ts create mode 100644 web/src/search/model.ts create mode 100644 web/src/search/results.ts create mode 100644 web/src/search/text.ts create mode 100644 web/src/search/unicode-provenance.json create mode 100644 web/src/search/unicodeCaseFolding.ts create mode 100644 web/tests/data/search-text.test.ts diff --git a/web/public/licenses/Unicode-3.0.txt b/web/public/licenses/Unicode-3.0.txt new file mode 100644 index 0000000..56da589 --- /dev/null +++ b/web/public/licenses/Unicode-3.0.txt @@ -0,0 +1,39 @@ +UNICODE LICENSE V3 + +COPYRIGHT AND PERMISSION NOTICE + +Copyright © 1991-2026 Unicode, Inc. + +NOTICE TO USER: Carefully read the following legal agreement. BY +DOWNLOADING, INSTALLING, COPYING OR OTHERWISE USING DATA FILES, AND/OR +SOFTWARE, YOU UNEQUIVOCALLY ACCEPT, AND AGREE TO BE BOUND BY, ALL OF THE +TERMS AND CONDITIONS OF THIS AGREEMENT. IF YOU DO NOT AGREE, DO NOT +DOWNLOAD, INSTALL, COPY, DISTRIBUTE OR USE THE DATA FILES OR SOFTWARE. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of data files and any associated documentation (the "Data Files") or +software and any associated documentation (the "Software") to deal in the +Data Files or Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, and/or sell +copies of the Data Files or Software, and to permit persons to whom the +Data Files or Software are furnished to do so, provided that either (a) +this copyright and permission notice appear with all copies of the Data +Files or Software, or (b) this copyright and permission notice appear in +associated Documentation. + +THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF +THIRD PARTY RIGHTS. + +IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS NOTICE +BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES, +OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, +WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, +ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THE DATA +FILES OR SOFTWARE. + +Except as contained in this notice, the name of a copyright holder shall +not be used in advertising or otherwise to promote the sale, use or other +dealings in these Data Files or Software without prior written +authorization of the copyright holder. diff --git a/web/scripts/generate-search-folding.mjs b/web/scripts/generate-search-folding.mjs new file mode 100644 index 0000000..406e0f2 --- /dev/null +++ b/web/scripts/generate-search-folding.mjs @@ -0,0 +1,61 @@ +// Rebuild the reviewed Unicode data adaptation from explicit local inputs. +// This script never downloads or executes upstream content. +import { readFile, writeFile, mkdir } from "node:fs/promises"; +import { createHash } from "node:crypto"; + +const dataSha = "ff8d8fefbf123574205085d6714c36149eb946d717a0c585c27f0f4ef58c4183"; +const licenseSha = "e7a93b009565cfce55919a381437ac4db883e9da2126fa28b91d12732bc53d96"; +const inputs = process.argv.slice(2); +if (inputs.length !== 2) throw new Error("Pass the reviewed CaseFolding.txt and LICENSE.txt paths."); +const [data, license] = await Promise.all(inputs.map(path => readFile(path))); +const digest = bytes => createHash("sha256").update(bytes).digest("hex"); +if (digest(data) !== dataSha || digest(license) !== licenseSha) throw new Error("Unicode source differs from the reviewed pin."); +const rows = new Map(); +for (const line of data.toString("utf8").split(/\r?\n/)) { + const body = line.split("#")[0].trim(); + if (!body) continue; + const fields = body.split(";").map(value => value.trim()); + if (!/^[0-9A-F]{4,6}$/.test(fields[0]) || !["C", "F", "S", "T"].includes(fields[1])) throw new Error("Unexpected Unicode record."); + if (fields[1] !== "C" && fields[1] !== "F") continue; + const point = Number.parseInt(fields[0], 16); + const targets = fields[2].split(" ").map(value => { + if (!/^[0-9A-F]{4,6}$/.test(value)) throw new Error("Unexpected mapping."); + const result = Number.parseInt(value, 16); + if (result > 0x10ffff || (result >= 0xd800 && result <= 0xdfff)) throw new Error("Invalid mapping scalar."); + return result; + }); + if (point > 0x10ffff || rows.has(point) || targets.length > 3) throw new Error("Unexpected mapping bounds."); + rows.set(point, String.fromCodePoint(...targets)); +} +let maxUnits = 2; +for (const value of rows.values()) maxUnits = Math.max(maxUnits, value.length); +const table = [...rows].sort((a, b) => a[0] - b[0]).map(([point, value]) => ` 0x${point.toString(16)}: ${JSON.stringify(value)},`).join("\n"); +const output = `/* Generated from Unicode CaseFolding-17.0.0, statuses C+F (default full folding). + * Copyright © 2025 Unicode, Inc. Unicode License V3; accompanying notice: + * public/licenses/Unicode-3.0.txt. Source https://www.unicode.org/Public/17.0.0/ucd/CaseFolding.txt + * SHA256 ${dataSha}. Rebuild with scripts/generate-search-folding.mjs. + */ +export const UNICODE_CASE_FOLDING_VERSION = "17.0.0"; +export const MAX_CASE_FOLD_UTF16_UNITS = ${maxUnits}; +const mapping: Readonly> = Object.freeze({ +${table} +}); +/** One code point only. Unknown values, including stored lone UTF16 units, + * retain their exact identity. Query well-formedness is checked separately. */ +export function foldCodePoint(point: number): string { + if (!Number.isInteger(point) || point < 0 || point > 0x10ffff) throw new RangeError("Invalid code point."); + return mapping[point] ?? String.fromCodePoint(point); +} +`; +await mkdir(new URL("../src/search/", import.meta.url), { recursive: true }); +await mkdir(new URL("../public/licenses/", import.meta.url), { recursive: true }); +await writeFile(new URL("../src/search/unicodeCaseFolding.ts", import.meta.url), output); +await writeFile(new URL("../public/licenses/Unicode-3.0.txt", import.meta.url), license); +await writeFile(new URL("../src/search/unicode-provenance.json", import.meta.url), JSON.stringify({ + version: "17.0.0", kind: "default full case folding, C+F; no Turkic override", + source: "https://www.unicode.org/Public/17.0.0/ucd/CaseFolding.txt", sourceSha256: dataSha, + license: "Unicode-3.0", licenseSource: "https://www.unicode.org/license.txt", licenseSha256: licenseSha, + notice: "public/licenses/Unicode-3.0.txt", generatedMappings: rows.size, + generatedSha256: digest(Buffer.from(output)), generator: "scripts/generate-search-folding.mjs", +}, null, 2) + "\n"); +console.log(`Generated ${rows.size} reviewed Unicode17 mappings; maximum ${maxUnits} UTF16 units.`); diff --git a/web/src/search/contracts.ts b/web/src/search/contracts.ts new file mode 100644 index 0000000..cef9be8 --- /dev/null +++ b/web/src/search/contracts.ts @@ -0,0 +1,98 @@ +import type { CardKey } from "../collections/contracts"; +import type { NoteTarget } from "../links/contracts"; + +declare const leaseBrand: unique symbol; +declare const nodeBrand: unique symbol; +declare const entryBrand: unique symbol; +export type SearchLease = Readonly<{ [leaseBrand]: true }>; +export type SearchNodeCursor = Readonly<{ [nodeBrand]: true }>; +export type SearchEntryCursor = Readonly<{ [entryBrand]: true }>; +export type SearchInvalidation = "changed" | "epoch" | "closed" | "unverified"; +export type SearchVersions = Readonly<{ + accountId: string; + library: Readonly<{ epoch: string; revision: number }>; + collections: Readonly<{ epoch: string; revision: number }>; +}>; +export type SearchNoteView = Readonly<{ + id: string; + revision: number; + title: string; + tags: readonly string[]; + updatedAt: string; + root: SearchNodeCursor; +}>; +export type SearchNodeView = Readonly<{ + type: string; + text: string | null; + target: NoteTarget | null; + children: number; +}>; +export type SearchClipView = Readonly<{ + id: string; + revision: number; + title: string; + originalUrl: string; + canonicalUrl: string; + text: string | null; + updatedAt: string; + archivedAt: string | null; +}>; +export type SearchDeckView = Readonly<{ + id: string; + revision: number; + title: string; +}>; +export type SearchEntryView = Readonly<{ + id: string; + revision: number; + deckId: string; + tags: readonly string[]; +}> & + ( + | Readonly<{ type: "basic" | "reversed"; front: string; back: string }> + | Readonly<{ + type: "cloze"; + text: string; + blankCount: number; + cursor: SearchEntryCursor; + }> + ); +export type SearchCardView = Readonly<{ + id: string; + revision: number; + entryId: string; + key: CardKey; +}>; +export type SearchSourceReader = Readonly<{ + versions: SearchVersions; + counts: Readonly<{ + notes: number; + clips: number; + decks: number; + entries: number; + cards: number; + }>; + note(index: number): SearchNoteView; + node(cursor: SearchNodeCursor): SearchNodeView; + child(cursor: SearchNodeCursor, index: number): SearchNodeCursor; + clip(index: number): SearchClipView; + deck(index: number): SearchDeckView; + entry(index: number): SearchEntryView; + blank( + cursor: SearchEntryCursor, + index: number, + ): Readonly<{ id: string; answer: string }>; + card(index: number): SearchCardView; +}>; +export type SearchSourceErrorCode = + | "search_source_invalid" + | "search_source_unavailable" + | "search_source_corrupt" + | "search_source_changed" + | "search_source_revoked"; +export class SearchSourceError extends Error { + constructor(public readonly code: SearchSourceErrorCode) { + super("The local search source is unavailable or changed."); + this.name = "SearchSourceError"; + } +} diff --git a/web/src/search/matching.ts b/web/src/search/matching.ts new file mode 100644 index 0000000..cc5e3cb --- /dev/null +++ b/web/src/search/matching.ts @@ -0,0 +1,182 @@ +import { foldCodePoint, MAX_CASE_FOLD_UTF16_UNITS } from "./unicodeCaseFolding"; +import type { SearchQuery } from "./text"; +import type { WorkEvent } from "./work"; + +export type SearchRange = Readonly<{ from: number; to: number }>; +export type SearchExcerpt = Readonly<{ + text: string; + matches: readonly SearchRange[]; + before: boolean; + after: boolean; +}>; +export type TextSlice = Readonly<{ source: string; from: number; to: number }>; +export type ChunkMatch = Readonly<{ mask: number; excerpt: SearchExcerpt | null }>; + +/** Descriptors only: the consumer reserves work before slicing/allocating. */ +export function* splitSearchText(source: string): Generator { + for (let from = 0; from < source.length; ) { + let to = Math.min(from + 4095, source.length); + if (to < source.length && high(source.charCodeAt(to - 1)) && low(source.charCodeAt(to))) to--; + yield { source, from, to }; + from = to; + } +} +const high = (unit: number) => unit >= 0xd800 && unit <= 0xdbff; +const low = (unit: number) => unit >= 0xdc00 && unit <= 0xdfff; +const utf8 = (point: number) => point <= 0x7f ? 1 : point <= 0x7ff ? 2 : point <= 0xffff ? 3 : 4; +function bytes(text: string): number { + let result = 0; + for (const scalar of text) result += utf8(scalar.codePointAt(0)!); + return result; +} +function prefix(term: string): readonly number[] { + const table = new Array(term.length).fill(0); + for (let i = 1, size = 0; i < term.length; i++) { + while (size > 0 && term[i] !== term[size]) size = table[size - 1]!; + if (term[i] === term[size]) size++; + table[i] = size; + } + return table; +} + +/** KMP states span contiguous text nodes/chunks. A semantic atom that is not + * searchable MUST call boundary(): it cannot make its two neighbours match. */ +export class LiteralMatcher { + private readonly terms: readonly string[]; + private readonly prefixes: readonly (readonly number[])[]; + private readonly carryUnits: number; + private states: number[]; + private tailFolded = ""; + private tailText = ""; + private tailFrom: number[] = []; + private tailTo: number[] = []; + private consumed = 0; + private pendingHigh = ""; + + constructor(query: SearchQuery) { + this.terms = query.terms; + this.prefixes = this.terms.map(prefix); + this.states = this.terms.map(() => 0); + this.carryUnits = Math.max(...this.terms.map(term => term.length)) - 1; + } + + boundary(): void { + this.states.fill(0); + this.tailFolded = ""; + this.tailText = ""; + this.tailFrom = []; + this.tailTo = []; + this.consumed = 0; + this.pendingHigh = ""; + } + + *push(input: string | TextSlice): Generator { + return yield* this.scan(input, false); + } + + /** Finish one contiguous field. A final unmatched high unit remains literal; + * the query compiler never admits a lone unit as a valid search term. */ + *finish(): Generator { + if (!this.pendingHigh) return { mask: 0, excerpt: null }; + return yield* this.scan("", true); + } + + private *scan(input: string | TextSlice, final: boolean): Generator { + const part = typeof input === "string" ? { source: input, from: 0, to: input.length } : input; + const units = part.to - part.from; + if (!Number.isSafeInteger(part.from) || !Number.isSafeInteger(part.to) || part.from < 0 || + part.to > part.source.length || units < 0 || units > 4096) + throw new Error("Search text requires a bounded valid chunk."); + // Leave room for the high unit buffered from a preceding marked node. + // Recursion is at most one level for a caller's single 4096-unit chunk. + if (units === 4096) { + const first = yield* this.scan({ source: part.source, from: part.from, to: part.from + 4095 }, false); + const second = yield* this.scan({ source: part.source, from: part.from + 4095, to: part.to }, final); + return { mask: first.mask | second.mask, excerpt: first.excerpt ?? second.excerpt }; + } + const admittedUnits = units + this.pendingHigh.length; + const maximumFolded = admittedUnits * MAX_CASE_FOLD_UTF16_UNITS; + const combinedBound = maximumFolded + this.tailFolded.length; + // Includes slicing, scalar folding, map construction, each linear KMP pass, + // bounded excerpt/range sorting and carry compaction. Actual settlement + // refunds the conservative Unicode/occurrence upper bound. + const reserve = admittedUnits * 12 + combinedBound * (40 + 12 * this.terms.length) + 100000; + yield { type: "reserve", bytes: reserve, visits: 1, scalars: admittedUnits }; + let text = this.pendingHigh + part.source.slice(part.from, part.to); + this.pendingHigh = ""; + if (!final && high(text.charCodeAt(text.length - 1))) { + this.pendingHigh = text.slice(-1); + text = text.slice(0, -1); + } + let folded = "", inputBytes = 0, foldedBytes = 0, scalars = 0; + const from = this.tailFrom.slice(), to = this.tailTo.slice(); + for (let offset = 0; offset < text.length; ) { + const point = text.codePointAt(offset)!; + const size = point > 0xffff ? 2 : 1; + const mapped = foldCodePoint(point); + folded += mapped; + inputBytes += utf8(point); + foldedBytes += bytes(mapped); + scalars++; + for (let i = 0; i < mapped.length; i++) { + from.push(this.tailText.length + offset); + to.push(this.tailText.length + offset + size); + } + offset += size; + } + const combined = this.tailText + text; + const foldedOffset = this.tailFolded.length; + let mask = 0, begin = -1, end = -1, comparisons = 0; + const ranges: SearchRange[] = []; + for (let i = 0; i < folded.length; i++) { + for (let termIndex = 0; termIndex < this.terms.length; termIndex++) { + const term = this.terms[termIndex]!; + const table = this.prefixes[termIndex]!; + let size = this.states[termIndex]!; + while (size > 0 && folded[i] !== term[size]) { comparisons++; size = table[size - 1]!; } + comparisons++; + if (folded[i] === term[size]) size++; + if (size === term.length) { + mask |= 1 << termIndex; + const hitFrom = from[foldedOffset + i - term.length + 1]!; + const hitTo = to[foldedOffset + i]!; + if (begin < 0) { + begin = Math.max(0, hitFrom - 80); + if (low(combined.charCodeAt(begin)) && high(combined.charCodeAt(begin - 1))) begin--; + end = Math.min(combined.length, begin + 240); + if (high(combined.charCodeAt(end - 1)) && low(combined.charCodeAt(end))) end--; + } + if (hitFrom < end && hitTo > begin) + ranges.push({ from: Math.max(begin, hitFrom) - begin, to: Math.min(end, hitTo) - begin }); + size = table[size - 1]!; + } + this.states[termIndex] = size; + } + } + ranges.sort((a, b) => a.from - b.from || a.to - b.to); + const merged: SearchRange[] = []; + for (const range of ranges) { + const previous = merged.at(-1); + if (previous && range.from <= previous.to) + merged[merged.length - 1] = Object.freeze({ from: previous.from, to: Math.max(previous.to, range.to) }); + else merged.push(Object.freeze(range)); + } + const excerpt = begin < 0 ? null : Object.freeze({ + text: combined.slice(begin, end), matches: Object.freeze(merged), + before: this.consumed - this.tailText.length + begin > 0, + after: end < combined.length || part.to < part.source.length || this.pendingHigh.length > 0, + }); + const combinedFolded = this.tailFolded + folded; + const keepFrom = Math.max(0, combinedFolded.length - this.carryUnits); + const keepOriginalFrom = from[keepFrom] ?? combined.length; + this.tailFolded = combinedFolded.slice(keepFrom); + this.tailText = combined.slice(keepOriginalFrom); + this.tailFrom = from.slice(keepFrom).map(value => value - keepOriginalFrom); + this.tailTo = to.slice(keepFrom).map(value => value - keepOriginalFrom); + this.consumed += text.length; + const actual = inputBytes * 2 + foldedBytes + from.length * 8 + comparisons * 2 + + combined.length * 2 + this.tailText.length * 2 + ranges.length * 32 + 512; + yield { type: "settle", bytes: actual, visits: 1, scalars }; + return Object.freeze({ mask, excerpt }); + } +} diff --git a/web/src/search/model.ts b/web/src/search/model.ts new file mode 100644 index 0000000..92e0e40 --- /dev/null +++ b/web/src/search/model.ts @@ -0,0 +1,847 @@ +import type { + SearchSourceReader, + SearchNoteView, + SearchEntryView, + SearchDeckView, + SearchCardView, + SearchNodeCursor, +} from "./contracts"; +import type { SearchQuery } from "./text"; +import { LiteralMatcher, splitSearchText } from "./matching"; +import type { SearchExcerpt, TextSlice } from "./matching"; +import type { + SearchCounts, + SearchKind, + SearchHit, + SearchProgress, +} from "./results"; +import type { WorkEvent, WorkCost } from "./work"; +import { SearchIdentityMap } from "./identity"; + +declare const indexBrand: unique symbol; +export type SearchIndex = Readonly<{ [indexBrand]: true }>; +type Segment = TextSlice | null | { placeholder: true }; +type ClozeSegment = TextSlice | { blankId: string; answer: string }; +type Entry = { + ordinal: number; + view: SearchEntryView; + segments: ClozeSegment[] | null; +}; +type Row = { + ordinal: number; + primary: boolean; + title: string; + updatedAt: string; + entryOrdinal: number; + card: SearchCardView | null; +}; +type State = { + reader: SearchSourceReader | null; + query: SearchQuery | null; + notes: SearchIdentityMap; + entries: SearchIdentityMap; + decks: SearchIdentityMap; + noteRows: Row[]; + clipRows: Row[]; + cardRows: Row[]; + disposed: boolean; + pages: Set>; +}; +const states = new WeakMap(); +function stateOf( + index: SearchIndex, +): State & { reader: SearchSourceReader; query: SearchQuery } { + const state = states.get(index); + if (!state || state.disposed || !state.reader || !state.query) + throw new Error("Search index was released or is invalid."); + return state as State & { reader: SearchSourceReader; query: SearchQuery }; +} +function* work( + bytes: number, + visits: number, + operation: () => T, + scalars = 0, +): Generator { + const cost: WorkCost = { bytes, visits, scalars }; + yield { type: "reserve", ...cost }; + const result = operation(); + yield { type: "settle", ...cost }; + return result; +} +function* read(operation: () => T): Generator { + return yield* work(2048, 1, operation); +} +function clear(state: State) { + state.disposed = true; + for (const page of state.pages) page.return(Object.freeze([])); + state.pages.clear(); + state.reader = null; + state.query = null; + state.notes.clear(); + state.entries.clear(); + state.decks.clear(); + state.noteRows.length = 0; + state.clipRows.length = 0; + state.cardRows.length = 0; +} +export function disposeSearchIndex(index: SearchIndex): void { + const state = states.get(index); + if (!state) throw new Error("Invalid search index."); + clear(state); +} +type Stream = Generator; +type Emit = (segment: Segment) => Stream; +function* plain(source: string, emit: Emit): Stream { + for (const part of splitSearchText(source)) yield* emit(part); +} +function* document(state: State, root: SearchNodeCursor, emit: Emit): Stream { + const reader = state.reader!; + const stack: { cursor: SearchNodeCursor; exit: boolean; block: boolean }[] = + []; + yield* work(48, 1, () => + stack.push({ cursor: root, exit: false, block: false }), + ); + while (stack.length) { + const next = yield* work(48, 1, () => stack.pop()!); + if (next.exit) { + if (next.block) yield* emit({ source: "\n", from: 0, to: 1 }); + continue; + } + const node = yield* read(() => reader.node(next.cursor)); + if (node.type === "text") { + if (node.text) yield* plain(node.text, emit); + continue; + } + if (node.type === "noteLink") { + const target = node.target, + versions = reader.versions; + if ( + target && + target.accountId === versions.accountId && + target.libraryEpoch === versions.library.epoch + ) { + const note = yield* state.notes.get(target.noteId); + if (note && note.title.length > 0) yield* plain(note.title, emit); + else yield* emit(null); + } else yield* emit(null); + continue; + } + if (node.type === "hardBreak" || node.type === "horizontalRule") { + yield* emit({ source: "\n", from: 0, to: 1 }); + continue; + } + const block = + node.type === "paragraph" || + node.type === "heading" || + node.type === "codeBlock" || + node.type === "blockquote" || + node.type === "listItem" || + node.type === "taskItem"; + if (block) yield* emit({ source: "\n", from: 0, to: 1 }); + yield* work(48, 1, () => + stack.push({ cursor: next.cursor, exit: true, block }), + ); + for (let i = node.children - 1; i >= 0; i--) { + const child = yield* read(() => reader.child(next.cursor, i)); + yield* work(48, 1, () => + stack.push({ cursor: child, exit: false, block: false }), + ); + } + } +} +function* parseCloze( + reader: SearchSourceReader, + view: Extract, +): Generator { + const blanks = new SearchIdentityMap(), + result: ClozeSegment[] = []; + try { + for (let i = 0; i < view.blankCount; i++) { + const blank = yield* read(() => reader.blank(view.cursor, i)); + yield* blanks.set(blank.id, blank.answer); + } + let start = 0, + open = -1, + idLength = 0, + stage = 0; + for (let from = 0; from < view.text.length;) { + const to = Math.min(view.text.length, from + 4096), + candidates: { from: number; to: number; id: string }[] = []; + yield* work( + (to - from) * 40 + 128, + 1, + () => { + for (let p = from; p < to; p++) { + const unit = view.text[p]!; + if (stage === 0) { + if (unit === "{") { + open = p; + stage = 1; + } + } else if (stage === 1) { + if (unit === "{") { + stage = 2; + idLength = 0; + } else { + stage = 0; + open = -1; + } + } else if (stage === 2) { + if (unit === "}") { + stage = 3; + } else if (unit === "{") { + open = idLength === 0 ? p - 1 : p; + stage = idLength === 0 ? 2 : 1; + idLength = 0; + } else { + if (idLength < 101) idLength++; + else { + stage = 0; + open = -1; + } + } + } else { + if (unit === "}" && idLength) { + candidates.push({ + from: open, + to: p + 1, + id: view.text.slice(open + 2, p - 1), + }); + stage = 0; + open = -1; + } else if (unit === "{") { + open = p; + stage = 1; + idLength = 0; + } else { + stage = 0; + open = -1; + } + } + } + }, + to - from, + ); + for (const candidate of candidates) { + const answer = yield* blanks.get(candidate.id); + if (answer === undefined) + throw new Error("Invalid admitted Cloze marker."); + yield* work(96, 1, () => { + if (candidate.from > start) + result.push({ source: view.text, from: start, to: candidate.from }); + result.push({ blankId: candidate.id, answer }); + start = candidate.to; + }); + } + from = to; + } + if (start < view.text.length) + yield* work(32, 1, () => + result.push({ source: view.text, from: start, to: view.text.length }), + ); + return result; + } finally { + blanks.clear(); + } +} +function* cardField( + entry: Entry, + key: string, + front: boolean, + emit: Emit, +): Stream { + const view = entry.view; + if (view.type !== "cloze") { + const source = + (key === "reversed:reverse") === front ? view.back : view.front; + yield* plain(source, emit); + return; + } + const selected = key.slice(6); + for (const segment of entry.segments!) { + yield* work(32, 1, () => undefined); + if ("blankId" in segment) { + if (front && segment.blankId === selected) + yield* emit({ placeholder: true }); + else yield* plain(segment.answer, emit); + } + if ("source" in segment) { + for (let from = segment.from; from < segment.to;) { + const to = Math.min(segment.to, from + 4095); + yield* emit({ source: segment.source, from, to }); + from = to; + } + } + } +} +type Scan = { + mask: number; + excerpt: SearchExcerpt | null; + preview: SearchExcerpt; +}; +function* scan( + query: SearchQuery, + stream: (emit: Emit) => Stream, +): Generator { + const queryUnits = query.terms.reduce((sum, term) => sum + term.length, 0); + const matcher = yield* work( + queryUnits * 16 + 256, + 1, + () => new LiteralMatcher(query), + queryUnits, + ); + let mask = 0, + excerpt: SearchExcerpt | null = null, + prefix = "", + more = false; + const contexts: { excerpt: SearchExcerpt; runFrom: number }[] = []; + let runFrom = 0; + const emit: Emit = function* (segment) { + if (segment === null || "placeholder" in segment) { + const ended = yield* matcher.finish(); + mask |= ended.mask; + excerpt ??= ended.excerpt; + yield* work(128, 1, () => matcher.boundary()); + if (segment) { + yield* work(16, 1, () => { + const add = "[...]"; + const remaining = 240 - prefix.length; + if (remaining > 0) prefix += add.slice(0, remaining); + if (add.length > remaining) more = true; + }); + } + if (excerpt) excerpt = Object.freeze({ ...excerpt, after: true }); + runFrom = prefix.length; + return; + } + const found = yield* matcher.push(segment); + mask |= found.mask; + if (excerpt) excerpt = Object.freeze({ ...excerpt, after: true }); + else excerpt = found.excerpt; + if (found.excerpt && prefix.length < 240) + contexts.push({ excerpt: found.excerpt, runFrom }); + yield* work(1024, 1, () => { + let take = Math.min(segment.to - segment.from, 240 - prefix.length); + if (take > 0) { + const last = segment.source.charCodeAt(segment.from + take - 1), + next = segment.source.charCodeAt(segment.from + take); + if ( + last >= 0xd800 && + last <= 0xdbff && + next >= 0xdc00 && + next <= 0xdfff + ) + take--; + prefix += segment.source.slice(segment.from, segment.from + take); + } + if (segment.to - segment.from > take) more = true; + }); + }; + yield* stream(emit); + const ended = yield* matcher.finish(); + mask |= ended.mask; + excerpt ??= ended.excerpt; + if (excerpt && !more) { + const marked = yield* work(256, 1, () => + new Array(prefix.length).fill(false), + ); + for (const context of contexts) { + const at = yield* findText(prefix, context.excerpt.text, context.runFrom); + if (at >= 0) + yield* work( + 2048, + 1, + () => { + for (const range of context.excerpt.matches) + for ( + let i = at + range.from; + i < Math.min(prefix.length, at + range.to); + i++ + ) + marked[i] = true; + }, + prefix.length, + ); + } + excerpt = yield* work( + 4096, + 1, + () => { + const ranges: { from: number; to: number }[] = []; + for (let i = 0; i < marked.length;) { + if (!marked[i]) { + i++; + continue; + } + const from = i; + while (marked[i]) i++; + ranges.push(Object.freeze({ from, to: i })); + } + return Object.freeze({ + text: prefix, + matches: Object.freeze(ranges), + before: false, + after: false, + }); + }, + prefix.length, + ); + } + + if (more && prefix.length) { + const last = prefix.charCodeAt(prefix.length - 1); + if (last >= 0xd800 && last <= 0xdbff) + prefix = yield* work(512, 1, () => prefix.slice(0, -1)); + } + return { + mask, + excerpt, + preview: Object.freeze({ + text: prefix, + matches: Object.freeze([]), + before: false, + after: more, + }), + }; +} +function* findText( + source: string, + text: string, + from = 0, +): Generator { + if (!text.length) return from; + let candidate = from, + offset = 0, + found = -1; + while (candidate <= source.length - text.length && found < 0) { + yield { type: "reserve", bytes: 12288, visits: 1, scalars: 4096 }; + let comparisons = 0; + while (comparisons < 2048 && candidate <= source.length - text.length) { + comparisons++; + if (source.charCodeAt(candidate + offset) === text.charCodeAt(offset)) { + if (++offset === text.length) { + found = candidate; + break; + } + } else { + candidate++; + offset = 0; + } + } + yield { + type: "settle", + bytes: comparisons * 6, + visits: 1, + scalars: comparisons * 2, + }; + } + return found; +} +function* titleExcerpt( + title: string, + match: SearchExcerpt | null, + maximum = 500, +): Generator { + if (title.length > maximum) throw new Error("Invalid admitted title."); + const offset = match ? yield* findText(title, match.text) : -1; + return yield* work( + 4096, + 1, + () => + Object.freeze({ + text: title, + matches: Object.freeze( + offset < 0 + ? [] + : match!.matches.map((range) => + Object.freeze({ + from: range.from + offset, + to: range.to + offset, + }), + ), + ), + before: false, + after: false, + }), + title.length, + ); +} + +type Matched = { mask: number; primary: boolean; hit?: SearchHit }; +function* matchNote( + state: State, + ordinal: number, + project: boolean, +): Generator { + const reader = state.reader!, + query = state.query!, + note = yield* read(() => reader.note(ordinal)); + const title = yield* scan(query, (emit) => plain(note.title, emit)); + let mask = title.mask; + const secondary: SearchExcerpt[] = []; + for (const tag of note.tags) { + const part = yield* scan(query, (emit) => plain(tag, emit)); + mask |= part.mask; + if (project && part.excerpt && secondary.length < 3) + secondary.push(part.excerpt); + } + const body = yield* scan(query, (emit) => document(state, note.root, emit)); + mask |= body.mask; + if (project && body.excerpt && secondary.length < 3) + secondary.push(body.excerpt); + const hit = project + ? Object.freeze({ + locator: Object.freeze({ + kind: "note" as const, + accountId: reader.versions.accountId, + libraryEpoch: reader.versions.library.epoch, + noteId: note.id, + }), + entityRevision: note.revision, + primary: yield* titleExcerpt(note.title, title.excerpt), + secondary: Object.freeze(secondary), + }) + : undefined; + return { mask, primary: title.mask !== 0, hit }; +} +function* matchClip( + state: State, + ordinal: number, + project: boolean, +): Generator { + const reader = state.reader!, + query = state.query!, + clip = yield* read(() => reader.clip(ordinal)), + title = yield* scan(query, (emit) => plain(clip.title, emit)); + let mask = title.mask; + const secondary: SearchExcerpt[] = []; + for (const value of [clip.originalUrl, clip.canonicalUrl, clip.text]) { + if (value === null) continue; + const part = yield* scan(query, (emit) => plain(value, emit)); + mask |= part.mask; + if (project && part.excerpt && secondary.length < 3) + secondary.push(part.excerpt); + } + return { + mask, + primary: title.mask !== 0, + hit: project + ? Object.freeze({ + locator: Object.freeze({ + kind: "clip" as const, + accountId: reader.versions.accountId, + collectionsEpoch: reader.versions.collections.epoch, + clipId: clip.id, + }), + entityRevision: clip.revision, + primary: yield* titleExcerpt(clip.title, title.excerpt), + secondary: Object.freeze(secondary), + archived: clip.archivedAt !== null, + clipContent: + clip.text === null + ? ("link-only" as const) + : ("saved-text" as const), + }) + : undefined, + }; +} +function* matchCard( + state: State, + card: SearchCardView, + project: boolean, +): Generator { + const reader = state.reader!, + query = state.query!, + entry = yield* state.entries.get(card.entryId); + if (!entry) throw new Error("Missing admitted entry."); + const deck = yield* state.decks.get(entry.view.deckId); + if (!deck) throw new Error("Missing admitted deck."); + const front = yield* scan(query, (emit) => + cardField(entry, card.key, true, emit), + ); + let mask = front.mask; + const secondary: SearchExcerpt[] = []; + const back = yield* scan(query, (emit) => + cardField(entry, card.key, false, emit), + ); + mask |= back.mask; + if (project && back.excerpt) secondary.push(back.excerpt); + const deckMatch = yield* scan(query, (emit) => plain(deck.title, emit)); + mask |= deckMatch.mask; + if (project && deckMatch.excerpt) + secondary.push(yield* titleExcerpt(deck.title, deckMatch.excerpt, 1000)); + for (const tag of entry.view.tags) { + const part = yield* scan(query, (emit) => plain(tag, emit)); + mask |= part.mask; + if (project && part.excerpt && secondary.length < 3) + secondary.push(part.excerpt); + } + return { + mask, + primary: front.mask !== 0, + hit: project + ? Object.freeze({ + locator: Object.freeze({ + kind: "card" as const, + accountId: reader.versions.accountId, + collectionsEpoch: reader.versions.collections.epoch, + cardId: card.id, + entryId: card.entryId, + key: card.key, + }), + entityRevision: card.revision, + primary: front.excerpt ?? front.preview, + secondary: Object.freeze(secondary), + deckTitle: deck.title, + }) + : undefined, + }; +} +function* compareText( + a: string, + b: string, +): Generator { + for (let from = 0; from < Math.min(a.length, b.length);) { + const to = Math.min(a.length, b.length, from + 2048); + const result = yield* work( + (to - from) * 6, + 1, + () => { + for (let i = from; i < to; i++) { + const delta = a.charCodeAt(i) - b.charCodeAt(i); + if (delta) return delta; + } + return 0; + }, + (to - from) * 2, + ); + if (result) return result; + from = to; + } + return a.length - b.length; +} +function* compare( + a: Row, + b: Row, + kind: SearchKind, +): Generator { + yield* work(32, 1, () => undefined); + if (a.primary !== b.primary) return a.primary ? -1 : 1; + if (kind !== "card") { + const date = yield* compareText(b.updatedAt, a.updatedAt); + if (date) return date; + } + const title = yield* compareText(a.title, b.title); + if (title) return title; + return kind === "card" + ? a.entryOrdinal - b.entryOrdinal || a.ordinal - b.ordinal + : a.ordinal - b.ordinal; +} +function* order( + rows: Row[], + kind: SearchKind, +): Generator { + let source = rows, + target = yield* work( + rows.length * 8 + 32, + 1, + () => new Array(rows.length), + ); + for (let width = 1; width < rows.length; width *= 2) { + for (let start = 0; start < rows.length; start += width * 2) { + let left = start, + right = Math.min(start + width, rows.length), + end = Math.min(start + width * 2, rows.length), + middle = right; + for (let position = start; position < end; position++) { + const useLeft = + left < middle && + (right >= end || + (yield* compare(source[left]!, source[right]!, kind)) <= 0); + yield* work(16, 1, () => { + target[position] = source[useLeft ? left++ : right++]!; + }); + } + } + const old = source; + source = target; + target = old; + } + return source; +} +export function* buildSearchIndex( + reader: SearchSourceReader, + query: SearchQuery, + onProgress?: (progress: SearchProgress) => void, +): Generator { + const state: State = { + reader, + query, + notes: new SearchIdentityMap(), + entries: new SearchIdentityMap(), + decks: new SearchIdentityMap(), + noteRows: [], + clipRows: [], + cardRows: [], + disposed: false, + pages: new Set(), + }; + let completed = false; + const total = reader.counts.notes + reader.counts.clips + reader.counts.cards; + let checked = 0; + const progress = (phase: SearchProgress["phase"]) => + onProgress?.( + Object.freeze({ phase, checkedEntities: checked, totalEntities: total }), + ); + try { + progress("indexing"); + for (let i = 0; i < reader.counts.notes; i++) { + const note = yield* read(() => reader.note(i)); + yield* state.notes.set(note.id, note); + } + for (let i = 0; i < reader.counts.decks; i++) { + const deck = yield* read(() => reader.deck(i)); + yield* state.decks.set(deck.id, deck); + } + for (let i = 0; i < reader.counts.entries; i++) { + const view = yield* read(() => reader.entry(i)), + segments = + view.type === "cloze" ? yield* parseCloze(reader, view) : null; + yield* state.entries.set(view.id, { ordinal: i, view, segments }); + } + const wanted = (1 << query.terms.length) - 1; + progress("matching"); + for (let i = 0; i < reader.counts.notes; i++) { + const found = yield* matchNote(state, i, false); + if (found.mask === wanted) { + const note = yield* read(() => reader.note(i)); + yield* work(96, 1, () => + state.noteRows.push({ + ordinal: i, + primary: found.primary, + title: note.title, + updatedAt: note.updatedAt, + entryOrdinal: 0, + card: null, + }), + ); + } + checked++; + progress("matching"); + } + for (let i = 0; i < reader.counts.clips; i++) { + const found = yield* matchClip(state, i, false); + if (found.mask === wanted) { + const clip = yield* read(() => reader.clip(i)); + yield* work(96, 1, () => + state.clipRows.push({ + ordinal: i, + primary: found.primary, + title: clip.title, + updatedAt: clip.updatedAt, + entryOrdinal: 0, + card: null, + }), + ); + } + checked++; + progress("matching"); + } + for (let i = 0; i < reader.counts.cards; i++) { + const card = yield* read(() => reader.card(i)), + found = yield* matchCard(state, card, false); + if (found.mask === wanted) { + const entry = (yield* state.entries.get(card.entryId))!, + deck = (yield* state.decks.get(entry.view.deckId))!; + yield* work(96, 1, () => + state.cardRows.push({ + ordinal: i, + primary: found.primary, + title: deck.title, + updatedAt: "", + entryOrdinal: entry.ordinal, + card, + }), + ); + } + checked++; + progress("matching"); + } + progress("ordering"); + state.noteRows = yield* order(state.noteRows, "note"); + state.clipRows = yield* order(state.clipRows, "clip"); + state.cardRows = yield* order(state.cardRows, "card"); + const index = Object.freeze({}) as SearchIndex; + states.set(index, state); + completed = true; + return index; + } finally { + if (!completed) clear(state); + } +} +export function searchIndexCounts(index: SearchIndex): SearchCounts { + const state = stateOf(index), + notes = state.noteRows.length, + clips = state.clipRows.length, + cards = state.cardRows.length; + return Object.freeze({ notes, clips, cards, all: notes + clips + cards }); +} +function* projectPage( + index: SearchIndex, + kind: SearchKind, + offset: number, + limit: number, +): Generator { + if ( + !Number.isSafeInteger(offset) || + offset < 0 || + !Number.isSafeInteger(limit) || + limit < 1 || + limit > 50 + ) + throw new Error("Invalid search page."); + const state = stateOf(index), + rows = + kind === "note" + ? state.noteRows + : kind === "clip" + ? state.clipRows + : kind === "card" + ? state.cardRows + : null; + if (!rows) throw new Error("Invalid search kind."); + const result: SearchHit[] = []; + for (let i = offset; i < Math.min(offset + limit, rows.length); i++) { + stateOf(index); + const row = rows[i]!, + found = + kind === "note" + ? yield* matchNote(state, row.ordinal, true) + : kind === "clip" + ? yield* matchClip(state, row.ordinal, true) + : yield* matchCard(state, row.card!, true); + stateOf(index); + if (!found.hit) throw new Error("Invalid search projection."); + yield* work(32, 1, () => result.push(found.hit!)); + } + return Object.freeze(result); +} + +export function projectSearchPage( + index: SearchIndex, + kind: SearchKind, + offset: number, + limit: number, +): Generator { + const state = stateOf(index); + let iterator!: Generator; + iterator = (function* (): Generator { + try { + return yield* projectPage(index, kind, offset, limit); + } finally { + state.pages.delete(iterator); + } + })(); + state.pages.add(iterator); + return iterator; +} diff --git a/web/src/search/results.ts b/web/src/search/results.ts new file mode 100644 index 0000000..be362ef --- /dev/null +++ b/web/src/search/results.ts @@ -0,0 +1,111 @@ +import type { CardKey } from "../collections/contracts"; +import type { SearchInvalidation, SearchVersions } from "./contracts"; +import type { SearchExcerpt } from "./matching"; + +export type SearchKind = "note" | "clip" | "card"; +/** Numeric presentation intent only; it grants no source/cursor identity. */ +export type SearchPageIntent = Readonly>>; +export type SearchCounts = Readonly<{ + notes: number; + clips: number; + cards: number; + all: number; +}>; +export type SearchLocator = + | Readonly<{ + kind: "note"; + accountId: string; + libraryEpoch: string; + noteId: string; + }> + | Readonly<{ + kind: "clip"; + accountId: string; + collectionsEpoch: string; + clipId: string; + }> + | Readonly<{ + kind: "card"; + accountId: string; + collectionsEpoch: string; + cardId: string; + entryId: string; + key: CardKey; + }>; +export type SearchHit = Readonly<{ + locator: SearchLocator; + entityRevision: number; + primary: SearchExcerpt; + secondary: readonly SearchExcerpt[]; + // Source metadata only. These labels are never fabricated searchable fields. + archived?: boolean; + clipContent?: "saved-text" | "link-only"; + deckTitle?: string; +}>; +export type SearchProgress = Readonly<{ + phase: "indexing" | "matching" | "ordering" | "projecting"; + checkedEntities: number; + totalEntities: number; +}>; +declare const handleBrand: unique symbol; +declare const cursorBrand: unique symbol; +export type SearchHandle = Readonly<{ [handleBrand]: true }>; +export type SearchCursor = Readonly<{ [cursorBrand]: true }>; +export type SearchGroup = Readonly<{ + kind: SearchKind; + items: readonly SearchHit[]; + offset: number; + previous: SearchCursor | null; + next: SearchCursor | null; +}>; +export type SearchPageState = + | Readonly<{ status: "pending"; kind: SearchKind }> + | Readonly<{ status: "paused"; kind: SearchKind; handle: SearchHandle }> + | Readonly<{ status: "error"; kind: SearchKind; handle: SearchHandle }>; +export type SearchState = + | Readonly<{ status: "idle"; raw: "" }> + | Readonly<{ + status: "pending"; + raw: string; + progress: SearchProgress | null; + }> + | Readonly<{ + status: "paused-limit"; + raw: string; + handle: SearchHandle; + progress: SearchProgress | null; + }> + | Readonly<{ status: "changed"; raw: string }> + | Readonly<{ + status: "error"; + raw: string; + code: + | "query-too-long" + | "query-invalid" + | "query-too-many-terms" + | "source-unavailable"; + }> + | Readonly<{ + status: "ready"; + raw: string; + handle: SearchHandle; + versions: SearchVersions; + counts: SearchCounts; + groups: readonly SearchGroup[]; + page: SearchPageState | null; + }>; +export interface LocalSearchService { + getSnapshot(): SearchState; + subscribe(listener: () => void): () => void; + onInvalidate(listener: (reason: SearchInvalidation) => void): () => void; + query( + raw: string, + signal?: AbortSignal, + initialPages?: SearchPageIntent, + ): Promise; + continue(handle: SearchHandle, signal?: AbortSignal): Promise; + page(cursor: SearchCursor, signal?: AbortSignal): Promise; + cancelPage(): void; + cancel(): void; + close(): void; +} diff --git a/web/src/search/text.ts b/web/src/search/text.ts new file mode 100644 index 0000000..e2ea0eb --- /dev/null +++ b/web/src/search/text.ts @@ -0,0 +1,43 @@ +import { foldCodePoint } from "./unicodeCaseFolding"; + +export type SearchQueryErrorCode = + | "query-too-long" + | "query-invalid" + | "query-too-many-terms"; + +export class SearchQueryError extends Error { + constructor(public readonly code: SearchQueryErrorCode) { + super( + code === "query-too-long" + ? "Use at most 200 characters for a search." + : code === "query-too-many-terms" + ? "Use at most 12 different words for a search." + : "Finish entering the current character before searching.", + ); + this.name = "SearchQueryError"; + } +} + +export type SearchQuery = Readonly<{ + raw: string; + terms: readonly string[]; +}>; + +/** Query input is bounded before any transform. Stored text is not validated + * here: legacy lone UTF16 units remain literal source text. */ +export function compileSearchQuery(raw: string): SearchQuery | null { + if (raw.length > 200) throw new SearchQueryError("query-too-long"); + let folded = ""; + for (let offset = 0; offset < raw.length; ) { + const point = raw.codePointAt(offset)!; + if (point >= 0xd800 && point <= 0xdfff) + throw new SearchQueryError("query-invalid"); + folded += foldCodePoint(point); + offset += point > 0xffff ? 2 : 1; + } + const terms = [...new Set(folded.split(/\s+/u).filter(Boolean))]; + if (terms.length > 12) throw new SearchQueryError("query-too-many-terms"); + return terms.length === 0 + ? null + : Object.freeze({ raw, terms: Object.freeze(terms) }); +} diff --git a/web/src/search/unicode-provenance.json b/web/src/search/unicode-provenance.json new file mode 100644 index 0000000..bd58db0 --- /dev/null +++ b/web/src/search/unicode-provenance.json @@ -0,0 +1,13 @@ +{ + "version": "17.0.0", + "kind": "default full case folding, C+F; no Turkic override", + "source": "https://www.unicode.org/Public/17.0.0/ucd/CaseFolding.txt", + "sourceSha256": "ff8d8fefbf123574205085d6714c36149eb946d717a0c585c27f0f4ef58c4183", + "license": "Unicode-3.0", + "licenseSource": "https://www.unicode.org/license.txt", + "licenseSha256": "e7a93b009565cfce55919a381437ac4db883e9da2126fa28b91d12732bc53d96", + "notice": "public/licenses/Unicode-3.0.txt", + "generatedMappings": 1585, + "generatedSha256": "2e86da64e7bcc39af1e147d185fcbece97298367ddff23e4ded77e8d2aff3ba5", + "generator": "scripts/generate-search-folding.mjs" +} diff --git a/web/src/search/unicodeCaseFolding.ts b/web/src/search/unicodeCaseFolding.ts new file mode 100644 index 0000000..af68695 --- /dev/null +++ b/web/src/search/unicodeCaseFolding.ts @@ -0,0 +1,1600 @@ +/* Generated from Unicode CaseFolding-17.0.0, statuses C+F (default full folding). + * Copyright © 2025 Unicode, Inc. Unicode License V3; accompanying notice: + * public/licenses/Unicode-3.0.txt. Source https://www.unicode.org/Public/17.0.0/ucd/CaseFolding.txt + * SHA256 ff8d8fefbf123574205085d6714c36149eb946d717a0c585c27f0f4ef58c4183. Rebuild with scripts/generate-search-folding.mjs. + */ +export const UNICODE_CASE_FOLDING_VERSION = "17.0.0"; +export const MAX_CASE_FOLD_UTF16_UNITS = 3; +const mapping: Readonly> = Object.freeze({ + 0x41: "a", + 0x42: "b", + 0x43: "c", + 0x44: "d", + 0x45: "e", + 0x46: "f", + 0x47: "g", + 0x48: "h", + 0x49: "i", + 0x4a: "j", + 0x4b: "k", + 0x4c: "l", + 0x4d: "m", + 0x4e: "n", + 0x4f: "o", + 0x50: "p", + 0x51: "q", + 0x52: "r", + 0x53: "s", + 0x54: "t", + 0x55: "u", + 0x56: "v", + 0x57: "w", + 0x58: "x", + 0x59: "y", + 0x5a: "z", + 0xb5: "μ", + 0xc0: "à", + 0xc1: "á", + 0xc2: "â", + 0xc3: "ã", + 0xc4: "ä", + 0xc5: "å", + 0xc6: "æ", + 0xc7: "ç", + 0xc8: "è", + 0xc9: "é", + 0xca: "ê", + 0xcb: "ë", + 0xcc: "ì", + 0xcd: "í", + 0xce: "î", + 0xcf: "ï", + 0xd0: "ð", + 0xd1: "ñ", + 0xd2: "ò", + 0xd3: "ó", + 0xd4: "ô", + 0xd5: "õ", + 0xd6: "ö", + 0xd8: "ø", + 0xd9: "ù", + 0xda: "ú", + 0xdb: "û", + 0xdc: "ü", + 0xdd: "ý", + 0xde: "þ", + 0xdf: "ss", + 0x100: "ā", + 0x102: "ă", + 0x104: "ą", + 0x106: "ć", + 0x108: "ĉ", + 0x10a: "ċ", + 0x10c: "č", + 0x10e: "ď", + 0x110: "đ", + 0x112: "ē", + 0x114: "ĕ", + 0x116: "ė", + 0x118: "ę", + 0x11a: "ě", + 0x11c: "ĝ", + 0x11e: "ğ", + 0x120: "ġ", + 0x122: "ģ", + 0x124: "ĥ", + 0x126: "ħ", + 0x128: "ĩ", + 0x12a: "ī", + 0x12c: "ĭ", + 0x12e: "į", + 0x130: "i̇", + 0x132: "ij", + 0x134: "ĵ", + 0x136: "ķ", + 0x139: "ĺ", + 0x13b: "ļ", + 0x13d: "ľ", + 0x13f: "ŀ", + 0x141: "ł", + 0x143: "ń", + 0x145: "ņ", + 0x147: "ň", + 0x149: "ʼn", + 0x14a: "ŋ", + 0x14c: "ō", + 0x14e: "ŏ", + 0x150: "ő", + 0x152: "œ", + 0x154: "ŕ", + 0x156: "ŗ", + 0x158: "ř", + 0x15a: "ś", + 0x15c: "ŝ", + 0x15e: "ş", + 0x160: "š", + 0x162: "ţ", + 0x164: "ť", + 0x166: "ŧ", + 0x168: "ũ", + 0x16a: "ū", + 0x16c: "ŭ", + 0x16e: "ů", + 0x170: "ű", + 0x172: "ų", + 0x174: "ŵ", + 0x176: "ŷ", + 0x178: "ÿ", + 0x179: "ź", + 0x17b: "ż", + 0x17d: "ž", + 0x17f: "s", + 0x181: "ɓ", + 0x182: "ƃ", + 0x184: "ƅ", + 0x186: "ɔ", + 0x187: "ƈ", + 0x189: "ɖ", + 0x18a: "ɗ", + 0x18b: "ƌ", + 0x18e: "ǝ", + 0x18f: "ə", + 0x190: "ɛ", + 0x191: "ƒ", + 0x193: "ɠ", + 0x194: "ɣ", + 0x196: "ɩ", + 0x197: "ɨ", + 0x198: "ƙ", + 0x19c: "ɯ", + 0x19d: "ɲ", + 0x19f: "ɵ", + 0x1a0: "ơ", + 0x1a2: "ƣ", + 0x1a4: "ƥ", + 0x1a6: "ʀ", + 0x1a7: "ƨ", + 0x1a9: "ʃ", + 0x1ac: "ƭ", + 0x1ae: "ʈ", + 0x1af: "ư", + 0x1b1: "ʊ", + 0x1b2: "ʋ", + 0x1b3: "ƴ", + 0x1b5: "ƶ", + 0x1b7: "ʒ", + 0x1b8: "ƹ", + 0x1bc: "ƽ", + 0x1c4: "dž", + 0x1c5: "dž", + 0x1c7: "lj", + 0x1c8: "lj", + 0x1ca: "nj", + 0x1cb: "nj", + 0x1cd: "ǎ", + 0x1cf: "ǐ", + 0x1d1: "ǒ", + 0x1d3: "ǔ", + 0x1d5: "ǖ", + 0x1d7: "ǘ", + 0x1d9: "ǚ", + 0x1db: "ǜ", + 0x1de: "ǟ", + 0x1e0: "ǡ", + 0x1e2: "ǣ", + 0x1e4: "ǥ", + 0x1e6: "ǧ", + 0x1e8: "ǩ", + 0x1ea: "ǫ", + 0x1ec: "ǭ", + 0x1ee: "ǯ", + 0x1f0: "ǰ", + 0x1f1: "dz", + 0x1f2: "dz", + 0x1f4: "ǵ", + 0x1f6: "ƕ", + 0x1f7: "ƿ", + 0x1f8: "ǹ", + 0x1fa: "ǻ", + 0x1fc: "ǽ", + 0x1fe: "ǿ", + 0x200: "ȁ", + 0x202: "ȃ", + 0x204: "ȅ", + 0x206: "ȇ", + 0x208: "ȉ", + 0x20a: "ȋ", + 0x20c: "ȍ", + 0x20e: "ȏ", + 0x210: "ȑ", + 0x212: "ȓ", + 0x214: "ȕ", + 0x216: "ȗ", + 0x218: "ș", + 0x21a: "ț", + 0x21c: "ȝ", + 0x21e: "ȟ", + 0x220: "ƞ", + 0x222: "ȣ", + 0x224: "ȥ", + 0x226: "ȧ", + 0x228: "ȩ", + 0x22a: "ȫ", + 0x22c: "ȭ", + 0x22e: "ȯ", + 0x230: "ȱ", + 0x232: "ȳ", + 0x23a: "ⱥ", + 0x23b: "ȼ", + 0x23d: "ƚ", + 0x23e: "ⱦ", + 0x241: "ɂ", + 0x243: "ƀ", + 0x244: "ʉ", + 0x245: "ʌ", + 0x246: "ɇ", + 0x248: "ɉ", + 0x24a: "ɋ", + 0x24c: "ɍ", + 0x24e: "ɏ", + 0x345: "ι", + 0x370: "ͱ", + 0x372: "ͳ", + 0x376: "ͷ", + 0x37f: "ϳ", + 0x386: "ά", + 0x388: "έ", + 0x389: "ή", + 0x38a: "ί", + 0x38c: "ό", + 0x38e: "ύ", + 0x38f: "ώ", + 0x390: "ΐ", + 0x391: "α", + 0x392: "β", + 0x393: "γ", + 0x394: "δ", + 0x395: "ε", + 0x396: "ζ", + 0x397: "η", + 0x398: "θ", + 0x399: "ι", + 0x39a: "κ", + 0x39b: "λ", + 0x39c: "μ", + 0x39d: "ν", + 0x39e: "ξ", + 0x39f: "ο", + 0x3a0: "π", + 0x3a1: "ρ", + 0x3a3: "σ", + 0x3a4: "τ", + 0x3a5: "υ", + 0x3a6: "φ", + 0x3a7: "χ", + 0x3a8: "ψ", + 0x3a9: "ω", + 0x3aa: "ϊ", + 0x3ab: "ϋ", + 0x3b0: "ΰ", + 0x3c2: "σ", + 0x3cf: "ϗ", + 0x3d0: "β", + 0x3d1: "θ", + 0x3d5: "φ", + 0x3d6: "π", + 0x3d8: "ϙ", + 0x3da: "ϛ", + 0x3dc: "ϝ", + 0x3de: "ϟ", + 0x3e0: "ϡ", + 0x3e2: "ϣ", + 0x3e4: "ϥ", + 0x3e6: "ϧ", + 0x3e8: "ϩ", + 0x3ea: "ϫ", + 0x3ec: "ϭ", + 0x3ee: "ϯ", + 0x3f0: "κ", + 0x3f1: "ρ", + 0x3f4: "θ", + 0x3f5: "ε", + 0x3f7: "ϸ", + 0x3f9: "ϲ", + 0x3fa: "ϻ", + 0x3fd: "ͻ", + 0x3fe: "ͼ", + 0x3ff: "ͽ", + 0x400: "ѐ", + 0x401: "ё", + 0x402: "ђ", + 0x403: "ѓ", + 0x404: "є", + 0x405: "ѕ", + 0x406: "і", + 0x407: "ї", + 0x408: "ј", + 0x409: "љ", + 0x40a: "њ", + 0x40b: "ћ", + 0x40c: "ќ", + 0x40d: "ѝ", + 0x40e: "ў", + 0x40f: "џ", + 0x410: "а", + 0x411: "б", + 0x412: "в", + 0x413: "г", + 0x414: "д", + 0x415: "е", + 0x416: "ж", + 0x417: "з", + 0x418: "и", + 0x419: "й", + 0x41a: "к", + 0x41b: "л", + 0x41c: "м", + 0x41d: "н", + 0x41e: "о", + 0x41f: "п", + 0x420: "р", + 0x421: "с", + 0x422: "т", + 0x423: "у", + 0x424: "ф", + 0x425: "х", + 0x426: "ц", + 0x427: "ч", + 0x428: "ш", + 0x429: "щ", + 0x42a: "ъ", + 0x42b: "ы", + 0x42c: "ь", + 0x42d: "э", + 0x42e: "ю", + 0x42f: "я", + 0x460: "ѡ", + 0x462: "ѣ", + 0x464: "ѥ", + 0x466: "ѧ", + 0x468: "ѩ", + 0x46a: "ѫ", + 0x46c: "ѭ", + 0x46e: "ѯ", + 0x470: "ѱ", + 0x472: "ѳ", + 0x474: "ѵ", + 0x476: "ѷ", + 0x478: "ѹ", + 0x47a: "ѻ", + 0x47c: "ѽ", + 0x47e: "ѿ", + 0x480: "ҁ", + 0x48a: "ҋ", + 0x48c: "ҍ", + 0x48e: "ҏ", + 0x490: "ґ", + 0x492: "ғ", + 0x494: "ҕ", + 0x496: "җ", + 0x498: "ҙ", + 0x49a: "қ", + 0x49c: "ҝ", + 0x49e: "ҟ", + 0x4a0: "ҡ", + 0x4a2: "ң", + 0x4a4: "ҥ", + 0x4a6: "ҧ", + 0x4a8: "ҩ", + 0x4aa: "ҫ", + 0x4ac: "ҭ", + 0x4ae: "ү", + 0x4b0: "ұ", + 0x4b2: "ҳ", + 0x4b4: "ҵ", + 0x4b6: "ҷ", + 0x4b8: "ҹ", + 0x4ba: "һ", + 0x4bc: "ҽ", + 0x4be: "ҿ", + 0x4c0: "ӏ", + 0x4c1: "ӂ", + 0x4c3: "ӄ", + 0x4c5: "ӆ", + 0x4c7: "ӈ", + 0x4c9: "ӊ", + 0x4cb: "ӌ", + 0x4cd: "ӎ", + 0x4d0: "ӑ", + 0x4d2: "ӓ", + 0x4d4: "ӕ", + 0x4d6: "ӗ", + 0x4d8: "ә", + 0x4da: "ӛ", + 0x4dc: "ӝ", + 0x4de: "ӟ", + 0x4e0: "ӡ", + 0x4e2: "ӣ", + 0x4e4: "ӥ", + 0x4e6: "ӧ", + 0x4e8: "ө", + 0x4ea: "ӫ", + 0x4ec: "ӭ", + 0x4ee: "ӯ", + 0x4f0: "ӱ", + 0x4f2: "ӳ", + 0x4f4: "ӵ", + 0x4f6: "ӷ", + 0x4f8: "ӹ", + 0x4fa: "ӻ", + 0x4fc: "ӽ", + 0x4fe: "ӿ", + 0x500: "ԁ", + 0x502: "ԃ", + 0x504: "ԅ", + 0x506: "ԇ", + 0x508: "ԉ", + 0x50a: "ԋ", + 0x50c: "ԍ", + 0x50e: "ԏ", + 0x510: "ԑ", + 0x512: "ԓ", + 0x514: "ԕ", + 0x516: "ԗ", + 0x518: "ԙ", + 0x51a: "ԛ", + 0x51c: "ԝ", + 0x51e: "ԟ", + 0x520: "ԡ", + 0x522: "ԣ", + 0x524: "ԥ", + 0x526: "ԧ", + 0x528: "ԩ", + 0x52a: "ԫ", + 0x52c: "ԭ", + 0x52e: "ԯ", + 0x531: "ա", + 0x532: "բ", + 0x533: "գ", + 0x534: "դ", + 0x535: "ե", + 0x536: "զ", + 0x537: "է", + 0x538: "ը", + 0x539: "թ", + 0x53a: "ժ", + 0x53b: "ի", + 0x53c: "լ", + 0x53d: "խ", + 0x53e: "ծ", + 0x53f: "կ", + 0x540: "հ", + 0x541: "ձ", + 0x542: "ղ", + 0x543: "ճ", + 0x544: "մ", + 0x545: "յ", + 0x546: "ն", + 0x547: "շ", + 0x548: "ո", + 0x549: "չ", + 0x54a: "պ", + 0x54b: "ջ", + 0x54c: "ռ", + 0x54d: "ս", + 0x54e: "վ", + 0x54f: "տ", + 0x550: "ր", + 0x551: "ց", + 0x552: "ւ", + 0x553: "փ", + 0x554: "ք", + 0x555: "օ", + 0x556: "ֆ", + 0x587: "եւ", + 0x10a0: "ⴀ", + 0x10a1: "ⴁ", + 0x10a2: "ⴂ", + 0x10a3: "ⴃ", + 0x10a4: "ⴄ", + 0x10a5: "ⴅ", + 0x10a6: "ⴆ", + 0x10a7: "ⴇ", + 0x10a8: "ⴈ", + 0x10a9: "ⴉ", + 0x10aa: "ⴊ", + 0x10ab: "ⴋ", + 0x10ac: "ⴌ", + 0x10ad: "ⴍ", + 0x10ae: "ⴎ", + 0x10af: "ⴏ", + 0x10b0: "ⴐ", + 0x10b1: "ⴑ", + 0x10b2: "ⴒ", + 0x10b3: "ⴓ", + 0x10b4: "ⴔ", + 0x10b5: "ⴕ", + 0x10b6: "ⴖ", + 0x10b7: "ⴗ", + 0x10b8: "ⴘ", + 0x10b9: "ⴙ", + 0x10ba: "ⴚ", + 0x10bb: "ⴛ", + 0x10bc: "ⴜ", + 0x10bd: "ⴝ", + 0x10be: "ⴞ", + 0x10bf: "ⴟ", + 0x10c0: "ⴠ", + 0x10c1: "ⴡ", + 0x10c2: "ⴢ", + 0x10c3: "ⴣ", + 0x10c4: "ⴤ", + 0x10c5: "ⴥ", + 0x10c7: "ⴧ", + 0x10cd: "ⴭ", + 0x13f8: "Ᏸ", + 0x13f9: "Ᏹ", + 0x13fa: "Ᏺ", + 0x13fb: "Ᏻ", + 0x13fc: "Ᏼ", + 0x13fd: "Ᏽ", + 0x1c80: "в", + 0x1c81: "д", + 0x1c82: "о", + 0x1c83: "с", + 0x1c84: "т", + 0x1c85: "т", + 0x1c86: "ъ", + 0x1c87: "ѣ", + 0x1c88: "ꙋ", + 0x1c89: "ᲊ", + 0x1c90: "ა", + 0x1c91: "ბ", + 0x1c92: "გ", + 0x1c93: "დ", + 0x1c94: "ე", + 0x1c95: "ვ", + 0x1c96: "ზ", + 0x1c97: "თ", + 0x1c98: "ი", + 0x1c99: "კ", + 0x1c9a: "ლ", + 0x1c9b: "მ", + 0x1c9c: "ნ", + 0x1c9d: "ო", + 0x1c9e: "პ", + 0x1c9f: "ჟ", + 0x1ca0: "რ", + 0x1ca1: "ს", + 0x1ca2: "ტ", + 0x1ca3: "უ", + 0x1ca4: "ფ", + 0x1ca5: "ქ", + 0x1ca6: "ღ", + 0x1ca7: "ყ", + 0x1ca8: "შ", + 0x1ca9: "ჩ", + 0x1caa: "ც", + 0x1cab: "ძ", + 0x1cac: "წ", + 0x1cad: "ჭ", + 0x1cae: "ხ", + 0x1caf: "ჯ", + 0x1cb0: "ჰ", + 0x1cb1: "ჱ", + 0x1cb2: "ჲ", + 0x1cb3: "ჳ", + 0x1cb4: "ჴ", + 0x1cb5: "ჵ", + 0x1cb6: "ჶ", + 0x1cb7: "ჷ", + 0x1cb8: "ჸ", + 0x1cb9: "ჹ", + 0x1cba: "ჺ", + 0x1cbd: "ჽ", + 0x1cbe: "ჾ", + 0x1cbf: "ჿ", + 0x1e00: "ḁ", + 0x1e02: "ḃ", + 0x1e04: "ḅ", + 0x1e06: "ḇ", + 0x1e08: "ḉ", + 0x1e0a: "ḋ", + 0x1e0c: "ḍ", + 0x1e0e: "ḏ", + 0x1e10: "ḑ", + 0x1e12: "ḓ", + 0x1e14: "ḕ", + 0x1e16: "ḗ", + 0x1e18: "ḙ", + 0x1e1a: "ḛ", + 0x1e1c: "ḝ", + 0x1e1e: "ḟ", + 0x1e20: "ḡ", + 0x1e22: "ḣ", + 0x1e24: "ḥ", + 0x1e26: "ḧ", + 0x1e28: "ḩ", + 0x1e2a: "ḫ", + 0x1e2c: "ḭ", + 0x1e2e: "ḯ", + 0x1e30: "ḱ", + 0x1e32: "ḳ", + 0x1e34: "ḵ", + 0x1e36: "ḷ", + 0x1e38: "ḹ", + 0x1e3a: "ḻ", + 0x1e3c: "ḽ", + 0x1e3e: "ḿ", + 0x1e40: "ṁ", + 0x1e42: "ṃ", + 0x1e44: "ṅ", + 0x1e46: "ṇ", + 0x1e48: "ṉ", + 0x1e4a: "ṋ", + 0x1e4c: "ṍ", + 0x1e4e: "ṏ", + 0x1e50: "ṑ", + 0x1e52: "ṓ", + 0x1e54: "ṕ", + 0x1e56: "ṗ", + 0x1e58: "ṙ", + 0x1e5a: "ṛ", + 0x1e5c: "ṝ", + 0x1e5e: "ṟ", + 0x1e60: "ṡ", + 0x1e62: "ṣ", + 0x1e64: "ṥ", + 0x1e66: "ṧ", + 0x1e68: "ṩ", + 0x1e6a: "ṫ", + 0x1e6c: "ṭ", + 0x1e6e: "ṯ", + 0x1e70: "ṱ", + 0x1e72: "ṳ", + 0x1e74: "ṵ", + 0x1e76: "ṷ", + 0x1e78: "ṹ", + 0x1e7a: "ṻ", + 0x1e7c: "ṽ", + 0x1e7e: "ṿ", + 0x1e80: "ẁ", + 0x1e82: "ẃ", + 0x1e84: "ẅ", + 0x1e86: "ẇ", + 0x1e88: "ẉ", + 0x1e8a: "ẋ", + 0x1e8c: "ẍ", + 0x1e8e: "ẏ", + 0x1e90: "ẑ", + 0x1e92: "ẓ", + 0x1e94: "ẕ", + 0x1e96: "ẖ", + 0x1e97: "ẗ", + 0x1e98: "ẘ", + 0x1e99: "ẙ", + 0x1e9a: "aʾ", + 0x1e9b: "ṡ", + 0x1e9e: "ss", + 0x1ea0: "ạ", + 0x1ea2: "ả", + 0x1ea4: "ấ", + 0x1ea6: "ầ", + 0x1ea8: "ẩ", + 0x1eaa: "ẫ", + 0x1eac: "ậ", + 0x1eae: "ắ", + 0x1eb0: "ằ", + 0x1eb2: "ẳ", + 0x1eb4: "ẵ", + 0x1eb6: "ặ", + 0x1eb8: "ẹ", + 0x1eba: "ẻ", + 0x1ebc: "ẽ", + 0x1ebe: "ế", + 0x1ec0: "ề", + 0x1ec2: "ể", + 0x1ec4: "ễ", + 0x1ec6: "ệ", + 0x1ec8: "ỉ", + 0x1eca: "ị", + 0x1ecc: "ọ", + 0x1ece: "ỏ", + 0x1ed0: "ố", + 0x1ed2: "ồ", + 0x1ed4: "ổ", + 0x1ed6: "ỗ", + 0x1ed8: "ộ", + 0x1eda: "ớ", + 0x1edc: "ờ", + 0x1ede: "ở", + 0x1ee0: "ỡ", + 0x1ee2: "ợ", + 0x1ee4: "ụ", + 0x1ee6: "ủ", + 0x1ee8: "ứ", + 0x1eea: "ừ", + 0x1eec: "ử", + 0x1eee: "ữ", + 0x1ef0: "ự", + 0x1ef2: "ỳ", + 0x1ef4: "ỵ", + 0x1ef6: "ỷ", + 0x1ef8: "ỹ", + 0x1efa: "ỻ", + 0x1efc: "ỽ", + 0x1efe: "ỿ", + 0x1f08: "ἀ", + 0x1f09: "ἁ", + 0x1f0a: "ἂ", + 0x1f0b: "ἃ", + 0x1f0c: "ἄ", + 0x1f0d: "ἅ", + 0x1f0e: "ἆ", + 0x1f0f: "ἇ", + 0x1f18: "ἐ", + 0x1f19: "ἑ", + 0x1f1a: "ἒ", + 0x1f1b: "ἓ", + 0x1f1c: "ἔ", + 0x1f1d: "ἕ", + 0x1f28: "ἠ", + 0x1f29: "ἡ", + 0x1f2a: "ἢ", + 0x1f2b: "ἣ", + 0x1f2c: "ἤ", + 0x1f2d: "ἥ", + 0x1f2e: "ἦ", + 0x1f2f: "ἧ", + 0x1f38: "ἰ", + 0x1f39: "ἱ", + 0x1f3a: "ἲ", + 0x1f3b: "ἳ", + 0x1f3c: "ἴ", + 0x1f3d: "ἵ", + 0x1f3e: "ἶ", + 0x1f3f: "ἷ", + 0x1f48: "ὀ", + 0x1f49: "ὁ", + 0x1f4a: "ὂ", + 0x1f4b: "ὃ", + 0x1f4c: "ὄ", + 0x1f4d: "ὅ", + 0x1f50: "ὐ", + 0x1f52: "ὒ", + 0x1f54: "ὔ", + 0x1f56: "ὖ", + 0x1f59: "ὑ", + 0x1f5b: "ὓ", + 0x1f5d: "ὕ", + 0x1f5f: "ὗ", + 0x1f68: "ὠ", + 0x1f69: "ὡ", + 0x1f6a: "ὢ", + 0x1f6b: "ὣ", + 0x1f6c: "ὤ", + 0x1f6d: "ὥ", + 0x1f6e: "ὦ", + 0x1f6f: "ὧ", + 0x1f80: "ἀι", + 0x1f81: "ἁι", + 0x1f82: "ἂι", + 0x1f83: "ἃι", + 0x1f84: "ἄι", + 0x1f85: "ἅι", + 0x1f86: "ἆι", + 0x1f87: "ἇι", + 0x1f88: "ἀι", + 0x1f89: "ἁι", + 0x1f8a: "ἂι", + 0x1f8b: "ἃι", + 0x1f8c: "ἄι", + 0x1f8d: "ἅι", + 0x1f8e: "ἆι", + 0x1f8f: "ἇι", + 0x1f90: "ἠι", + 0x1f91: "ἡι", + 0x1f92: "ἢι", + 0x1f93: "ἣι", + 0x1f94: "ἤι", + 0x1f95: "ἥι", + 0x1f96: "ἦι", + 0x1f97: "ἧι", + 0x1f98: "ἠι", + 0x1f99: "ἡι", + 0x1f9a: "ἢι", + 0x1f9b: "ἣι", + 0x1f9c: "ἤι", + 0x1f9d: "ἥι", + 0x1f9e: "ἦι", + 0x1f9f: "ἧι", + 0x1fa0: "ὠι", + 0x1fa1: "ὡι", + 0x1fa2: "ὢι", + 0x1fa3: "ὣι", + 0x1fa4: "ὤι", + 0x1fa5: "ὥι", + 0x1fa6: "ὦι", + 0x1fa7: "ὧι", + 0x1fa8: "ὠι", + 0x1fa9: "ὡι", + 0x1faa: "ὢι", + 0x1fab: "ὣι", + 0x1fac: "ὤι", + 0x1fad: "ὥι", + 0x1fae: "ὦι", + 0x1faf: "ὧι", + 0x1fb2: "ὰι", + 0x1fb3: "αι", + 0x1fb4: "άι", + 0x1fb6: "ᾶ", + 0x1fb7: "ᾶι", + 0x1fb8: "ᾰ", + 0x1fb9: "ᾱ", + 0x1fba: "ὰ", + 0x1fbb: "ά", + 0x1fbc: "αι", + 0x1fbe: "ι", + 0x1fc2: "ὴι", + 0x1fc3: "ηι", + 0x1fc4: "ήι", + 0x1fc6: "ῆ", + 0x1fc7: "ῆι", + 0x1fc8: "ὲ", + 0x1fc9: "έ", + 0x1fca: "ὴ", + 0x1fcb: "ή", + 0x1fcc: "ηι", + 0x1fd2: "ῒ", + 0x1fd3: "ΐ", + 0x1fd6: "ῖ", + 0x1fd7: "ῗ", + 0x1fd8: "ῐ", + 0x1fd9: "ῑ", + 0x1fda: "ὶ", + 0x1fdb: "ί", + 0x1fe2: "ῢ", + 0x1fe3: "ΰ", + 0x1fe4: "ῤ", + 0x1fe6: "ῦ", + 0x1fe7: "ῧ", + 0x1fe8: "ῠ", + 0x1fe9: "ῡ", + 0x1fea: "ὺ", + 0x1feb: "ύ", + 0x1fec: "ῥ", + 0x1ff2: "ὼι", + 0x1ff3: "ωι", + 0x1ff4: "ώι", + 0x1ff6: "ῶ", + 0x1ff7: "ῶι", + 0x1ff8: "ὸ", + 0x1ff9: "ό", + 0x1ffa: "ὼ", + 0x1ffb: "ώ", + 0x1ffc: "ωι", + 0x2126: "ω", + 0x212a: "k", + 0x212b: "å", + 0x2132: "ⅎ", + 0x2160: "ⅰ", + 0x2161: "ⅱ", + 0x2162: "ⅲ", + 0x2163: "ⅳ", + 0x2164: "ⅴ", + 0x2165: "ⅵ", + 0x2166: "ⅶ", + 0x2167: "ⅷ", + 0x2168: "ⅸ", + 0x2169: "ⅹ", + 0x216a: "ⅺ", + 0x216b: "ⅻ", + 0x216c: "ⅼ", + 0x216d: "ⅽ", + 0x216e: "ⅾ", + 0x216f: "ⅿ", + 0x2183: "ↄ", + 0x24b6: "ⓐ", + 0x24b7: "ⓑ", + 0x24b8: "ⓒ", + 0x24b9: "ⓓ", + 0x24ba: "ⓔ", + 0x24bb: "ⓕ", + 0x24bc: "ⓖ", + 0x24bd: "ⓗ", + 0x24be: "ⓘ", + 0x24bf: "ⓙ", + 0x24c0: "ⓚ", + 0x24c1: "ⓛ", + 0x24c2: "ⓜ", + 0x24c3: "ⓝ", + 0x24c4: "ⓞ", + 0x24c5: "ⓟ", + 0x24c6: "ⓠ", + 0x24c7: "ⓡ", + 0x24c8: "ⓢ", + 0x24c9: "ⓣ", + 0x24ca: "ⓤ", + 0x24cb: "ⓥ", + 0x24cc: "ⓦ", + 0x24cd: "ⓧ", + 0x24ce: "ⓨ", + 0x24cf: "ⓩ", + 0x2c00: "ⰰ", + 0x2c01: "ⰱ", + 0x2c02: "ⰲ", + 0x2c03: "ⰳ", + 0x2c04: "ⰴ", + 0x2c05: "ⰵ", + 0x2c06: "ⰶ", + 0x2c07: "ⰷ", + 0x2c08: "ⰸ", + 0x2c09: "ⰹ", + 0x2c0a: "ⰺ", + 0x2c0b: "ⰻ", + 0x2c0c: "ⰼ", + 0x2c0d: "ⰽ", + 0x2c0e: "ⰾ", + 0x2c0f: "ⰿ", + 0x2c10: "ⱀ", + 0x2c11: "ⱁ", + 0x2c12: "ⱂ", + 0x2c13: "ⱃ", + 0x2c14: "ⱄ", + 0x2c15: "ⱅ", + 0x2c16: "ⱆ", + 0x2c17: "ⱇ", + 0x2c18: "ⱈ", + 0x2c19: "ⱉ", + 0x2c1a: "ⱊ", + 0x2c1b: "ⱋ", + 0x2c1c: "ⱌ", + 0x2c1d: "ⱍ", + 0x2c1e: "ⱎ", + 0x2c1f: "ⱏ", + 0x2c20: "ⱐ", + 0x2c21: "ⱑ", + 0x2c22: "ⱒ", + 0x2c23: "ⱓ", + 0x2c24: "ⱔ", + 0x2c25: "ⱕ", + 0x2c26: "ⱖ", + 0x2c27: "ⱗ", + 0x2c28: "ⱘ", + 0x2c29: "ⱙ", + 0x2c2a: "ⱚ", + 0x2c2b: "ⱛ", + 0x2c2c: "ⱜ", + 0x2c2d: "ⱝ", + 0x2c2e: "ⱞ", + 0x2c2f: "ⱟ", + 0x2c60: "ⱡ", + 0x2c62: "ɫ", + 0x2c63: "ᵽ", + 0x2c64: "ɽ", + 0x2c67: "ⱨ", + 0x2c69: "ⱪ", + 0x2c6b: "ⱬ", + 0x2c6d: "ɑ", + 0x2c6e: "ɱ", + 0x2c6f: "ɐ", + 0x2c70: "ɒ", + 0x2c72: "ⱳ", + 0x2c75: "ⱶ", + 0x2c7e: "ȿ", + 0x2c7f: "ɀ", + 0x2c80: "ⲁ", + 0x2c82: "ⲃ", + 0x2c84: "ⲅ", + 0x2c86: "ⲇ", + 0x2c88: "ⲉ", + 0x2c8a: "ⲋ", + 0x2c8c: "ⲍ", + 0x2c8e: "ⲏ", + 0x2c90: "ⲑ", + 0x2c92: "ⲓ", + 0x2c94: "ⲕ", + 0x2c96: "ⲗ", + 0x2c98: "ⲙ", + 0x2c9a: "ⲛ", + 0x2c9c: "ⲝ", + 0x2c9e: "ⲟ", + 0x2ca0: "ⲡ", + 0x2ca2: "ⲣ", + 0x2ca4: "ⲥ", + 0x2ca6: "ⲧ", + 0x2ca8: "ⲩ", + 0x2caa: "ⲫ", + 0x2cac: "ⲭ", + 0x2cae: "ⲯ", + 0x2cb0: "ⲱ", + 0x2cb2: "ⲳ", + 0x2cb4: "ⲵ", + 0x2cb6: "ⲷ", + 0x2cb8: "ⲹ", + 0x2cba: "ⲻ", + 0x2cbc: "ⲽ", + 0x2cbe: "ⲿ", + 0x2cc0: "ⳁ", + 0x2cc2: "ⳃ", + 0x2cc4: "ⳅ", + 0x2cc6: "ⳇ", + 0x2cc8: "ⳉ", + 0x2cca: "ⳋ", + 0x2ccc: "ⳍ", + 0x2cce: "ⳏ", + 0x2cd0: "ⳑ", + 0x2cd2: "ⳓ", + 0x2cd4: "ⳕ", + 0x2cd6: "ⳗ", + 0x2cd8: "ⳙ", + 0x2cda: "ⳛ", + 0x2cdc: "ⳝ", + 0x2cde: "ⳟ", + 0x2ce0: "ⳡ", + 0x2ce2: "ⳣ", + 0x2ceb: "ⳬ", + 0x2ced: "ⳮ", + 0x2cf2: "ⳳ", + 0xa640: "ꙁ", + 0xa642: "ꙃ", + 0xa644: "ꙅ", + 0xa646: "ꙇ", + 0xa648: "ꙉ", + 0xa64a: "ꙋ", + 0xa64c: "ꙍ", + 0xa64e: "ꙏ", + 0xa650: "ꙑ", + 0xa652: "ꙓ", + 0xa654: "ꙕ", + 0xa656: "ꙗ", + 0xa658: "ꙙ", + 0xa65a: "ꙛ", + 0xa65c: "ꙝ", + 0xa65e: "ꙟ", + 0xa660: "ꙡ", + 0xa662: "ꙣ", + 0xa664: "ꙥ", + 0xa666: "ꙧ", + 0xa668: "ꙩ", + 0xa66a: "ꙫ", + 0xa66c: "ꙭ", + 0xa680: "ꚁ", + 0xa682: "ꚃ", + 0xa684: "ꚅ", + 0xa686: "ꚇ", + 0xa688: "ꚉ", + 0xa68a: "ꚋ", + 0xa68c: "ꚍ", + 0xa68e: "ꚏ", + 0xa690: "ꚑ", + 0xa692: "ꚓ", + 0xa694: "ꚕ", + 0xa696: "ꚗ", + 0xa698: "ꚙ", + 0xa69a: "ꚛ", + 0xa722: "ꜣ", + 0xa724: "ꜥ", + 0xa726: "ꜧ", + 0xa728: "ꜩ", + 0xa72a: "ꜫ", + 0xa72c: "ꜭ", + 0xa72e: "ꜯ", + 0xa732: "ꜳ", + 0xa734: "ꜵ", + 0xa736: "ꜷ", + 0xa738: "ꜹ", + 0xa73a: "ꜻ", + 0xa73c: "ꜽ", + 0xa73e: "ꜿ", + 0xa740: "ꝁ", + 0xa742: "ꝃ", + 0xa744: "ꝅ", + 0xa746: "ꝇ", + 0xa748: "ꝉ", + 0xa74a: "ꝋ", + 0xa74c: "ꝍ", + 0xa74e: "ꝏ", + 0xa750: "ꝑ", + 0xa752: "ꝓ", + 0xa754: "ꝕ", + 0xa756: "ꝗ", + 0xa758: "ꝙ", + 0xa75a: "ꝛ", + 0xa75c: "ꝝ", + 0xa75e: "ꝟ", + 0xa760: "ꝡ", + 0xa762: "ꝣ", + 0xa764: "ꝥ", + 0xa766: "ꝧ", + 0xa768: "ꝩ", + 0xa76a: "ꝫ", + 0xa76c: "ꝭ", + 0xa76e: "ꝯ", + 0xa779: "ꝺ", + 0xa77b: "ꝼ", + 0xa77d: "ᵹ", + 0xa77e: "ꝿ", + 0xa780: "ꞁ", + 0xa782: "ꞃ", + 0xa784: "ꞅ", + 0xa786: "ꞇ", + 0xa78b: "ꞌ", + 0xa78d: "ɥ", + 0xa790: "ꞑ", + 0xa792: "ꞓ", + 0xa796: "ꞗ", + 0xa798: "ꞙ", + 0xa79a: "ꞛ", + 0xa79c: "ꞝ", + 0xa79e: "ꞟ", + 0xa7a0: "ꞡ", + 0xa7a2: "ꞣ", + 0xa7a4: "ꞥ", + 0xa7a6: "ꞧ", + 0xa7a8: "ꞩ", + 0xa7aa: "ɦ", + 0xa7ab: "ɜ", + 0xa7ac: "ɡ", + 0xa7ad: "ɬ", + 0xa7ae: "ɪ", + 0xa7b0: "ʞ", + 0xa7b1: "ʇ", + 0xa7b2: "ʝ", + 0xa7b3: "ꭓ", + 0xa7b4: "ꞵ", + 0xa7b6: "ꞷ", + 0xa7b8: "ꞹ", + 0xa7ba: "ꞻ", + 0xa7bc: "ꞽ", + 0xa7be: "ꞿ", + 0xa7c0: "ꟁ", + 0xa7c2: "ꟃ", + 0xa7c4: "ꞔ", + 0xa7c5: "ʂ", + 0xa7c6: "ᶎ", + 0xa7c7: "ꟈ", + 0xa7c9: "ꟊ", + 0xa7cb: "ɤ", + 0xa7cc: "ꟍ", + 0xa7ce: "꟏", + 0xa7d0: "ꟑ", + 0xa7d2: "ꟓ", + 0xa7d4: "ꟕ", + 0xa7d6: "ꟗ", + 0xa7d8: "ꟙ", + 0xa7da: "ꟛ", + 0xa7dc: "ƛ", + 0xa7f5: "ꟶ", + 0xab70: "Ꭰ", + 0xab71: "Ꭱ", + 0xab72: "Ꭲ", + 0xab73: "Ꭳ", + 0xab74: "Ꭴ", + 0xab75: "Ꭵ", + 0xab76: "Ꭶ", + 0xab77: "Ꭷ", + 0xab78: "Ꭸ", + 0xab79: "Ꭹ", + 0xab7a: "Ꭺ", + 0xab7b: "Ꭻ", + 0xab7c: "Ꭼ", + 0xab7d: "Ꭽ", + 0xab7e: "Ꭾ", + 0xab7f: "Ꭿ", + 0xab80: "Ꮀ", + 0xab81: "Ꮁ", + 0xab82: "Ꮂ", + 0xab83: "Ꮃ", + 0xab84: "Ꮄ", + 0xab85: "Ꮅ", + 0xab86: "Ꮆ", + 0xab87: "Ꮇ", + 0xab88: "Ꮈ", + 0xab89: "Ꮉ", + 0xab8a: "Ꮊ", + 0xab8b: "Ꮋ", + 0xab8c: "Ꮌ", + 0xab8d: "Ꮍ", + 0xab8e: "Ꮎ", + 0xab8f: "Ꮏ", + 0xab90: "Ꮐ", + 0xab91: "Ꮑ", + 0xab92: "Ꮒ", + 0xab93: "Ꮓ", + 0xab94: "Ꮔ", + 0xab95: "Ꮕ", + 0xab96: "Ꮖ", + 0xab97: "Ꮗ", + 0xab98: "Ꮘ", + 0xab99: "Ꮙ", + 0xab9a: "Ꮚ", + 0xab9b: "Ꮛ", + 0xab9c: "Ꮜ", + 0xab9d: "Ꮝ", + 0xab9e: "Ꮞ", + 0xab9f: "Ꮟ", + 0xaba0: "Ꮠ", + 0xaba1: "Ꮡ", + 0xaba2: "Ꮢ", + 0xaba3: "Ꮣ", + 0xaba4: "Ꮤ", + 0xaba5: "Ꮥ", + 0xaba6: "Ꮦ", + 0xaba7: "Ꮧ", + 0xaba8: "Ꮨ", + 0xaba9: "Ꮩ", + 0xabaa: "Ꮪ", + 0xabab: "Ꮫ", + 0xabac: "Ꮬ", + 0xabad: "Ꮭ", + 0xabae: "Ꮮ", + 0xabaf: "Ꮯ", + 0xabb0: "Ꮰ", + 0xabb1: "Ꮱ", + 0xabb2: "Ꮲ", + 0xabb3: "Ꮳ", + 0xabb4: "Ꮴ", + 0xabb5: "Ꮵ", + 0xabb6: "Ꮶ", + 0xabb7: "Ꮷ", + 0xabb8: "Ꮸ", + 0xabb9: "Ꮹ", + 0xabba: "Ꮺ", + 0xabbb: "Ꮻ", + 0xabbc: "Ꮼ", + 0xabbd: "Ꮽ", + 0xabbe: "Ꮾ", + 0xabbf: "Ꮿ", + 0xfb00: "ff", + 0xfb01: "fi", + 0xfb02: "fl", + 0xfb03: "ffi", + 0xfb04: "ffl", + 0xfb05: "st", + 0xfb06: "st", + 0xfb13: "մն", + 0xfb14: "մե", + 0xfb15: "մի", + 0xfb16: "վն", + 0xfb17: "մխ", + 0xff21: "ａ", + 0xff22: "ｂ", + 0xff23: "ｃ", + 0xff24: "ｄ", + 0xff25: "ｅ", + 0xff26: "ｆ", + 0xff27: "ｇ", + 0xff28: "ｈ", + 0xff29: "ｉ", + 0xff2a: "ｊ", + 0xff2b: "ｋ", + 0xff2c: "ｌ", + 0xff2d: "ｍ", + 0xff2e: "ｎ", + 0xff2f: "ｏ", + 0xff30: "ｐ", + 0xff31: "ｑ", + 0xff32: "ｒ", + 0xff33: "ｓ", + 0xff34: "ｔ", + 0xff35: "ｕ", + 0xff36: "ｖ", + 0xff37: "ｗ", + 0xff38: "ｘ", + 0xff39: "ｙ", + 0xff3a: "ｚ", + 0x10400: "𐐨", + 0x10401: "𐐩", + 0x10402: "𐐪", + 0x10403: "𐐫", + 0x10404: "𐐬", + 0x10405: "𐐭", + 0x10406: "𐐮", + 0x10407: "𐐯", + 0x10408: "𐐰", + 0x10409: "𐐱", + 0x1040a: "𐐲", + 0x1040b: "𐐳", + 0x1040c: "𐐴", + 0x1040d: "𐐵", + 0x1040e: "𐐶", + 0x1040f: "𐐷", + 0x10410: "𐐸", + 0x10411: "𐐹", + 0x10412: "𐐺", + 0x10413: "𐐻", + 0x10414: "𐐼", + 0x10415: "𐐽", + 0x10416: "𐐾", + 0x10417: "𐐿", + 0x10418: "𐑀", + 0x10419: "𐑁", + 0x1041a: "𐑂", + 0x1041b: "𐑃", + 0x1041c: "𐑄", + 0x1041d: "𐑅", + 0x1041e: "𐑆", + 0x1041f: "𐑇", + 0x10420: "𐑈", + 0x10421: "𐑉", + 0x10422: "𐑊", + 0x10423: "𐑋", + 0x10424: "𐑌", + 0x10425: "𐑍", + 0x10426: "𐑎", + 0x10427: "𐑏", + 0x104b0: "𐓘", + 0x104b1: "𐓙", + 0x104b2: "𐓚", + 0x104b3: "𐓛", + 0x104b4: "𐓜", + 0x104b5: "𐓝", + 0x104b6: "𐓞", + 0x104b7: "𐓟", + 0x104b8: "𐓠", + 0x104b9: "𐓡", + 0x104ba: "𐓢", + 0x104bb: "𐓣", + 0x104bc: "𐓤", + 0x104bd: "𐓥", + 0x104be: "𐓦", + 0x104bf: "𐓧", + 0x104c0: "𐓨", + 0x104c1: "𐓩", + 0x104c2: "𐓪", + 0x104c3: "𐓫", + 0x104c4: "𐓬", + 0x104c5: "𐓭", + 0x104c6: "𐓮", + 0x104c7: "𐓯", + 0x104c8: "𐓰", + 0x104c9: "𐓱", + 0x104ca: "𐓲", + 0x104cb: "𐓳", + 0x104cc: "𐓴", + 0x104cd: "𐓵", + 0x104ce: "𐓶", + 0x104cf: "𐓷", + 0x104d0: "𐓸", + 0x104d1: "𐓹", + 0x104d2: "𐓺", + 0x104d3: "𐓻", + 0x10570: "𐖗", + 0x10571: "𐖘", + 0x10572: "𐖙", + 0x10573: "𐖚", + 0x10574: "𐖛", + 0x10575: "𐖜", + 0x10576: "𐖝", + 0x10577: "𐖞", + 0x10578: "𐖟", + 0x10579: "𐖠", + 0x1057a: "𐖡", + 0x1057c: "𐖣", + 0x1057d: "𐖤", + 0x1057e: "𐖥", + 0x1057f: "𐖦", + 0x10580: "𐖧", + 0x10581: "𐖨", + 0x10582: "𐖩", + 0x10583: "𐖪", + 0x10584: "𐖫", + 0x10585: "𐖬", + 0x10586: "𐖭", + 0x10587: "𐖮", + 0x10588: "𐖯", + 0x10589: "𐖰", + 0x1058a: "𐖱", + 0x1058c: "𐖳", + 0x1058d: "𐖴", + 0x1058e: "𐖵", + 0x1058f: "𐖶", + 0x10590: "𐖷", + 0x10591: "𐖸", + 0x10592: "𐖹", + 0x10594: "𐖻", + 0x10595: "𐖼", + 0x10c80: "𐳀", + 0x10c81: "𐳁", + 0x10c82: "𐳂", + 0x10c83: "𐳃", + 0x10c84: "𐳄", + 0x10c85: "𐳅", + 0x10c86: "𐳆", + 0x10c87: "𐳇", + 0x10c88: "𐳈", + 0x10c89: "𐳉", + 0x10c8a: "𐳊", + 0x10c8b: "𐳋", + 0x10c8c: "𐳌", + 0x10c8d: "𐳍", + 0x10c8e: "𐳎", + 0x10c8f: "𐳏", + 0x10c90: "𐳐", + 0x10c91: "𐳑", + 0x10c92: "𐳒", + 0x10c93: "𐳓", + 0x10c94: "𐳔", + 0x10c95: "𐳕", + 0x10c96: "𐳖", + 0x10c97: "𐳗", + 0x10c98: "𐳘", + 0x10c99: "𐳙", + 0x10c9a: "𐳚", + 0x10c9b: "𐳛", + 0x10c9c: "𐳜", + 0x10c9d: "𐳝", + 0x10c9e: "𐳞", + 0x10c9f: "𐳟", + 0x10ca0: "𐳠", + 0x10ca1: "𐳡", + 0x10ca2: "𐳢", + 0x10ca3: "𐳣", + 0x10ca4: "𐳤", + 0x10ca5: "𐳥", + 0x10ca6: "𐳦", + 0x10ca7: "𐳧", + 0x10ca8: "𐳨", + 0x10ca9: "𐳩", + 0x10caa: "𐳪", + 0x10cab: "𐳫", + 0x10cac: "𐳬", + 0x10cad: "𐳭", + 0x10cae: "𐳮", + 0x10caf: "𐳯", + 0x10cb0: "𐳰", + 0x10cb1: "𐳱", + 0x10cb2: "𐳲", + 0x10d50: "𐵰", + 0x10d51: "𐵱", + 0x10d52: "𐵲", + 0x10d53: "𐵳", + 0x10d54: "𐵴", + 0x10d55: "𐵵", + 0x10d56: "𐵶", + 0x10d57: "𐵷", + 0x10d58: "𐵸", + 0x10d59: "𐵹", + 0x10d5a: "𐵺", + 0x10d5b: "𐵻", + 0x10d5c: "𐵼", + 0x10d5d: "𐵽", + 0x10d5e: "𐵾", + 0x10d5f: "𐵿", + 0x10d60: "𐶀", + 0x10d61: "𐶁", + 0x10d62: "𐶂", + 0x10d63: "𐶃", + 0x10d64: "𐶄", + 0x10d65: "𐶅", + 0x118a0: "𑣀", + 0x118a1: "𑣁", + 0x118a2: "𑣂", + 0x118a3: "𑣃", + 0x118a4: "𑣄", + 0x118a5: "𑣅", + 0x118a6: "𑣆", + 0x118a7: "𑣇", + 0x118a8: "𑣈", + 0x118a9: "𑣉", + 0x118aa: "𑣊", + 0x118ab: "𑣋", + 0x118ac: "𑣌", + 0x118ad: "𑣍", + 0x118ae: "𑣎", + 0x118af: "𑣏", + 0x118b0: "𑣐", + 0x118b1: "𑣑", + 0x118b2: "𑣒", + 0x118b3: "𑣓", + 0x118b4: "𑣔", + 0x118b5: "𑣕", + 0x118b6: "𑣖", + 0x118b7: "𑣗", + 0x118b8: "𑣘", + 0x118b9: "𑣙", + 0x118ba: "𑣚", + 0x118bb: "𑣛", + 0x118bc: "𑣜", + 0x118bd: "𑣝", + 0x118be: "𑣞", + 0x118bf: "𑣟", + 0x16e40: "𖹠", + 0x16e41: "𖹡", + 0x16e42: "𖹢", + 0x16e43: "𖹣", + 0x16e44: "𖹤", + 0x16e45: "𖹥", + 0x16e46: "𖹦", + 0x16e47: "𖹧", + 0x16e48: "𖹨", + 0x16e49: "𖹩", + 0x16e4a: "𖹪", + 0x16e4b: "𖹫", + 0x16e4c: "𖹬", + 0x16e4d: "𖹭", + 0x16e4e: "𖹮", + 0x16e4f: "𖹯", + 0x16e50: "𖹰", + 0x16e51: "𖹱", + 0x16e52: "𖹲", + 0x16e53: "𖹳", + 0x16e54: "𖹴", + 0x16e55: "𖹵", + 0x16e56: "𖹶", + 0x16e57: "𖹷", + 0x16e58: "𖹸", + 0x16e59: "𖹹", + 0x16e5a: "𖹺", + 0x16e5b: "𖹻", + 0x16e5c: "𖹼", + 0x16e5d: "𖹽", + 0x16e5e: "𖹾", + 0x16e5f: "𖹿", + 0x16ea0: "𖺻", + 0x16ea1: "𖺼", + 0x16ea2: "𖺽", + 0x16ea3: "𖺾", + 0x16ea4: "𖺿", + 0x16ea5: "𖻀", + 0x16ea6: "𖻁", + 0x16ea7: "𖻂", + 0x16ea8: "𖻃", + 0x16ea9: "𖻄", + 0x16eaa: "𖻅", + 0x16eab: "𖻆", + 0x16eac: "𖻇", + 0x16ead: "𖻈", + 0x16eae: "𖻉", + 0x16eaf: "𖻊", + 0x16eb0: "𖻋", + 0x16eb1: "𖻌", + 0x16eb2: "𖻍", + 0x16eb3: "𖻎", + 0x16eb4: "𖻏", + 0x16eb5: "𖻐", + 0x16eb6: "𖻑", + 0x16eb7: "𖻒", + 0x16eb8: "𖻓", + 0x1e900: "𞤢", + 0x1e901: "𞤣", + 0x1e902: "𞤤", + 0x1e903: "𞤥", + 0x1e904: "𞤦", + 0x1e905: "𞤧", + 0x1e906: "𞤨", + 0x1e907: "𞤩", + 0x1e908: "𞤪", + 0x1e909: "𞤫", + 0x1e90a: "𞤬", + 0x1e90b: "𞤭", + 0x1e90c: "𞤮", + 0x1e90d: "𞤯", + 0x1e90e: "𞤰", + 0x1e90f: "𞤱", + 0x1e910: "𞤲", + 0x1e911: "𞤳", + 0x1e912: "𞤴", + 0x1e913: "𞤵", + 0x1e914: "𞤶", + 0x1e915: "𞤷", + 0x1e916: "𞤸", + 0x1e917: "𞤹", + 0x1e918: "𞤺", + 0x1e919: "𞤻", + 0x1e91a: "𞤼", + 0x1e91b: "𞤽", + 0x1e91c: "𞤾", + 0x1e91d: "𞤿", + 0x1e91e: "𞥀", + 0x1e91f: "𞥁", + 0x1e920: "𞥂", + 0x1e921: "𞥃", +}); +/** One code point only. Unknown values, including stored lone UTF16 units, + * retain their exact identity. Query well-formedness is checked separately. */ +export function foldCodePoint(point: number): string { + if (!Number.isInteger(point) || point < 0 || point > 0x10ffff) throw new RangeError("Invalid code point."); + return mapping[point] ?? String.fromCodePoint(point); +} diff --git a/web/tests/data/search-text.test.ts b/web/tests/data/search-text.test.ts new file mode 100644 index 0000000..a639d07 --- /dev/null +++ b/web/tests/data/search-text.test.ts @@ -0,0 +1,80 @@ +import { describe, expect, it } from "vitest"; +import { compileSearchQuery, SearchQueryError } from "../../src/search/text"; +import { LiteralMatcher, splitSearchText } from "../../src/search/matching"; +import { SearchWork } from "../../src/search/work"; + +describe("Unicode literal query contract", () => { + it.each([ + ["Straße ẞ STRASSE", ["strasse", "ss"]], + ["Σ σ ς", ["σ"]], + ["I İ ı", ["i", "i\u0307", "ı"]], + ["CAFÉ cafe\u0301", ["café", "cafe\u0301"]], + ["\u{10400} \u{10428}", ["\u{10428}"]], + ["ffi FFI