From 26b9774095bbba1a406ed235337c33a697652541 Mon Sep 17 00:00:00 2001 From: Brandon McAnsh Date: Thu, 8 Oct 2026 09:52:13 -0400 Subject: [PATCH] feat(text-format): shared chat text-format parser Add :libs:text-format, a pure-Kotlin KMP module that parses chat markup (bold, italic, strike, code, code blocks, quotes, lists, masked links) into display text, styled spans and display ranges. Word-character classes come from a generated Unicode 16.0.0 table so the JVM and iOS classify identically. The module is exported from :kmp:shared-core. The parser reads the 122 vectors of text_format.json from one commonTest, which runs on the JVM and the iOS simulator target. --- .github/workflows/shared-core-tests.yml | 4 +- kmp/shared-core/build.gradle.kts | 2 + libs/text-format/build.gradle.kts | 43 + .../flipcash/libs/textformat/TextFormat.kt | 48 + .../libs/textformat/TextFormatParser.kt | 405 ++++ .../libs/textformat/TextFormatUnicode.kt | 43 + .../libs/textformat/TextFormatUnicodeTest.kt | 53 + .../libs/textformat/TextFormatVectorTest.kt | 97 + .../src/commonTest/resources/text_format.json | 1831 +++++++++++++++++ settings.gradle.kts | 2 + 10 files changed, 2527 insertions(+), 1 deletion(-) create mode 100644 libs/text-format/build.gradle.kts create mode 100644 libs/text-format/src/commonMain/kotlin/com/flipcash/libs/textformat/TextFormat.kt create mode 100644 libs/text-format/src/commonMain/kotlin/com/flipcash/libs/textformat/TextFormatParser.kt create mode 100644 libs/text-format/src/commonMain/kotlin/com/flipcash/libs/textformat/TextFormatUnicode.kt create mode 100644 libs/text-format/src/commonTest/kotlin/com/flipcash/libs/textformat/TextFormatUnicodeTest.kt create mode 100644 libs/text-format/src/commonTest/kotlin/com/flipcash/libs/textformat/TextFormatVectorTest.kt create mode 100644 libs/text-format/src/commonTest/resources/text_format.json diff --git a/.github/workflows/shared-core-tests.yml b/.github/workflows/shared-core-tests.yml index d1fbb7eb79..2a28273337 100644 --- a/.github/workflows/shared-core-tests.yml +++ b/.github/workflows/shared-core-tests.yml @@ -13,6 +13,7 @@ on: - 'libs/encryption/**' - 'libs/reporting/**' - 'libs/analytics-events/**' + - 'libs/text-format/**' - 'build-logic/convention/src/main/kotlin/AnalyticsCatalogueConventionPlugin.kt' - 'build-logic/convention/src/main/kotlin/com/getcode/buildlogic/analytics/**' - 'gradle/libs.versions.toml' @@ -78,7 +79,8 @@ jobs: :libs:encryption:mnemonic:iosSimulatorArm64Test \ :libs:encryption:sha256:iosSimulatorArm64Test \ :libs:encryption:sha512:iosSimulatorArm64Test \ - :libs:reporting:iosSimulatorArm64Test + :libs:reporting:iosSimulatorArm64Test \ + :libs:text-format:iosSimulatorArm64Test # The facade suite has to run against the Kotlin in this checkout, not against the last # published release, or a facade written for an unreleased Kotlin change is untested until diff --git a/kmp/shared-core/build.gradle.kts b/kmp/shared-core/build.gradle.kts index 5b274a4017..a9bfafc66e 100644 --- a/kmp/shared-core/build.gradle.kts +++ b/kmp/shared-core/build.gradle.kts @@ -39,6 +39,7 @@ kotlin { isStatic = true export(project(":libs:codes:kikcode")) export(project(":libs:encryption:base58")) + export(project(":libs:text-format")) export(project(":libs:encryption:sha256")) export(project(":libs:encryption:sha512")) export(project(":libs:encryption:hmac")) @@ -56,6 +57,7 @@ kotlin { dependencies { api(project(":libs:codes:kikcode")) api(project(":libs:encryption:base58")) + api(project(":libs:text-format")) api(project(":libs:encryption:sha256")) api(project(":libs:encryption:sha512")) api(project(":libs:encryption:hmac")) diff --git a/libs/text-format/build.gradle.kts b/libs/text-format/build.gradle.kts new file mode 100644 index 0000000000..16de7b7f18 --- /dev/null +++ b/libs/text-format/build.gradle.kts @@ -0,0 +1,43 @@ +plugins { + alias(libs.plugins.flipcash.kmp.library) + alias(libs.plugins.flipcash.kmp.test.fixtures) +} + +// Compiles `src/commonTest/resources` into a generated `TestFixtures.kt` on `commonTest`, readable +// from every target -- see the `flipcash.kmp.test.fixtures` convention plugin. +testFixtures { + packageName = "com.flipcash.libs.textformat" +} + +kotlin { + android { + namespace = "com.flipcash.libs.textformat" + compileSdk { + version = release(libs.versions.android.compileSdk.get().toInt()) { + minorApiLevel = libs.versions.android.compileSdkMinor.get().toInt() + } + } + minSdk = 29 + withHostTest {} + } + + iosArm64() + iosSimulatorArm64() + iosX64() + macosArm64() + macosX64() + + sourceSets { + commonMain { + // Pure Kotlin -- no external dependencies needed. + } + commonTest { + dependencies { + implementation(kotlin("test")) + implementation(libs.kotlinx.serialization.json) + // Hashes the copied fixture so a stale copy fails loudly. + implementation(libs.kotlincrypto.hash.sha2) + } + } + } +} diff --git a/libs/text-format/src/commonMain/kotlin/com/flipcash/libs/textformat/TextFormat.kt b/libs/text-format/src/commonMain/kotlin/com/flipcash/libs/textformat/TextFormat.kt new file mode 100644 index 0000000000..a6d87969c4 --- /dev/null +++ b/libs/text-format/src/commonMain/kotlin/com/flipcash/libs/textformat/TextFormat.kt @@ -0,0 +1,48 @@ +package com.flipcash.libs.textformat + +/** What a detected range is. The parser never detects these itself; callers pass them in. */ +enum class RangeKind { Link, Mention } + +/** + * A link or mention that detection found in the raw text. [start] and [end] are UTF-16 code unit + * offsets (Kotlin `String` indices), half-open. A marker inside a range is part of it and never + * opens or closes a style. + */ +data class ProtectedRange( + val start: Int, + val end: Int, + val kind: RangeKind, +) + +/** The styles the parser produces. */ +enum class FormatStyle { Bold, Italic, Strike, Code, CodeBlock, Quote, Bullet, Numbered } + +/** A [style] over [FormattedText.display], UTF-16 code unit offsets, half-open. */ +data class StyledSpan( + val start: Int, + val end: Int, + val style: FormatStyle, +) + +/** + * An input range moved to [FormattedText.display] offsets. A masked link `[text](url)` becomes a + * [RangeKind.Link] over `text` with the URL as [target]. A range without a [target] opens its own + * text. + */ +data class DisplayRange( + val start: Int, + val end: Int, + val kind: RangeKind, + val target: String? = null, +) + +/** + * The parser's result. [display] is the raw text minus the consumed markers. [spans] are + * normalized per style (touching or overlapping runs merged) and ordered by start, then longest + * first, then style name. [displayRanges] are ordered by start, end, kind, then target. + */ +data class FormattedText( + val display: String, + val spans: List, + val displayRanges: List, +) diff --git a/libs/text-format/src/commonMain/kotlin/com/flipcash/libs/textformat/TextFormatParser.kt b/libs/text-format/src/commonMain/kotlin/com/flipcash/libs/textformat/TextFormatParser.kt new file mode 100644 index 0000000000..65055203f1 --- /dev/null +++ b/libs/text-format/src/commonMain/kotlin/com/flipcash/libs/textformat/TextFormatParser.kt @@ -0,0 +1,405 @@ +package com.flipcash.libs.textformat + +/** + * Parses chat text markup (bold, italic, strike, code, code blocks, quotes, lists, masked links). + * + * [ranges] are the links and mentions that detection found in [text]; a marker inside one never + * opens or closes a style. Offsets in and out are UTF-16 code units. The rules are in + * `docs/superpowers/specs/2026-10-07-chat-text-formatting-design.md` in the orchestrator repo, and + * `text_format.json` is the authority. + */ +fun parseTextFormat(text: String, ranges: List): FormattedText = + TextFormatParser(text, ranges).parse() + +private val INLINE_MARKERS = charArrayOf('*', '_', '~') +private const val FENCE = "```" + +// A backslash before one of these is removed and the character after it is literal. +private const val ESCAPABLE = "\\*_~`>-.[]" + +private fun styleOf(marker: Int): FormatStyle = when (marker) { + '*'.code -> FormatStyle.Bold + '_'.code -> FormatStyle.Italic + else -> FormatStyle.Strike +} + +/** Works on code points, as the reference parser does; offsets convert at the edges. */ +private class TextFormatParser(private val text: String, inputRanges: List) { + // Code points of [text], and the UTF-16 offset where each starts (one extra entry at the end). + private val cps: IntArray + private val u16: IntArray + private val n: Int + + private val protected: BooleanArray + private val inRange: BooleanArray + private val removed: BooleanArray + private val escaped: BooleanArray + + private class RawRange(val kind: RangeKind, val start: Int, val end: Int) + private class Masked(val textStart: Int, val textEnd: Int, val urlStart: Int, val urlEnd: Int) + private class RawSpan(val style: FormatStyle, val start: Int, val end: Int) + + private val ranges: List + private val links: List + private val masked = ArrayList() + private val spans = ArrayList() + private val blocksRaw = ArrayList() + + init { + val points = ArrayList(text.length) + val starts = ArrayList(text.length + 1) + var i = 0 + while (i < text.length) { + val c = text[i] + starts.add(i) + if (c.isHighSurrogate() && i + 1 < text.length && text[i + 1].isLowSurrogate()) { + points.add(((c.code - 0xD800) shl 10) + (text[i + 1].code - 0xDC00) + 0x10000) + i += 2 + } else { + points.add(c.code) + i += 1 + } + } + starts.add(text.length) + cps = points.toIntArray() + u16 = starts.toIntArray() + n = cps.size + + // UTF-16 offset -> code point index. An offset inside a surrogate pair rounds down. + val toCp = IntArray(text.length + 1) + for (k in 0 until n) for (u in u16[k] until u16[k + 1]) toCp[u] = k + toCp[text.length] = n + ranges = inputRanges.map { + RawRange(it.kind, toCp[it.start.coerceIn(0, text.length)], toCp[it.end.coerceIn(0, text.length)]) + } + links = ranges.filter { it.kind == RangeKind.Link } + + protected = BooleanArray(n) + inRange = BooleanArray(n) + removed = BooleanArray(n) + escaped = BooleanArray(n) + for (r in ranges) for (k in r.start until r.end) { + protected[k] = true + inRange[k] = true + } + } + + // --- character classes (spec rule 2) --- + + private fun classOf(cp: Int) = TextFormatUnicode.classify(cp) + + private fun isWord(cp: Int): Boolean = when (classOf(cp)) { + CodePointClass.LETTER, CodePointClass.MARK, CodePointClass.DIGIT -> true + else -> false + } + + private fun isSpace(cp: Int): Boolean = + cp == '\t'.code || cp == '\r'.code || cp == '\n'.code || classOf(cp) == CodePointClass.SPACE + + // CommonMark whitespace; null (a line edge) counts too. + private fun isCmSpace(cp: Int?): Boolean = + cp == null || cp == '\t'.code || cp == '\n'.code || cp == 0x0C || cp == '\r'.code || + classOf(cp) == CodePointClass.SPACE + + private fun isCmPunct(cp: Int?): Boolean = cp != null && classOf(cp) == CodePointClass.PUNCT + + // --- lookups --- + + private fun markerAt(i: Int, m: Int): Boolean = i in 0 until n && cps[i] == m && !escaped[i] + + private fun inRun(i: Int): Boolean { + val m = cps[i] + return markerAt(i - 1, m) || markerAt(i + 1, m) + } + + /** The code point before [i], skipping combining marks back to their base. */ + private fun before(i: Int): Int? { + var j = i - 1 + while (j >= 0 && classOf(cps[j]) == CodePointClass.MARK) j-- + return if (j >= 0) cps[j] else null + } + + private fun after(i: Int): Int? = if (i + 1 < n) cps[i + 1] else null + + private fun canOpen(i: Int): Boolean { + if (protected[i] || inRun(i)) return false + if (cps[i] == '_'.code) { + val prev = before(i) + val nxt = after(i) + val (left, right) = flanking(prev, nxt) + return left && (!right || isCmPunct(prev)) + } + val prev = before(i) + val nxt = after(i) + if (prev != null && isWord(prev)) return false + return nxt != null && !isSpace(nxt) + } + + private fun canClose(i: Int): Boolean { + if (protected[i] || inRun(i)) return false + val prev = before(i) + val nxt = after(i) + if (cps[i] == '_'.code) { + val (left, right) = flanking(prev, nxt) + return right && (!left || isCmPunct(nxt)) + } + if (nxt != null && isWord(nxt)) return false + return prev != null && !isSpace(prev) + } + + /** CommonMark 0.31.2 left- and right-flanking, for `_`. */ + private fun flanking(prev: Int?, nxt: Int?): Pair { + val left = !isCmSpace(nxt) && (!isCmPunct(nxt) || isCmSpace(prev) || isCmPunct(prev)) + val right = !isCmSpace(prev) && (!isCmPunct(prev) || isCmSpace(nxt) || isCmPunct(nxt)) + return left to right + } + + private fun startsWithFence(i: Int): Boolean = + i + 3 <= n && cps[i] == '`'.code && cps[i + 1] == '`'.code && cps[i + 2] == '`'.code + + private fun fenceAt(i: Int): Boolean = + startsWithFence(i) && !(i > 0 && cps[i - 1] == '\\'.code) && + !(protected[i] || protected[i + 1] || protected[i + 2]) + + // --- rule 3: code blocks, before anything else --- + + private fun codeBlocks() { + var i = 0 + while (i < n) { + val lineStart = i == 0 || cps[i - 1] == '\n'.code + if (lineStart && fenceAt(i)) { + var close = -1 + for (k in i + 3 until n - 2) { + val lineEnd = k + 3 == n || cps[k + 3] == '\n'.code + if (lineEnd && fenceAt(k)) { + close = k + break + } + } + if (close >= 0) { + var a = i + 3 + var b = close + if (a < b && cps[a] == '\n'.code) a++ + if (a < b && cps[b - 1] == '\n'.code) b-- + if (a < b) { + for (k in i until a) removed[k] = true + for (k in b until close + 3) removed[k] = true + spans.add(RawSpan(FormatStyle.CodeBlock, a, b)) + for (k in i until close + 3) protected[k] = true + blocksRaw.add(intArrayOf(i, close + 3)) + i = close + 3 + continue + } + } + } + i++ + } + } + + // --- rule 4: quote, then list, at the start of a line --- + + private fun blocks(a: Int, b: Int) { + var c = a + if (b - a >= 3 && cps[a] == '>'.code && cps[a + 1] == ' '.code && !protected[a]) { + removed[a] = true + removed[a + 1] = true + spans.add(RawSpan(FormatStyle.Quote, a + 2, b)) + c = a + 2 + } + if (c < n && protected[c]) return + if (b - c >= 3 && (cps[c] == '-'.code || cps[c] == '*'.code) && cps[c + 1] == ' '.code) { + spans.add(RawSpan(FormatStyle.Bullet, c, b)) + return + } + var d = c + while (d < b && d - c < 3 && cps[d] >= '0'.code && cps[d] <= '9'.code) d++ + if (c < d && b - d >= 3 && cps[d] == '.'.code && cps[d + 1] == ' '.code) { + spans.add(RawSpan(FormatStyle.Numbered, c, b)) + } + } + + // --- rule 5: escapes and inline code in one left-to-right pass --- + + private fun escapesAndCode(a: Int, b: Int) { + var i = a + while (i < b) { + if (cps[i] == '\\'.code && !protected[i] && i + 1 < b && + cps[i + 1] < 0x80 && ESCAPABLE.indexOf(cps[i + 1].toChar()) >= 0 && !protected[i + 1] + ) { + removed[i] = true + escaped[i + 1] = true + protected[i + 1] = true + i += 2 + continue + } + if (cps[i] == '`'.code && canOpen(i)) { + var j = -1 + for (k in i + 2 until b) { + if (cps[k] == '`'.code && canClose(k)) { + j = k + break + } + } + if (j >= 0) { + spans.add(RawSpan(FormatStyle.Code, i + 1, j)) + removed[i] = true + removed[j] = true + for (k in i..j) protected[k] = true + i = j + 1 + continue + } + } + i++ + } + } + + private fun styles(a: Int, b: Int, allowed: Set) { + var i = a + while (i < b) { + val m = cps[i] + if (m in allowed && canOpen(i)) { + var j = -1 + for (k in i + 2 until b) { + if (cps[k] == m && canClose(k)) { + j = k + break + } + } + if (j >= 0) { + spans.add(RawSpan(styleOf(m), i + 1, j)) + removed[i] = true + removed[j] = true + styles(i + 1, j, allowed - m) + i = j + 1 + continue + } + } + i++ + } + } + + // --- rule 6: masked links, after escapes and code, before the other styles --- + + private fun maskedLinks(a: Int, b: Int) { + var i = a + while (i < b) { + if (cps[i] == '['.code && !protected[i]) { + var j = -1 + for (k in i + 1 until b) { + if (cps[k] == ']'.code && !protected[k]) { + j = k + break + } + } + var url: RawRange? = null + if (j >= 0 && j > i + 1 && j + 1 < b && cps[j + 1] == '('.code) { + url = links.firstOrNull { it.start == j + 2 } + } + if (url != null && url.end < b && cps[url.end] == ')'.code && + (i + 1 until j).none { inRange[it] } + ) { + val close = url.end + removed[i] = true + removed[j] = true + removed[j + 1] = true + removed[close] = true + for (k in url.start until url.end) removed[k] = true + protected[i] = true + protected[j] = true + protected[j + 1] = true + protected[close] = true + masked.add(Masked(i + 1, j, url.start, url.end)) + i = close + 1 + continue + } + } + i++ + } + } + + // --- driver --- + + fun parse(): FormattedText { + codeBlocks() + val allowed = INLINE_MARKERS.map { it.code }.toSet() + var a = 0 + while (a <= n) { + var b = a + while (b < n && cps[b] != '\n'.code) b++ + // A code block starts at a line start and ends at a line end, so a line is either + // wholly inside one or wholly outside. + if (blocksRaw.none { it[0] <= a && b <= it[1] }) { + blocks(a, b) + escapesAndCode(a, b) + maskedLinks(a, b) + styles(a, b, allowed) + } + a = b + 1 + } + return output() + } + + private fun output(): FormattedText { + // raw code point index -> UTF-16 offset in display + val displayOffset = IntArray(n + 1) + val display = StringBuilder(text.length) + for (k in 0 until n) { + displayOffset[k] = display.length + if (!removed[k]) display.appendRange(text, u16[k], u16[k + 1]) + } + displayOffset[n] = display.length + + val styled = spans.map { StyledSpan(displayOffset[it.start], displayOffset[it.end], it.style) } + val consumed = masked.map { it.urlStart to it.urlEnd }.toSet() + val out = ArrayList() + for (r in ranges) { + if ((r.start to r.end) in consumed) continue + out.add(DisplayRange(displayOffset[r.start], displayOffset[r.end], r.kind)) + } + for (m in masked) { + val target = text.substring(u16[m.urlStart], u16[m.urlEnd]) + out.add(DisplayRange(displayOffset[m.textStart], displayOffset[m.textEnd], RangeKind.Link, target)) + } + out.sortWith( + compareBy({ it.start }, { it.end }, { kindName(it.kind) }, { it.target ?: "" }), + ) + return FormattedText(display.toString(), normalize(styled), out) + } + + // Matches the fixture's lowercase kind names, so ordering agrees with the reference. + private fun kindName(kind: RangeKind): String = if (kind == RangeKind.Link) "link" else "mention" + + /** Per style, merge spans that touch or overlap; order by start, longest first, then style. */ + private fun normalize(input: List): List { + val merged = ArrayList() + for ((style, group) in input.filter { it.start < it.end }.groupBy { it.style }) { + val sorted = group.sortedBy { it.start } + var curA = sorted[0].start + var curB = sorted[0].end + for (s in sorted.drop(1)) { + if (s.start <= curB) { + curB = maxOf(curB, s.end) + } else { + merged.add(StyledSpan(curA, curB, style)) + curA = s.start + curB = s.end + } + } + merged.add(StyledSpan(curA, curB, style)) + } + return merged.sortedWith( + compareBy({ it.start }, { -it.end }, { styleName(it.style) }), + ) + } + + private fun styleName(style: FormatStyle): String = when (style) { + FormatStyle.Bold -> "bold" + FormatStyle.Italic -> "italic" + FormatStyle.Strike -> "strike" + FormatStyle.Code -> "code" + FormatStyle.CodeBlock -> "codeBlock" + FormatStyle.Quote -> "quote" + FormatStyle.Bullet -> "bullet" + FormatStyle.Numbered -> "numbered" + } +} diff --git a/libs/text-format/src/commonMain/kotlin/com/flipcash/libs/textformat/TextFormatUnicode.kt b/libs/text-format/src/commonMain/kotlin/com/flipcash/libs/textformat/TextFormatUnicode.kt new file mode 100644 index 0000000000..aa2bc84a9e --- /dev/null +++ b/libs/text-format/src/commonMain/kotlin/com/flipcash/libs/textformat/TextFormatUnicode.kt @@ -0,0 +1,43 @@ +// Generated by code/test-vectors/gen_text_format_unicode.py. Do not edit. +// Unicode 16.0.0. Regenerate with: +// python3 gen_text_format_unicode.py table > TextFormatUnicode.kt +package com.flipcash.libs.textformat + +internal enum class CodePointClass { OTHER, LETTER, MARK, DIGIT, PUNCT, SPACE } + +internal object TextFormatUnicode { + const val UNICODE_VERSION: String = "16.0.0" + + // Each run is 7 hex digits: a 6-digit start code point, then a class + // ordinal. A run lasts until the next run's start. + private val packed: Array = arrayOf( + "000000000002050000214000030300003a4000041100005b4000061100007b400007f00000a050000a140000aa10000ab40000ad00000ae40000b200000b440000b510000b640000b900000ba10000bb40000bc00000bf40000c010000d740000d810000f740000f810002c240002c610002d240002e010002e540002ec10002ed40002ee10002ef40003002000370100037540003761000378000037a100037e400037f10003800000384400038610003874000388100038b000038c100038d000038e10003a200003a310003f640003f710004824000483200048a1000530000053110005570000559100055a40005601000589400058b000058d4000590000059120005be40005bf20005c040005c120005c340005c420005c640005c720005c800005d010005eb00005ef10005f340005f500006064000610200061b400061c000061d4000620100064b2000660300066a400066e1000670200067110006d440006d510006d620006dd00006de40006df20006e510006e720006e940006ea20006ee10006f030006fa10006fd40006ff1000700400070e0000710100071120007121000730200074b000074d10007a620007b110007b200007c030007ca10007eb20007f410007f640007fa10007fb00007fd20007fe40008001000816200081a100081b2000824100082520008281000829200082e0000830400083f00008401000859200085c000085e400085f0000860100086b000087010008884000889100088f000089720008a010008ca20008e200008e32000904100093a200093d100093e20009501000951200095810009622000964400096630009704000971100098120009840000985100098d000098f1000991000099310009a900009aa10009b100009b210009b300009b610009ba00009bc20009bd10009be20009c500009c720009c900009cb20009ce10009cf00009d720009d800009dc10009de00009df10009e220009e400009e630009f010009f240009f400009fa40009fc10009fd40009fe20009ff0000a012000a040000a051000a0b0000a0f1000a110000a131000a290000a2a1000a310000a321000a340000a351000a370000a381000a3a0000a3c2000a3d0000a3e2000a430000a472000a490000a4b2000a4e0000a512000a520000a591000a5d0000a5e1000a5f0000a663000a702000a721000a752000a764000a770000a812000a840000a851000a8e0000a8f1000a920000a931000aa90000aaa1000ab10000ab21000ab40000ab51000aba0000abc2000abd1000abe2000ac60000ac72000aca0000acb2000ace0000ad01000ad10000ae01000ae22000ae40000ae63000af04000af20000af91000afa2000b000000b012000b040000b051000b0d0000b0f1000b110000b131000b290000b2a1000b310000b321000b340000b351000b3a0000b3c2000b3d1000b3e2000b450000b472000b490000b4b2000b4e0000b552000b580000b5c1000b5e0000b5f1000b622000b640000b663000b704000b711000b720000b822000b831000b840000b851000b8b0000b8e1000b910000b921000b960000b991000b9b0000b9c1000b9d0000b9e1000ba00000ba31000ba50000ba81000bab0000bae1000bba0000bbe2000bc30000bc62000bc90000bca2000bce0000bd01000bd10000bd72000bd80000be63000bf00000bf34000bfb0000c002000c051000c0d0000c0e1000c110000c121000c290000c2a1000c3a0000c3c2000c3d1000c3e2000c450000c462000c490000c4a2000c4e0000c552000c570000c581000c5b0000c5d1000c5e0000c601000c622000c640000c663000c700000c774000c780000c7f4000c801000c812000c844000c851000c8d0000c8e1000c910000c921000ca90000caa1000cb40000cb51000cba0000cbc2000cbd1000cbe2000cc50000cc62000cc90000cca2000cce0000cd52000cd70000cdd1000cdf0000ce01000ce22000ce40000ce63000cf00000cf11000cf32000cf40000d002000d041000d0d0000d0e1000d110000d121000d3b2000d3d1000d3e2000d450000d462000d490000d4a2000d4e1000d4f4000d500000d541000d572000d580000d5f1000d622000d640000d663000d700000d794000d7a1000d800000d812000d840000d851000d970000d9a1000db20000db31000dbc0000dbd1000dbe0000dc01000dc70000dca2000dcb0000dcf2000dd50000dd62000dd70000dd82000de00000de63000df00000df22000df44000df50000e011000e312000e321000e342000e3b0000e3f4000e401000e472000e4f4000e503000e5a4000e5c0000e811000e830000e841000e850000e861000e8b0000e8c1000ea40000ea51000ea60000ea71000eb12000eb21000eb42000ebd1000ebe0000ec01000ec50000ec61000ec70000ec82000ecf0000ed03000eda0000edc1000ee00000f001000f014000f182000f1a4000f203000f2a0000f344000f352000f364000f372000f384000f392000f3a4000f3e2000f401000f480000f491000f6d0000f712000f854000f862000f881000f8d2000f980000f992000fbd0000fbe4000fc62000fc74000fcd0000fce4000fdb0001000100102b200103f1001040300104a40010501001056200105a100105e2001061100106220010651001067200106e100107120010751001082200108e100108f2001090300109a200109e40010a010010c600010c710010c800010cd10010ce00010d010010fb40010fc1001249000124a100124e0001250100125700012581001259000125a100125e00012601001289000128a100128e000129010012b100012b210012b600012b810012bf00012c010012c100012c210012c600012c810012d700012d81001311000131210013160001318100135b000135d2001360400136900013801001390400139a00013a010013f600013f810013fe00014004001401100166d400166f10016805001681100169b400169d00016a010016eb40016ee00016f110016f9000170010017122001716000171f1001732200173540017370001740100175220017540001760100176d000176e100177100017722001774000178010017b420017d440017d710017d840017dc10017dd20017de00017e030017ea0001800400180b200180e000180f2001810300181a0001820100187900018801001885200188710018a920018aa10018ab00018b010018f60001900100191f0001920200192c0001930200193c00019404001941000194440019463001950100196e00019701001975000198010019ac00019b010019ca00019d030019da00019de4001a001001a172001a1c0001a1e4001a201001a552001a5f0001a602001a7d0001a7f2001a803001a8a0001a903001a9a0001aa04001aa71001aa84001aae0001ab02001acf0001b002001b051001b342001b451001b4d0001b4e4001b503001b5a4001b6b2001b744001b802001b831001ba12001bae1001bb03001bba1001be62001bf40001bfc4001c001001c242001c380001c3b4001c403001c4a0001c4d1001c503001c5a1001c7e4001c801001c8b0001c901001cbb0001cbd1001cc04001cc80001cd02001cd34001cd42001ce91001ced2001cee1001cf42001cf51001cf72001cfa1001cfb0001d001001dc02001e001001f160001f181001f1e0001f201001f460001f481001f4e0001f501001f580001f591001f5a0001f5b1001f5c0001f5d1001f5e0001f5f1001f7e0001f801001fb50001fb61001fbd4001fbe1001fbf4001fc21001fc50001fc61001fcd4001fd01001fd40001fd61001fdc0001fdd4001fe01001fed4001ff00001ff21001ff50001ff61001ffd4001fff0002000500200b00020104002028000202f5002030400205f500206000020711002072000207a400207f1002080000208a400208f0002090100209d00020a040020c100020d020020f100021004002102100210340021071002108400210a1002114400211510021164002119100211e400212410021254002126100212740021281002129400212a100212e400212f100213a400213c10021404002145100214a400214e100214f400215000021831002185000218a400218c0002190400242a0002440400244b000249c40024ea0002500400277600027944002b740002b764002b960002b974002c001002ce54002ceb1002cef2002cf21002cf40002cf94002cfd0002cfe4002d001002d260002d271002d280002d2d1002d2e0002d301002d680002d6f1002d704002d710002d7f2002d801002d970002da01002da70002da81002daf0002db01002db70002db81002dbf0002dc01002dc70002dc81002dcf0002dd01002dd70002dd81002ddf0002de02002e004002e2f1002e304002e5e0002e804002e9a0002e9b4002ef40002f004002fd60002ff0400300050030014003005100300700030084003021000302a2003030400303110030364003038000303b100303d4003040000304110030970003099200309b400309d10030a040030a110030fb40030fc1003100000310510031300003131100318f00031904003192000319640031a010031c040031e600031ef40031f01003200400321f000322a40032480003250400325100032604003280000328a40032b100032c040034001004dc04004e00100a48d000a490400a4c7000a4d0100a4fe400a500100a60d400a610100a620300a62a100a62c000a640100a66f200a673400a674200a67e400a67f100a69e200a6a0100a6e6000a6f0200a6f2400a6f8000a700400a717100a720400a722100a789400a78b100a7ce000a7d0100a7d2000a7d3100a7d4000a7d5100a7dd000a7f2100a802200a8031", + "00a806200a807100a80b200a80c100a823200a828400a82c200a82d000a836400a83a000a840100a874400a878000a880200a882100a8b4200a8c6000a8ce400a8d0300a8da000a8e0200a8f2100a8f8400a8fb100a8fc400a8fd100a8ff200a900300a90a100a926200a92e400a930100a947200a954000a95f400a960100a97d000a980200a984100a9b3200a9c1400a9ce000a9cf100a9d0300a9da000a9de400a9e0100a9e5200a9e6100a9f0300a9fa100a9ff000aa00100aa29200aa37000aa40100aa43200aa44100aa4c200aa4e000aa50300aa5a000aa5c400aa60100aa77400aa7a100aa7b200aa7e100aab0200aab1100aab2200aab5100aab7200aab9100aabe200aac0100aac1200aac2100aac3000aadb100aade400aae0100aaeb200aaf0400aaf2100aaf5200aaf7000ab01100ab07000ab09100ab0f000ab11100ab17000ab20100ab27000ab28100ab2f000ab30100ab5b400ab5c100ab6a400ab6c000ab70100abe3200abeb400abec200abee000abf0300abfa000ac00100d7a4000d7b0100d7c7000d7cb100d7fc000f900100fa6e000fa70100fada000fb00100fb07000fb13100fb18000fb1d100fb1e200fb1f100fb29400fb2a100fb37000fb38100fb3d000fb3e100fb3f000fb40100fb42000fb43100fb45000fb46100fbb2400fbc3000fbd3100fd3e400fd50100fd90000fd92100fdc8000fdcf400fdd0000fdf0100fdfc400fe00200fe10400fe1a000fe20200fe30400fe53000fe54400fe67000fe68400fe6c000fe70100fe75000fe76100fefd000ff01400ff10300ff1a400ff21100ff3b400ff41100ff5b400ff66100ffbf000ffc2100ffc8000ffca100ffd0000ffd2100ffd8000ffda100ffdd000ffe0400ffe7000ffe8400ffef000fffc400fffe0010000101000c001000d10100270010028101003b001003c101003e001003f101004e0010050101005e001008010100fb00101004010103001013740101400010179401018a001018c401018f0010190401019d00101a040101a100101d040101fd20101fe0010280101029d00102a010102d100102e020102e100103001010320001032d10103410010342101034a00103501010376201037b0010380101039e001039f40103a010103c400103c810103d040103d10010400101049e00104a030104aa00104b010104d400104d810104fc0010500101052800105301010564001056f4010570101057b001057c101058b001058c101059300105941010596001059710105a200105a310105b200105b310105ba00105bb10105bd00105c010105f400106001010737001074010107560010760101076800107801010786001078710107b100107b210107bb0010800101080600108081010809001080a101083600108371010839001083c101083d001083f1010856001085740108580010860101087740108790010880101089f00108e010108f300108f410108f600109001010916001091f4010920101093a001093f4010940001098010109b800109be10109c00010a001010a012010a040010a052010a070010a0c2010a101010a140010a151010a180010a191010a360010a382010a3b0010a3f2010a400010a504010a590010a601010a7d0010a7f4010a801010a9d0010ac01010ac84010ac91010ae52010ae70010af04010af70010b001010b360010b394010b401010b560010b601010b730010b801010b920010b994010b9d0010c001010c490010c801010cb30010cc01010cf30010d001010d242010d280010d303010d3a0010d403010d4a1010d660010d692010d6e4010d6f1010d860010d8e4010d900010e801010eaa0010eab2010ead4010eae0010eb01010eb20010ec21010ec50010efc2010f001010f1d0010f271010f280010f301010f462010f510010f554010f5a0010f701010f822010f864010f8a0010fb01010fc50010fe01010ff70011000201100310110382011047401104e001106630110702011071101107320110751011076001107f201108310110b020110bb40110bd00110be40110c220110c300110d010110e900110f030110fa0011100201110310111272011135001113630111404011144101114520111471011148001115010111732011174401117610111770011180201118310111b320111c110111c540111c920111cd40111ce20111d030111da10111db40111dc10111dd40111e0001120010112120011213101122c2011238401123e201123f101124120112420011280101128700112881011289001128a101128e001128f101129e001129f10112a940112aa00112b010112df20112eb00112f030112fa001130020113040011305101130d001130f101131100113131011329001132a1011331001133210113340011335101133a001133b201133d101133e201134500113472011349001134b201134e0011350101135100113572011358001135d101136220113640011366201136d001137020113750011380101138a001138b101138c001138e101138f001139010113b600113b710113b820113c100113c220113c300113c520113c600113c720113cb00113cc20113d110113d220113d310113d440113d600113d740113d900113e120113e3001140010114352011447101144b4011450301145a401145c001145d401145e201145f1011462001148010114b020114c410114c640114c710114c800114d030114da001158010115af20115b600115b820115c140115d810115dc20115de001160010116302011641401164410116450011650301165a0011660401166d001168010116ab20116b810116b940116ba00116c030116ca00116d030116e40011700101171b001171d201172c0011730301173a001173c401174010117470011800101182c201183b401183c00118a010118e030118ea00118ff10119070011909101190a001190c10119140011915101191700119181011930201193600119372011939001193b201193f101194020119411011942201194440119470011950301195a00119a010119a800119aa10119d120119d800119da20119e110119e240119e310119e420119e50011a001011a012011a0b1011a332011a3a1011a3b2011a3f4011a472011a480011a501011a512011a5c1011a8a2011a9a4011a9d1011a9e4011aa30011ab01011af90011b004011b0a0011bc01011be14011be20011bf03011bfa0011c001011c090011c0a1011c2f2011c370011c382011c401011c414011c460011c503011c5a0011c704011c721011c900011c922011ca80011ca92011cb70011d001011d070011d081011d0a0011d0b1011d312011d370011d3a2011d3b0011d3c2011d3e0011d3f2011d461011d472011d480011d503011d5a0011d601011d660011d671011d690011d6a1011d8a2011d8f0011d902011d920011d932011d981011d990011da03011daa0011ee01011ef32011ef74011ef90011f002011f021011f032011f041011f110011f121011f342011f3b0011f3e2011f434011f503011f5a2011f5b0011fb01011fb10011fd54011ff20011fff4012000101239a00124704012475001248010125440012f901012ff14012ff3001300010134300013440201344110134472013456001346010143fb001440010146470016100101611e2016130301613a00168001016a390016a401016a5f0016a603016a6a0016a6e4016a701016abf0016ac03016aca0016ad01016aee0016af02016af54016af60016b001016b302016b374016b401016b444016b460016b503016b5a0016b631016b780016b7d1016b900016d401016d6d4016d703016d7a0016e401016e800016e974016e9b0016f001016f4b0016f4f2016f501016f512016f880016f8f2016f931016fa00016fe01016fe24016fe31016fe42016fe50016ff02016ff2001700010187f800188001018cd60018cff1018d09001aff0101aff4001aff5101affc001affd101afff001b000101b123001b132101b133001b150101b153001b155101b156001b164101b168001b170101b2fc001bc00101bc6b001bc70101bc7d001bc80101bc89001bc90101bc9a001bc9c401bc9d201bc9f401bca0001cc00401ccf0301ccfa001cd00401ceb4001cf00201cf2e001cf30201cf47001cf50401cfc4001d000401d0f6001d100401d127001d129401d165201d16a401d16d201d173001d17b201d183401d185201d18c401d1aa201d1ae401d1eb001d200401d242201d245401d246001d300401d357001d400101d455001d456101d49d001d49e101d4a0001d4a2101d4a3001d4a5101d4a7001d4a9101d4ad001d4ae101d4ba001d4bb101d4bc001d4bd101d4c4001d4c5101d506001d507101d50b001d50d101d515001d516101d51d001d51e101d53a001d53b101d53f001d540101d545001d546101d547001d54a101d551001d552101d6a6001d6a8101d6c1401d6c2101d6db401d6dc101d6fb401d6fc101d715401d716101d735401d736101d74f401d750101d76f401d770101d789401d78a101d7a9401d7aa101d7c3401d7c4101d7cc001d7ce301d800401da00201da37401da3b201da6d401da75201da76401da84201da85401da8c001da9b201daa0001daa1201dab0001df00101df1f001df25101df2b001e000201e007001e008201e019001e01b201e022001e023201e025001e026201e02b001e030101e06e001e08f201e090001e100101e12d001e130201e137101e13e001e140301e14a001e14e101e14f401e150001e290101e2ae201e2af001e2c0101e2ec201e2f0301e2fa001e2ff401e300001e4d0101e4ec201e4f0301e4fa001e5d0101e5ee201e5f0101e5f1301e5fb001e5ff401e600001e7e01", + "01e7e7001e7e8101e7ec001e7ed101e7ef001e7f0101e7ff001e800101e8c5001e8d0201e8d7001e900101e944201e94b101e94c001e950301e95a001e95e401e960001ecac401ecad001ecb0401ecb1001ed2e401ed2f001ee00101ee04001ee05101ee20001ee21101ee23001ee24101ee25001ee27101ee28001ee29101ee33001ee34101ee38001ee39101ee3a001ee3b101ee3c001ee42101ee43001ee47101ee48001ee49101ee4a001ee4b101ee4c001ee4d101ee50001ee51101ee53001ee54101ee55001ee57101ee58001ee59101ee5a001ee5b101ee5c001ee5d101ee5e001ee5f101ee60001ee61101ee63001ee64101ee65001ee67101ee6b001ee6c101ee73001ee74101ee78001ee79101ee7d001ee7e101ee7f001ee80101ee8a001ee8b101ee9c001eea1101eea4001eea5101eeaa001eeab101eebc001eef0401eef2001f000401f02c001f030401f094001f0a0401f0af001f0b1401f0c0001f0c1401f0d0001f0d1401f0f6001f10d401f1ae001f1e6401f203001f210401f23c001f240401f249001f250401f252001f260401f266001f300401f6d8001f6dc401f6ed001f6f0401f6fd001f700401f777001f77b401f7da001f7e0401f7ec001f7f0401f7f1001f800401f80c001f810401f848001f850401f85a001f860401f888001f890401f8ae001f8b0401f8bc001f8c0401f8c2001f900401fa54001fa60401fa6e001fa70401fa7d001fa80401fa8a001fa8f401fac7001face401fadd001fadf401faea001faf0401faf9001fb00401fb93001fb94401fbf0301fbfa0020000102a6e0002a700102b73a002b740102b81e002b820102cea2002ceb0102ebe1002ebf0102ee5e002f800102fa1e0030000103134b003135010323b000e010020e01f00", + ) + + private val starts: IntArray + private val classes: ByteArray + + init { + val all = packed.joinToString("") + val n = all.length / 7 + starts = IntArray(n) + classes = ByteArray(n) + for (i in 0 until n) { + starts[i] = all.substring(i * 7, i * 7 + 6).toInt(16) + classes[i] = (all[i * 7 + 6] - '0').toByte() + } + } + + fun classify(codePoint: Int): CodePointClass { + if (codePoint < 0 || codePoint > 0x10FFFF) return CodePointClass.OTHER + var lo = 0 + var hi = starts.size - 1 + while (lo < hi) { + val mid = (lo + hi + 1) ushr 1 + if (starts[mid] <= codePoint) lo = mid else hi = mid - 1 + } + return CodePointClass.entries[classes[lo].toInt()] + } +} diff --git a/libs/text-format/src/commonTest/kotlin/com/flipcash/libs/textformat/TextFormatUnicodeTest.kt b/libs/text-format/src/commonTest/kotlin/com/flipcash/libs/textformat/TextFormatUnicodeTest.kt new file mode 100644 index 0000000000..9d347eb358 --- /dev/null +++ b/libs/text-format/src/commonTest/kotlin/com/flipcash/libs/textformat/TextFormatUnicodeTest.kt @@ -0,0 +1,53 @@ +// Generated by code/test-vectors/gen_text_format_unicode.py. Do not edit. +// Unicode 16.0.0. Regenerate with: +// python3 gen_text_format_unicode.py test > TextFormatUnicodeTest.kt +package com.flipcash.libs.textformat + +import kotlin.test.Test +import kotlin.test.assertEquals + +class TextFormatUnicodeTest { + @Test + fun version() { + assertEquals("16.0.0", TextFormatUnicode.UNICODE_VERSION) + } + + @Test + fun probes() { + assertEquals(CodePointClass.LETTER, TextFormatUnicode.classify(0x41)) // A + assertEquals(CodePointClass.LETTER, TextFormatUnicode.classify(0xE9)) // e acute, precomposed + assertEquals(CodePointClass.MARK, TextFormatUnicode.classify(0x301)) // combining acute + assertEquals(CodePointClass.MARK, TextFormatUnicode.classify(0xFE0F)) // variation selector 16, after emoji + assertEquals(CodePointClass.DIGIT, TextFormatUnicode.classify(0x30)) // 0 + assertEquals(CodePointClass.DIGIT, TextFormatUnicode.classify(0x663)) // Arabic-Indic three + assertEquals(CodePointClass.OTHER, TextFormatUnicode.classify(0xB2)) // superscript two is No, not Nd + assertEquals(CodePointClass.LETTER, TextFormatUnicode.classify(0x5B57)) // Han character + assertEquals(CodePointClass.LETTER, TextFormatUnicode.classify(0x1D400)) // mathematical bold A, outside the BMP + assertEquals(CodePointClass.PUNCT, TextFormatUnicode.classify(0x1F600)) // grinning face is So + assertEquals(CodePointClass.PUNCT, TextFormatUnicode.classify(0x2A)) // asterisk + assertEquals(CodePointClass.PUNCT, TextFormatUnicode.classify(0x5F)) // low line + assertEquals(CodePointClass.PUNCT, TextFormatUnicode.classify(0x24)) // dollar sign is Sc + assertEquals(CodePointClass.PUNCT, TextFormatUnicode.classify(0xB4)) // acute accent is Sk + assertEquals(CodePointClass.SPACE, TextFormatUnicode.classify(0x20)) // space + assertEquals(CodePointClass.SPACE, TextFormatUnicode.classify(0xA0)) // no-break space + assertEquals(CodePointClass.SPACE, TextFormatUnicode.classify(0x3000)) // ideographic space + assertEquals(CodePointClass.OTHER, TextFormatUnicode.classify(0x9)) // tab is Cc; the parser handles it + assertEquals(CodePointClass.OTHER, TextFormatUnicode.classify(0x200B)) // zero width space is Cf + assertEquals(CodePointClass.OTHER, TextFormatUnicode.classify(0x200D)) // zero width joiner is Cf + assertEquals(CodePointClass.OTHER, TextFormatUnicode.classify(0xD800)) // lone surrogate + assertEquals(CodePointClass.OTHER, TextFormatUnicode.classify(0x10FFFF)) // last code point + } + + // Per-class totals over every code point catch a corrupted or truncated table. + @Test + fun classTotals() { + val counts = IntArray(6) + for (cp in 0..0x10FFFF) counts[TextFormatUnicode.classify(cp).ordinal]++ + assertEquals(960437, counts[CodePointClass.OTHER.ordinal]) + assertEquals(141028, counts[CodePointClass.LETTER.ordinal]) + assertEquals(2501, counts[CodePointClass.MARK.ordinal]) + assertEquals(760, counts[CodePointClass.DIGIT.ordinal]) + assertEquals(9369, counts[CodePointClass.PUNCT.ordinal]) + assertEquals(17, counts[CodePointClass.SPACE.ordinal]) + } +} diff --git a/libs/text-format/src/commonTest/kotlin/com/flipcash/libs/textformat/TextFormatVectorTest.kt b/libs/text-format/src/commonTest/kotlin/com/flipcash/libs/textformat/TextFormatVectorTest.kt new file mode 100644 index 0000000000..d2c5daa9a1 --- /dev/null +++ b/libs/text-format/src/commonTest/kotlin/com/flipcash/libs/textformat/TextFormatVectorTest.kt @@ -0,0 +1,97 @@ +package com.flipcash.libs.textformat + +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonArray +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.int +import kotlinx.serialization.json.jsonArray +import kotlinx.serialization.json.jsonObject +import kotlinx.serialization.json.jsonPrimitive +import org.kotlincrypto.hash.sha2.SHA256 +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertTrue + +/** + * GATE: the parser must reproduce the canonical cross-platform fixture exactly. + * + * `text_format.json` is copied byte for byte from the orchestrator's `test-vectors/`, where + * `gen_text_format.py` writes it. Compiled into `commonTest`, this runs on the JVM and on the iOS + * simulator from one source. + */ +class TextFormatVectorTest { + + @Test + fun fixtureIsTheExpectedCopy() { + val digest = SHA256().digest(readTestResource("text_format.json").encodeToByteArray()) + val hex = digest.joinToString("") { (it.toInt() and 0xFF).toString(16).padStart(2, '0') } + assertEquals( + FIXTURE_SHA256, + hex, + "text_format.json differs from the orchestrator's copy; recopy it and update FIXTURE_SHA256", + ) + } + + @Test + fun parser_matches_canonical_vectors() { + val root = Json.parseToJsonElement(readTestResource("text_format.json")).jsonObject + val vectors = root["vectors"]!!.jsonArray + assertEquals(VECTOR_COUNT, vectors.size, "vector count") + + for (el in vectors) { + val v = el.jsonObject + val name = v["name"]!!.jsonPrimitive.content + val text = v["text"]!!.jsonPrimitive.content + val ranges = v["ranges"]!!.jsonArray.map { + val r = it.jsonObject + ProtectedRange(r.int("start"), r.int("end"), r.kind()) + } + + val got = parseTextFormat(text, ranges) + + assertEquals(v["display"]!!.jsonPrimitive.content, got.display, "display for '$name'") + assertEquals(expectedSpans(v["spans"]!!.jsonArray), got.spans, "spans for '$name'") + assertEquals( + expectedDisplayRanges(v["displayRanges"]!!.jsonArray), + got.displayRanges, + "displayRanges for '$name'", + ) + } + assertTrue(vectors.isNotEmpty(), "no vectors loaded") + } + + private fun expectedSpans(array: JsonArray): List = array.map { + val s = it.jsonObject + StyledSpan(s.int("start"), s.int("end"), s["style"]!!.jsonPrimitive.content.toStyle()) + } + + private fun expectedDisplayRanges(array: JsonArray): List = array.map { + val r = it.jsonObject + DisplayRange(r.int("start"), r.int("end"), r.kind(), r["target"]?.jsonPrimitive?.content) + } + + private fun JsonObject.int(key: String): Int = this[key]!!.jsonPrimitive.int + + private fun JsonObject.kind(): RangeKind = when (val k = this["kind"]!!.jsonPrimitive.content) { + "link" -> RangeKind.Link + "mention" -> RangeKind.Mention + else -> error("unknown range kind '$k'") + } + + private fun String.toStyle(): FormatStyle = when (this) { + "bold" -> FormatStyle.Bold + "italic" -> FormatStyle.Italic + "strike" -> FormatStyle.Strike + "code" -> FormatStyle.Code + "codeBlock" -> FormatStyle.CodeBlock + "quote" -> FormatStyle.Quote + "bullet" -> FormatStyle.Bullet + "numbered" -> FormatStyle.Numbered + else -> error("unknown style '$this'") + } + + private companion object { + const val FIXTURE_SHA256 = "cde3d3baaf8d7f7028f91244cd9dd4ff311c256a7085d757b6ac81040b6a5eb6" + const val VECTOR_COUNT = 122 + } +} diff --git a/libs/text-format/src/commonTest/resources/text_format.json b/libs/text-format/src/commonTest/resources/text_format.json new file mode 100644 index 0000000000..061895a1cf --- /dev/null +++ b/libs/text-format/src/commonTest/resources/text_format.json @@ -0,0 +1,1831 @@ +{ + "algorithm": "text-format", + "note": "Chat text formatting carried as markup in TextContent.text. Input: `text` and the `ranges` (links and mentions) detection found in it (UTF-16, half-open, raw offsets); markers inside a range are literal. Output: `display` (consumed markers removed), `spans` over display, and `displayRanges` (the input ranges at display offsets). Spans are normalized: per style, touching or overlapping runs are merged; ordered by start, then longest first, then style. Behavior parity, not a computation with an external reference.", + "styles": [ + "bold", + "italic", + "strike", + "code", + "codeBlock", + "quote", + "bullet", + "numbered" + ], + "vectors": [ + { + "name": "plain", + "text": "hello world", + "ranges": [], + "display": "hello world", + "spans": [], + "displayRanges": [], + "note": "No markup: display is the raw text." + }, + { + "name": "bold", + "text": "*hi*", + "ranges": [], + "display": "hi", + "spans": [ + { + "start": 0, + "end": 2, + "style": "bold" + } + ], + "displayRanges": [], + "note": "" + }, + { + "name": "italic", + "text": "_hi_", + "ranges": [], + "display": "hi", + "spans": [ + { + "start": 0, + "end": 2, + "style": "italic" + } + ], + "displayRanges": [], + "note": "" + }, + { + "name": "strike", + "text": "~hi~", + "ranges": [], + "display": "hi", + "spans": [ + { + "start": 0, + "end": 2, + "style": "strike" + } + ], + "displayRanges": [], + "note": "" + }, + { + "name": "code", + "text": "`hi`", + "ranges": [], + "display": "hi", + "spans": [ + { + "start": 0, + "end": 2, + "style": "code" + } + ], + "displayRanges": [], + "note": "" + }, + { + "name": "bold-mid-sentence", + "text": "say *hello there* now", + "ranges": [], + "display": "say hello there now", + "spans": [ + { + "start": 4, + "end": 15, + "style": "bold" + } + ], + "displayRanges": [], + "note": "" + }, + { + "name": "punctuation-edges", + "text": "(*wow*)!", + "ranges": [], + "display": "(wow)!", + "spans": [ + { + "start": 1, + "end": 4, + "style": "bold" + } + ], + "displayRanges": [], + "note": "Punctuation is not a word character, so it does not block a marker." + }, + { + "name": "curly-quotes", + "text": "“*really*”", + "ranges": [], + "display": "“really”", + "spans": [ + { + "start": 1, + "end": 7, + "style": "bold" + } + ], + "displayRanges": [], + "note": "" + }, + { + "name": "comma-after-close", + "text": "*wow*, really", + "ranges": [], + "display": "wow, really", + "spans": [ + { + "start": 0, + "end": 3, + "style": "bold" + } + ], + "displayRanges": [], + "note": "" + }, + { + "name": "apostrophe-inside", + "text": "it's *Bob's* car", + "ranges": [], + "display": "it's Bob's car", + "spans": [ + { + "start": 5, + "end": 10, + "style": "bold" + } + ], + "displayRanges": [], + "note": "" + }, + { + "name": "snake-case", + "text": "snake_case_name", + "ranges": [], + "display": "snake_case_name", + "spans": [], + "displayRanges": [], + "note": "Every underscore has a letter on both sides." + }, + { + "name": "arithmetic", + "text": "2*3*4", + "ranges": [], + "display": "2*3*4", + "spans": [], + "displayRanges": [], + "note": "Digits are word characters." + }, + { + "name": "closer-glued-to-word", + "text": "*foo*bar", + "ranges": [], + "display": "*foo*bar", + "spans": [], + "displayRanges": [], + "note": "The only candidate closer has a letter after it." + }, + { + "name": "space-after-opener", + "text": "a * not bold*", + "ranges": [], + "display": "a * not bold*", + "spans": [], + "displayRanges": [], + "note": "" + }, + { + "name": "space-before-closer", + "text": "*not bold *", + "ranges": [], + "display": "*not bold *", + "spans": [], + "displayRanges": [], + "note": "" + }, + { + "name": "spaced-asterisks", + "text": "a * b * c", + "ranges": [], + "display": "a * b * c", + "spans": [], + "displayRanges": [], + "note": "" + }, + { + "name": "approximately", + "text": "~5 minutes, ~10 tops", + "ranges": [], + "display": "~5 minutes, ~10 tops", + "spans": [], + "displayRanges": [], + "note": "Tilde as 'about': the second tilde follows a space, so it cannot close." + }, + { + "name": "italic-with-snake-inside", + "text": "_my snake_case var_", + "ranges": [], + "display": "my snake_case var", + "spans": [ + { + "start": 0, + "end": 17, + "style": "italic" + } + ], + "displayRanges": [], + "note": "The inner underscore is intraword and skipped as a closer." + }, + { + "name": "digit-content", + "text": "*2*", + "ranges": [], + "display": "2", + "spans": [ + { + "start": 0, + "end": 1, + "style": "bold" + } + ], + "displayRanges": [], + "note": "" + }, + { + "name": "digit-before-opener", + "text": "v*2*", + "ranges": [], + "display": "v*2*", + "spans": [], + "displayRanges": [], + "note": "" + }, + { + "name": "accented-letter-before", + "text": "é*x*", + "ranges": [], + "display": "é*x*", + "spans": [], + "displayRanges": [], + "note": "A non-ASCII letter is a word character." + }, + { + "name": "combining-mark-before", + "text": "café*x*", + "ranges": [], + "display": "café*x*", + "spans": [], + "displayRanges": [], + "note": "U+0301 is category Mn. Counting marks as word characters keeps a decomposed é behaving like a precomposed one." + }, + { + "name": "cjk-glued", + "text": "日本*語*", + "ranges": [], + "display": "日本*語*", + "spans": [], + "displayRanges": [], + "note": "Han characters are letters, so markup inside unspaced CJK text stays literal. Known limit; see the spec." + }, + { + "name": "cjk-spaced", + "text": "日本 *語*", + "ranges": [], + "display": "日本 語", + "spans": [ + { + "start": 3, + "end": 4, + "style": "bold" + } + ], + "displayRanges": [], + "note": "" + }, + { + "name": "emoji-before-opener", + "text": "👍*nice*", + "ranges": [], + "display": "👍nice", + "spans": [ + { + "start": 2, + "end": 6, + "style": "bold" + } + ], + "displayRanges": [], + "note": "An emoji is not a word character. Offsets are UTF-16: the emoji is two units." + }, + { + "name": "emoji-inside", + "text": "*🔥 hot*", + "ranges": [], + "display": "🔥 hot", + "spans": [ + { + "start": 0, + "end": 6, + "style": "bold" + } + ], + "displayRanges": [], + "note": "" + }, + { + "name": "astral-letter-before", + "text": "𝐀*x*", + "ranges": [], + "display": "𝐀*x*", + "spans": [], + "displayRanges": [], + "note": "U+1D400 is a letter (Lu) outside the BMP. A parser that looks at the UTF-16 unit before the marker sees a low surrogate (Cs), wrongly opens, and passes every other vector. Read code points." + }, + { + "name": "emoji-vs16-before-opener", + "text": "❤️*love*", + "ranges": [], + "display": "❤️love", + "spans": [ + { + "start": 2, + "end": 6, + "style": "bold" + } + ], + "displayRanges": [], + "note": "U+FE0F is a mark, but it is skipped back to the heart (So), which is not a word character. Without the skip, every emoji with a variation selector would glue." + }, + { + "name": "emoji-vs16-before-underscore", + "text": "❤️_love_", + "ranges": [], + "display": "❤️love", + "spans": [ + { + "start": 2, + "end": 6, + "style": "italic" + } + ], + "displayRanges": [], + "note": "The same skip applies to `_`. Plain CommonMark sees U+FE0F as alphanumeric and would leave this literal: the one place this spec departs from its flanking rules." + }, + { + "name": "nbsp-before-opener", + "text": "a *b*", + "ranges": [], + "display": "a b", + "spans": [ + { + "start": 2, + "end": 3, + "style": "bold" + } + ], + "displayRanges": [], + "note": "NBSP is category Zs." + }, + { + "name": "cm-357", + "text": "_foo bar_", + "ranges": [], + "display": "foo bar", + "spans": [ + { + "start": 0, + "end": 7, + "style": "italic" + } + ], + "displayRanges": [], + "note": "CommonMark example 357." + }, + { + "name": "cm-358", + "text": "_ foo bar_", + "ranges": [], + "display": "_ foo bar_", + "spans": [], + "displayRanges": [], + "note": "CommonMark example 358." + }, + { + "name": "cm-359", + "text": "a_\"foo\"_", + "ranges": [], + "display": "a_\"foo\"_", + "spans": [], + "displayRanges": [], + "note": "CommonMark example 359." + }, + { + "name": "cm-360", + "text": "foo_bar_", + "ranges": [], + "display": "foo_bar_", + "spans": [], + "displayRanges": [], + "note": "CommonMark example 360." + }, + { + "name": "cm-361", + "text": "5_6_78", + "ranges": [], + "display": "5_6_78", + "spans": [], + "displayRanges": [], + "note": "CommonMark example 361." + }, + { + "name": "cm-362", + "text": "пристаням_стремятся_", + "ranges": [], + "display": "пристаням_стремятся_", + "spans": [], + "displayRanges": [], + "note": "CommonMark example 362." + }, + { + "name": "cm-363", + "text": "aa_\"bb\"_cc", + "ranges": [], + "display": "aa_\"bb\"_cc", + "spans": [], + "displayRanges": [], + "note": "CommonMark example 363." + }, + { + "name": "cm-364", + "text": "foo-_(bar)_", + "ranges": [], + "display": "foo-(bar)", + "spans": [ + { + "start": 4, + "end": 9, + "style": "italic" + } + ], + "displayRanges": [], + "note": "CommonMark example 364." + }, + { + "name": "cm-371", + "text": "_foo bar _", + "ranges": [], + "display": "_foo bar _", + "spans": [], + "displayRanges": [], + "note": "CommonMark example 371." + }, + { + "name": "cm-372", + "text": "_(_foo)", + "ranges": [], + "display": "_(_foo)", + "spans": [], + "displayRanges": [], + "note": "CommonMark example 372." + }, + { + "name": "cm-374", + "text": "_foo_bar", + "ranges": [], + "display": "_foo_bar", + "spans": [], + "displayRanges": [], + "note": "CommonMark example 374." + }, + { + "name": "cm-375", + "text": "_пристаням_стремятся", + "ranges": [], + "display": "_пристаням_стремятся", + "spans": [], + "displayRanges": [], + "note": "CommonMark example 375." + }, + { + "name": "cm-376", + "text": "_foo_bar_baz_", + "ranges": [], + "display": "foo_bar_baz", + "spans": [ + { + "start": 0, + "end": 11, + "style": "italic" + } + ], + "displayRanges": [], + "note": "CommonMark example 376." + }, + { + "name": "cm-377", + "text": "_(bar)_.", + "ranges": [], + "display": "(bar).", + "spans": [ + { + "start": 0, + "end": 5, + "style": "italic" + } + ], + "displayRanges": [], + "note": "CommonMark example 377." + }, + { + "name": "markdown-double", + "text": "**markdown**", + "ranges": [], + "display": "**markdown**", + "spans": [], + "displayRanges": [], + "note": "A marker next to the same marker is never a delimiter, so Markdown-style doubling shows as typed instead of half-formatting." + }, + { + "name": "triple", + "text": "***x***", + "ranges": [], + "display": "***x***", + "spans": [], + "displayRanges": [], + "note": "" + }, + { + "name": "empty-pair", + "text": "**", + "ranges": [], + "display": "**", + "spans": [], + "displayRanges": [], + "note": "" + }, + { + "name": "lone-marker", + "text": "*", + "ranges": [], + "display": "*", + "spans": [], + "displayRanges": [], + "note": "" + }, + { + "name": "unclosed", + "text": "*unclosed", + "ranges": [], + "display": "*unclosed", + "spans": [], + "displayRanges": [], + "note": "" + }, + { + "name": "nested", + "text": "*bold _and italic_*", + "ranges": [], + "display": "bold and italic", + "spans": [ + { + "start": 0, + "end": 15, + "style": "bold" + }, + { + "start": 5, + "end": 15, + "style": "italic" + } + ], + "displayRanges": [], + "note": "" + }, + { + "name": "leftmost-nearest", + "text": "*a *b* c*", + "ranges": [], + "display": "a *b c*", + "spans": [ + { + "start": 0, + "end": 4, + "style": "bold" + } + ], + "displayRanges": [], + "note": "The `*` after `a ` cannot close (space before it) and the one before `b` finds no closer inside the bold, so both stay literal." + }, + { + "name": "crossing", + "text": "*a _b* c_", + "ranges": [], + "display": "a _b c_", + "spans": [ + { + "start": 0, + "end": 4, + "style": "bold" + } + ], + "displayRanges": [], + "note": "Styles do not cross. The bold closes first; the italic has no closer inside it." + }, + { + "name": "adjacent-same-style", + "text": "*a*_*b*_", + "ranges": [], + "display": "ab", + "spans": [ + { + "start": 0, + "end": 2, + "style": "bold" + }, + { + "start": 1, + "end": 2, + "style": "italic" + } + ], + "displayRanges": [], + "note": "Two touching bold runs are normalized into one span." + }, + { + "name": "code-protects", + "text": "`*not bold*`", + "ranges": [], + "display": "*not bold*", + "spans": [ + { + "start": 0, + "end": 10, + "style": "code" + } + ], + "displayRanges": [], + "note": "" + }, + { + "name": "code-inside-bold", + "text": "*run `ls` now*", + "ranges": [], + "display": "run ls now", + "spans": [ + { + "start": 0, + "end": 10, + "style": "bold" + }, + { + "start": 4, + "end": 6, + "style": "code" + } + ], + "displayRanges": [], + "note": "" + }, + { + "name": "code-beats-crossing-bold", + "text": "*a `b* c`", + "ranges": [], + "display": "*a b* c", + "spans": [ + { + "start": 3, + "end": 7, + "style": "code" + } + ], + "displayRanges": [], + "note": "Code is matched before the other styles, so the bold's only closer is inside code and the bold stays literal." + }, + { + "name": "unclosed-backtick", + "text": "`*x*", + "ranges": [], + "display": "`x", + "spans": [ + { + "start": 1, + "end": 2, + "style": "bold" + } + ], + "displayRanges": [], + "note": "" + }, + { + "name": "mixed", + "text": "*a* `*b*` _c_", + "ranges": [], + "display": "a *b* c", + "spans": [ + { + "start": 0, + "end": 1, + "style": "bold" + }, + { + "start": 2, + "end": 5, + "style": "code" + }, + { + "start": 6, + "end": 7, + "style": "italic" + } + ], + "displayRanges": [], + "note": "" + }, + { + "name": "escape-pair", + "text": "\\*not bold\\*", + "ranges": [], + "display": "*not bold*", + "spans": [], + "displayRanges": [], + "note": "The backslashes are removed." + }, + { + "name": "escape-opener", + "text": "\\*not emphasized*", + "ranges": [], + "display": "*not emphasized*", + "spans": [], + "displayRanges": [], + "note": "CommonMark example 14, first line. The closer has no opener left." + }, + { + "name": "escape-backslash", + "text": "\\\\*emphasis*", + "ranges": [], + "display": "\\emphasis", + "spans": [ + { + "start": 1, + "end": 9, + "style": "bold" + } + ], + "displayRanges": [], + "note": "CommonMark example 15. `\\\\` is one literal backslash, which is not a word character." + }, + { + "name": "escape-backtick", + "text": "\\`not code`", + "ranges": [], + "display": "`not code`", + "spans": [], + "displayRanges": [], + "note": "CommonMark example 14." + }, + { + "name": "escape-run", + "text": "\\**x*", + "ranges": [], + "display": "*x", + "spans": [ + { + "start": 1, + "end": 2, + "style": "bold" + } + ], + "displayRanges": [], + "note": "An escaped marker is not part of a run, so the second `*` can open." + }, + { + "name": "escape-not-escapable", + "text": "C:\\Users\\me", + "ranges": [], + "display": "C:\\Users\\me", + "spans": [], + "displayRanges": [], + "note": "Only `\\ * _ ~ ` > - .` can be escaped. A backslash before anything else stays." + }, + { + "name": "escape-trailing", + "text": "a\\", + "ranges": [], + "display": "a\\", + "spans": [], + "displayRanges": [], + "note": "" + }, + { + "name": "escape-in-code", + "text": "`\\*x\\*`", + "ranges": [], + "display": "\\*x\\*", + "spans": [ + { + "start": 0, + "end": 5, + "style": "code" + } + ], + "displayRanges": [], + "note": "A backslash inside code is literal." + }, + { + "name": "escape-closing-backtick", + "text": "`a\\`", + "ranges": [], + "display": "a\\", + "spans": [ + { + "start": 0, + "end": 2, + "style": "code" + } + ], + "displayRanges": [], + "note": "Not an escape: inside code the backslash is literal, so the backtick closes." + }, + { + "name": "escape-in-link", + "text": "example.com/a\\_b_", + "ranges": [ + { + "start": 0, + "end": 17, + "kind": "link" + } + ], + "display": "example.com/a\\_b_", + "spans": [], + "displayRanges": [ + { + "start": 0, + "end": 17, + "kind": "link" + } + ], + "note": "A backslash inside a link is part of the URL." + }, + { + "name": "escape-quote", + "text": "\\> not a quote", + "ranges": [], + "display": "> not a quote", + "spans": [], + "displayRanges": [], + "note": "" + }, + { + "name": "escape-bullet", + "text": "\\- not a list", + "ranges": [], + "display": "- not a list", + "spans": [], + "displayRanges": [], + "note": "" + }, + { + "name": "escape-bullet-star", + "text": "\\* not a list", + "ranges": [], + "display": "* not a list", + "spans": [], + "displayRanges": [], + "note": "CommonMark example 14." + }, + { + "name": "escape-numbered", + "text": "1\\. not a list", + "ranges": [], + "display": "1. not a list", + "spans": [], + "displayRanges": [], + "note": "CommonMark example 14." + }, + { + "name": "escape-fence", + "text": "```\nx\\```\n```", + "ranges": [], + "display": "x\\```", + "spans": [ + { + "start": 0, + "end": 5, + "style": "codeBlock" + } + ], + "displayRanges": [], + "note": "A fence after a backslash does not close. Inside the block the backslash is literal." + }, + { + "name": "shrug", + "text": "¯\\_(ツ)_/¯", + "ranges": [], + "display": "¯_(ツ)_/¯", + "spans": [], + "displayRanges": [], + "note": "The escape keeps both arms but eats the backslash. Without escapes this would be an italic `(ツ)` with both underscores gone. Typing `\\\\_` keeps all of it. Known limit." + }, + { + "name": "no-multiline-inline", + "text": "*line one\nline two*", + "ranges": [], + "display": "*line one\nline two*", + "spans": [], + "displayRanges": [], + "note": "" + }, + { + "name": "per-line", + "text": "*one*\n*two*", + "ranges": [], + "display": "one\ntwo", + "spans": [ + { + "start": 0, + "end": 3, + "style": "bold" + }, + { + "start": 4, + "end": 7, + "style": "bold" + } + ], + "displayRanges": [], + "note": "" + }, + { + "name": "bold-link", + "text": "*see example.com*", + "ranges": [ + { + "start": 5, + "end": 16, + "kind": "link" + } + ], + "display": "see example.com", + "spans": [ + { + "start": 0, + "end": 15, + "style": "bold" + } + ], + "displayRanges": [ + { + "start": 4, + "end": 15, + "kind": "link" + } + ], + "note": "A whole link inside a style keeps both." + }, + { + "name": "marker-inside-link", + "text": "go to example.com/*x*", + "ranges": [ + { + "start": 6, + "end": 21, + "kind": "link" + } + ], + "display": "go to example.com/*x*", + "spans": [], + "displayRanges": [ + { + "start": 6, + "end": 21, + "kind": "link" + } + ], + "note": "Markers inside a link are part of the URL." + }, + { + "name": "link-swallows-closer", + "text": "*example.com/foo*", + "ranges": [ + { + "start": 1, + "end": 17, + "kind": "link" + } + ], + "display": "*example.com/foo*", + "spans": [], + "displayRanges": [ + { + "start": 1, + "end": 17, + "kind": "link" + } + ], + "note": "If link detection keeps the trailing `*`, the bold has no closer. Detectors trim it (spec decision 1, link_detection.json `bold-wrapped-link`); this pins the parser for a detector that does not." + }, + { + "name": "link-trimmed-closer", + "text": "*example.com/foo*", + "ranges": [ + { + "start": 1, + "end": 16, + "kind": "link" + } + ], + "display": "example.com/foo", + "spans": [ + { + "start": 0, + "end": 15, + "style": "bold" + } + ], + "displayRanges": [ + { + "start": 0, + "end": 15, + "kind": "link" + } + ], + "note": "With the trailing `*` trimmed by the detector, the bold wraps the whole link." + }, + { + "name": "link-in-code", + "text": "`example.com`", + "ranges": [ + { + "start": 1, + "end": 12, + "kind": "link" + } + ], + "display": "example.com", + "spans": [ + { + "start": 0, + "end": 11, + "style": "code" + } + ], + "displayRanges": [ + { + "start": 0, + "end": 11, + "kind": "link" + } + ], + "note": "A link inside code stays tappable." + }, + { + "name": "mention-protects", + "text": "@_jeff_ hi", + "ranges": [ + { + "start": 0, + "end": 7, + "kind": "mention" + } + ], + "display": "@_jeff_ hi", + "spans": [], + "displayRanges": [ + { + "start": 0, + "end": 7, + "kind": "mention" + } + ], + "note": "Handles may contain `_`. Without protection this is an italic `jeff` and a broken mention." + }, + { + "name": "bold-mention", + "text": "*@jeff*", + "ranges": [ + { + "start": 1, + "end": 6, + "kind": "mention" + } + ], + "display": "@jeff", + "spans": [ + { + "start": 0, + "end": 5, + "style": "bold" + } + ], + "displayRanges": [ + { + "start": 0, + "end": 5, + "kind": "mention" + } + ], + "note": "Mention detection accepts `*` before the `@`, so a bold mention works." + }, + { + "name": "italic-mention", + "text": "_@jeff_", + "ranges": [ + { + "start": 1, + "end": 6, + "kind": "mention" + } + ], + "display": "@jeff", + "spans": [ + { + "start": 0, + "end": 5, + "style": "italic" + } + ], + "displayRanges": [ + { + "start": 0, + "end": 5, + "kind": "mention" + } + ], + "note": "Needs the mention-detection change in the spec: `_` may come before the `@`, and then a trailing `_` closes rather than ending the handle." + }, + { + "name": "italic-ending-in-mention", + "text": "_hey @jeff_", + "ranges": [ + { + "start": 5, + "end": 11, + "kind": "mention" + } + ], + "display": "_hey @jeff_", + "spans": [], + "displayRanges": [ + { + "start": 5, + "end": 11, + "kind": "mention" + } + ], + "note": "Handles may end in `_`, and here no `_` comes before the `@`, so detection takes `jeff_` and the italic has no closer. Known limit; see the spec." + }, + { + "name": "masked-link", + "text": "[docs](https://example.com)", + "ranges": [ + { + "start": 7, + "end": 26, + "kind": "link" + } + ], + "display": "docs", + "spans": [], + "displayRanges": [ + { + "start": 0, + "end": 4, + "kind": "link", + "target": "https://example.com" + } + ], + "note": "The brackets, parentheses and URL leave the display text. The link range covers the text and carries the URL as `target`." + }, + { + "name": "masked-link-in-sentence", + "text": "read [the docs](example.com/a) now", + "ranges": [ + { + "start": 16, + "end": 29, + "kind": "link" + } + ], + "display": "read the docs now", + "spans": [], + "displayRanges": [ + { + "start": 5, + "end": 13, + "kind": "link", + "target": "example.com/a" + } + ], + "note": "" + }, + { + "name": "masked-link-styled-text", + "text": "[*docs*](https://example.com)", + "ranges": [ + { + "start": 9, + "end": 28, + "kind": "link" + } + ], + "display": "docs", + "spans": [ + { + "start": 0, + "end": 4, + "style": "bold" + } + ], + "displayRanges": [ + { + "start": 0, + "end": 4, + "kind": "link", + "target": "https://example.com" + } + ], + "note": "" + }, + { + "name": "bold-masked-link", + "text": "*[docs](https://example.com)*", + "ranges": [ + { + "start": 8, + "end": 27, + "kind": "link" + } + ], + "display": "docs", + "spans": [ + { + "start": 0, + "end": 4, + "style": "bold" + } + ], + "displayRanges": [ + { + "start": 0, + "end": 4, + "kind": "link", + "target": "https://example.com" + } + ], + "note": "" + }, + { + "name": "masked-link-url-with-parens", + "text": "[film](https://en.wikipedia.org/wiki/Foo_(film))", + "ranges": [ + { + "start": 7, + "end": 47, + "kind": "link" + } + ], + "display": "film", + "spans": [], + "displayRanges": [ + { + "start": 0, + "end": 4, + "kind": "link", + "target": "https://en.wikipedia.org/wiki/Foo_(film)" + } + ], + "note": "Link detection keeps brackets the URL opened, so the `)` after it closes the mask." + }, + { + "name": "masked-link-text-is-a-link", + "text": "[example.com](https://evil.example)", + "ranges": [ + { + "start": 1, + "end": 12, + "kind": "link" + }, + { + "start": 14, + "end": 34, + "kind": "link" + } + ], + "display": "[example.com](https://evil.example)", + "spans": [], + "displayRanges": [ + { + "start": 1, + "end": 12, + "kind": "link" + }, + { + "start": 14, + "end": 34, + "kind": "link" + } + ], + "note": "Link text that is itself a link would hide where the link goes, so the mask is literal and both links show." + }, + { + "name": "masked-link-text-has-mention", + "text": "[@jeff](https://example.com)", + "ranges": [ + { + "start": 8, + "end": 27, + "kind": "link" + }, + { + "start": 1, + "end": 6, + "kind": "mention" + } + ], + "display": "[@jeff](https://example.com)", + "spans": [], + "displayRanges": [ + { + "start": 1, + "end": 6, + "kind": "mention" + }, + { + "start": 8, + "end": 27, + "kind": "link" + } + ], + "note": "A mention cannot sit inside a link." + }, + { + "name": "masked-link-empty-text", + "text": "[](https://example.com)", + "ranges": [ + { + "start": 3, + "end": 22, + "kind": "link" + } + ], + "display": "[](https://example.com)", + "spans": [], + "displayRanges": [ + { + "start": 3, + "end": 22, + "kind": "link" + } + ], + "note": "" + }, + { + "name": "masked-link-not-a-url", + "text": "[docs](not a url)", + "ranges": [], + "display": "[docs](not a url)", + "spans": [], + "displayRanges": [], + "note": "" + }, + { + "name": "masked-link-detector-took-paren", + "text": "[docs](https://example.com)", + "ranges": [ + { + "start": 7, + "end": 27, + "kind": "link" + } + ], + "display": "[docs](https://example.com)", + "spans": [], + "displayRanges": [ + { + "start": 7, + "end": 27, + "kind": "link" + } + ], + "note": "If link detection keeps the `)`, nothing closes the mask. Detection already drops a `)` the URL did not open, so this should not happen." + }, + { + "name": "masked-link-space-before-paren", + "text": "[docs] (https://example.com)", + "ranges": [ + { + "start": 8, + "end": 27, + "kind": "link" + } + ], + "display": "[docs] (https://example.com)", + "spans": [], + "displayRanges": [ + { + "start": 8, + "end": 27, + "kind": "link" + } + ], + "note": "" + }, + { + "name": "masked-link-nested-brackets", + "text": "[a [b] c](https://example.com)", + "ranges": [ + { + "start": 10, + "end": 29, + "kind": "link" + } + ], + "display": "[a [b] c](https://example.com)", + "spans": [], + "displayRanges": [ + { + "start": 10, + "end": 29, + "kind": "link" + } + ], + "note": "The first `]` after a `[` decides. Escape a bracket to put one in link text." + }, + { + "name": "masked-link-escaped-bracket", + "text": "[a \\] b](https://example.com)", + "ranges": [ + { + "start": 9, + "end": 28, + "kind": "link" + } + ], + "display": "a ] b", + "spans": [], + "displayRanges": [ + { + "start": 0, + "end": 5, + "kind": "link", + "target": "https://example.com" + } + ], + "note": "" + }, + { + "name": "masked-link-escaped", + "text": "\\[docs](https://example.com)", + "ranges": [ + { + "start": 8, + "end": 27, + "kind": "link" + } + ], + "display": "[docs](https://example.com)", + "spans": [], + "displayRanges": [ + { + "start": 7, + "end": 26, + "kind": "link" + } + ], + "note": "" + }, + { + "name": "masked-link-in-code", + "text": "`[docs](https://example.com)`", + "ranges": [ + { + "start": 8, + "end": 27, + "kind": "link" + } + ], + "display": "[docs](https://example.com)", + "spans": [ + { + "start": 0, + "end": 27, + "style": "code" + } + ], + "displayRanges": [ + { + "start": 7, + "end": 26, + "kind": "link" + } + ], + "note": "Code wins. The URL inside stays tappable, as in `link-in-code`." + }, + { + "name": "code-block", + "text": "```\nlet x = 1\n```", + "ranges": [], + "display": "let x = 1", + "spans": [ + { + "start": 0, + "end": 9, + "style": "codeBlock" + } + ], + "displayRanges": [], + "note": "The newline after the opening fence and before the closing fence are removed." + }, + { + "name": "code-block-one-line", + "text": "```x```", + "ranges": [], + "display": "x", + "spans": [ + { + "start": 0, + "end": 1, + "style": "codeBlock" + } + ], + "displayRanges": [], + "note": "" + }, + { + "name": "code-block-literal", + "text": "```\n*a* _b_\n> c\n```", + "ranges": [], + "display": "*a* _b_\n> c", + "spans": [ + { + "start": 0, + "end": 11, + "style": "codeBlock" + } + ], + "displayRanges": [], + "note": "Nothing is parsed inside a code block, block prefixes included." + }, + { + "name": "code-block-surrounded", + "text": "before\n```\ncode\n```\nafter", + "ranges": [], + "display": "before\ncode\nafter", + "spans": [ + { + "start": 7, + "end": 11, + "style": "codeBlock" + } + ], + "displayRanges": [], + "note": "" + }, + { + "name": "code-block-unclosed", + "text": "```\ncode", + "ranges": [], + "display": "```\ncode", + "spans": [], + "displayRanges": [], + "note": "" + }, + { + "name": "code-block-empty", + "text": "```\n```", + "ranges": [], + "display": "```\n```", + "spans": [], + "displayRanges": [], + "note": "A block with no content is literal." + }, + { + "name": "code-block-language-is-content", + "text": "```swift\nlet x = 1\n```", + "ranges": [], + "display": "swift\nlet x = 1", + "spans": [ + { + "start": 0, + "end": 15, + "style": "codeBlock" + } + ], + "displayRanges": [], + "note": "No language tags: text after the opening fence is content, as `code-block-one-line` needs." + }, + { + "name": "code-block-not-line-start", + "text": "x ```code```", + "ranges": [], + "display": "x ```code```", + "spans": [], + "displayRanges": [], + "note": "A fence must start a line, and a triple backtick is a run, so inline code does not take it either." + }, + { + "name": "quote", + "text": "> hello", + "ranges": [], + "display": "hello", + "spans": [ + { + "start": 0, + "end": 5, + "style": "quote" + } + ], + "displayRanges": [], + "note": "The `> ` prefix is removed." + }, + { + "name": "quote-lines", + "text": "> one\n> two", + "ranges": [], + "display": "one\ntwo", + "spans": [ + { + "start": 0, + "end": 3, + "style": "quote" + }, + { + "start": 4, + "end": 7, + "style": "quote" + } + ], + "displayRanges": [], + "note": "One span per line. Renderers join consecutive quote lines into one bar." + }, + { + "name": "quote-needs-space", + "text": ">hello", + "ranges": [], + "display": ">hello", + "spans": [], + "displayRanges": [], + "note": "" + }, + { + "name": "quote-empty", + "text": "> ", + "ranges": [], + "display": "> ", + "spans": [], + "displayRanges": [], + "note": "" + }, + { + "name": "quote-with-bold", + "text": "> *hi*", + "ranges": [], + "display": "hi", + "spans": [ + { + "start": 0, + "end": 2, + "style": "bold" + }, + { + "start": 0, + "end": 2, + "style": "quote" + } + ], + "displayRanges": [], + "note": "" + }, + { + "name": "bullet-dash", + "text": "- milk", + "ranges": [], + "display": "- milk", + "spans": [ + { + "start": 0, + "end": 6, + "style": "bullet" + } + ], + "displayRanges": [], + "note": "List markers stay in the display text. The renderer may draw a bullet glyph over `- ` and indents wrapped lines." + }, + { + "name": "bullet-star", + "text": "* milk", + "ranges": [], + "display": "* milk", + "spans": [ + { + "start": 0, + "end": 6, + "style": "bullet" + } + ], + "displayRanges": [], + "note": "`* ` is a bullet, never a bold opener, because a space follows it." + }, + { + "name": "bullet-with-bold", + "text": "- *milk*", + "ranges": [], + "display": "- milk", + "spans": [ + { + "start": 0, + "end": 6, + "style": "bullet" + }, + { + "start": 2, + "end": 6, + "style": "bold" + } + ], + "displayRanges": [], + "note": "" + }, + { + "name": "bullet-not-at-start", + "text": "a - b", + "ranges": [], + "display": "a - b", + "spans": [], + "displayRanges": [], + "note": "" + }, + { + "name": "numbered", + "text": "1. first\n2. second", + "ranges": [], + "display": "1. first\n2. second", + "spans": [ + { + "start": 0, + "end": 8, + "style": "numbered" + }, + { + "start": 9, + "end": 18, + "style": "numbered" + } + ], + "displayRanges": [], + "note": "" + }, + { + "name": "numbered-three-digits", + "text": "100. x", + "ranges": [], + "display": "100. x", + "spans": [ + { + "start": 0, + "end": 6, + "style": "numbered" + } + ], + "displayRanges": [], + "note": "" + }, + { + "name": "numbered-four-digits", + "text": "1000. x", + "ranges": [], + "display": "1000. x", + "spans": [], + "displayRanges": [], + "note": "At most three digits." + }, + { + "name": "numbered-needs-space", + "text": "1.5 kg", + "ranges": [], + "display": "1.5 kg", + "spans": [], + "displayRanges": [], + "note": "" + }, + { + "name": "quoted-list", + "text": "> - a", + "ranges": [], + "display": "- a", + "spans": [ + { + "start": 0, + "end": 3, + "style": "bullet" + }, + { + "start": 0, + "end": 3, + "style": "quote" + } + ], + "displayRanges": [], + "note": "Quote first, then a list inside it." + } + ] +} diff --git a/settings.gradle.kts b/settings.gradle.kts index 3c348af3bc..0e0b9aa37d 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -241,6 +241,7 @@ include( ":libs:quickresponse", ":libs:reporting", ":libs:search", + ":libs:text-format", ":libs:vibrator:bindings", ":libs:vibrator:impl", @@ -321,6 +322,7 @@ val kmpUnitTestModules = setOf( ":libs:encryption:sha256", ":libs:encryption:sha512", ":libs:encryption:hmac", + ":libs:text-format", // ed25519's androidHostTest is a JNI vector test that needs a host-native lib // (macOS .dylib / Linux .so); it can't load on the Linux CI runner. Its ed25519.json // parity is gated via the iOS cinterop path (macOS) instead — so it's excluded here.