From 79a292616eae75cc8190b0bee5ad36027f28c183 Mon Sep 17 00:00:00 2001 From: Ran Luo Date: Mon, 8 Jun 2026 17:52:46 +0800 Subject: [PATCH 1/5] fix(muya): treat CJK as punctuation for strong/em flanking (#4307) (#4401) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(muya): treat CJK as punctuation for strong/em flanking (#4307) Strong/em delimited with `**` directly against a CJK character whose inner content is punctuation-bounded did not bold in muya, e.g. `例子例子**"加粗"**例子例子`, `日本語**(強調)**日本語`, `한국어**[강조]**한국어`, and the non-BMP `𠀀𠀁**"加粗"**𠀀𠀁`. The legacy muyajs engine bolded these. CommonMark §6.2 classifies CJK ideographs / Hangul / Kana as "other" (Lo), neither whitespace nor punctuation, so a `**` run wrapped by CJK with punctuation-bounded content is not left/right-flanking and stays literal. muya has TWO inline-tokenization paths and both carry the same CJK-as- punctuation widening the legacy engine shipped: - Static / export path (marked@16): a `cjkEmStrong` tokenizer override that rebuilds marked's emStrong flanking regexes with CJK folded into the punctuation class and removed from the alphanumeric class, registered in getHighlightHtml and getClipboardHtml. Faithful copy of marked's emStrong body; rules are swapped in/out per-call so the shared tokenizer rules are never left mutated. - Live editor path (inlineRenderer): CJK widening added to canOpen/canCloseEmphasis in inlineRenderer/utils.ts, plus full code-point reading so the non-BMP CJK Ext-B surrogate-pair branch is live. CJK ranges (matching legacy CJK_REG): Hiragana+Katakana U+3040–U+30FF, CJK Ext-A U+3400–U+4DBF, CJK Unified U+4E00–U+9FFF, CJK Compatibility U+F900–U+FAFF, Hangul Syllables U+AC00–U+D7AF, Halfwidth Katakana U+FF66–U+FF9D, and CJK Ext-B U+20000–U+2A6DF (non-BMP). The widening is additive — it never bolds anything CommonMark accepts as non-emphasis — so the CommonMark 0.31 + GFM conformance suites are unchanged (1347/1347, no unexpected passes). Promotes the #4399 `it.fails` placeholders to passing `it` cases and adds live-editor-path coverage plus negative cases. Co-Authored-By: Claude Opus 4.8 (1M context) * test(muya): assert negative CJK flanking cases emit neither strong nor em The static/export-path negative cases in strongCjkFlanking.spec.ts only checked for via rendersStrong. Since #4307 widens the emphasis/strong flanking logic, a regression could surface as unexpected output while still passing a -only assertion. Add a rendersEm helper mirroring rendersStrong and assert NEITHER tag is produced for the negative cases. The live editor path already covers this: tokenizesEmphasis returns true for either a strong or em token, so its negative .toBe(false) already rejects both. Only the static path needed strengthening. Addresses Copilot review on #4401. Co-Authored-By: Claude Opus 4.8 (1M context) --------- Co-authored-by: Claude Opus 4.8 (1M context) --- packages/muya/src/inlineRenderer/utils.ts | 83 +++++- .../state/__tests__/strongCjkFlanking.spec.ts | 181 ++++++++----- .../utils/marked/extensions/cjkEmStrong.ts | 251 ++++++++++++++++++ .../muya/src/utils/marked/getClipboardHtml.ts | 5 + .../muya/src/utils/marked/getHighlightHtml.ts | 6 + 5 files changed, 462 insertions(+), 64 deletions(-) create mode 100644 packages/muya/src/utils/marked/extensions/cjkEmStrong.ts diff --git a/packages/muya/src/inlineRenderer/utils.ts b/packages/muya/src/inlineRenderer/utils.ts index 4335e666a0..80c4d90d93 100644 --- a/packages/muya/src/inlineRenderer/utils.ts +++ b/packages/muya/src/inlineRenderer/utils.ts @@ -65,6 +65,71 @@ export const WHITELIST_ATTRIBUTES = [ const UNICODE_WHITESPACE_REG = /^\s/; +// NON-STANDARD EXTENSION — a deliberate divergence from CommonMark, ported +// from the legacy muyajs tokenizer (packages/muyajs/lib/parser/utils.js). +// +// CommonMark §6.2 only counts Unicode whitespace and Unicode punctuation as +// emphasis flanking boundaries. CJK ideographs are Lo (Letter, other) — +// neither whitespace nor punctuation — so under a literal reading of the spec +// `中文**"加粗"**中文` MUST NOT open a strong run. But CJK scripts don't use +// spaces between words, so that denies emphasis to virtually any CJK paragraph +// that wraps the `**` run with punctuation (quotes, parentheses, brackets, …). +// Typora, VSCode markdownlint, Joplin and the legacy muyajs engine all widen +// the flanking check so CJK counts as a boundary; we match that here so the +// live editor (inlineRenderer) bolds these spans consistently with the +// marked-based static / export render path. +// +// The widening is ADDITIVE: CJK is only ever accepted as an extra boundary on +// top of the CommonMark whitespace/punctuation set, never used to reject +// emphasis CommonMark accepts — so spec-conformant Latin inputs are unchanged. +// +// Ranges (BMP via the first alternative; CJK Ext-B non-BMP via the surrogate +// pair in the second): +// U+3040–U+30FF Hiragana + Katakana +// U+3400–U+4DBF CJK Unified Ideographs Extension A +// U+4E00–U+9FFF CJK Unified Ideographs +// U+F900–U+FAFF CJK Compatibility Ideographs +// U+AC00–U+D7AF Hangul Syllables +// U+FF66–U+FF9D Halfwidth Katakana +// U+20000–U+2A6DF CJK Unified Ideographs Extension B (D840-D87F DC00-DFFF) +// +// Tracking: marktext/marktext#4307. +// eslint-disable-next-line regexp/no-obscure-range +const CJK_REG = /[぀-ヿ㐀-䶿一-鿿豈-﫿가-힯ｦ-ﾝ]|[\uD840-\uD87F][\uDC00-\uDFFF]/; + +// Extract the trailing Unicode code point of `s` as a 1- or 2-char string, or +// '' when `s` is empty. Bracket indexing / charAt return a single UTF-16 code +// unit, splitting a non-BMP code point into raw surrogate halves that never +// match PUNCTUATION_REG / CJK_REG / UNICODE_WHITESPACE_REG. Reading the full +// code point keeps CJK_REG's surrogate-pair branch live for Ext-B ideographs. +function lastCodePointChar(s: string): string { + if (!s) + return ''; + const len = s.length; + const lastUnit = s.charCodeAt(len - 1); + if (lastUnit >= 0xDC00 && lastUnit <= 0xDFFF && len >= 2) { + const prevUnit = s.charCodeAt(len - 2); + if (prevUnit >= 0xD800 && prevUnit <= 0xDBFF) + return s.slice(len - 2); + } + return s.charAt(len - 1); +} + +// Same idea at an arbitrary index. Returns undefined past the end so the +// existing `|| '\n'` / `UNICODE_WHITESPACE_REG.test(undefined)` semantics at +// callers are preserved verbatim. +function codePointCharAt(s: string, i: number): string | undefined { + if (i >= s.length) + return undefined; + const unit = s.charCodeAt(i); + if (unit >= 0xD800 && unit <= 0xDBFF && i + 1 < s.length) { + const next = s.charCodeAt(i + 1); + if (next >= 0xDC00 && next <= 0xDFFF) + return s.slice(i, i + 2); + } + return s.charAt(i); +} + function validWidthAndHeight(value: string) { if (!/^\d+$/.test(value)) return ''; @@ -159,8 +224,10 @@ function canOpenEmphasis(src: string, marker: string, pending: string) { if (pending.length > 0 && pending.charAt(pending.length - 1) === markerChar) return false; - const precededChar = pending.charAt(pending.length - 1) || '\n'; - const followedChar = src[marker.length]; + const precededChar = lastCodePointChar(pending) || '\n'; + // Past end of src → '' (matches neither whitespace nor punctuation), + // preserving the legacy `RegExp.test(undefined)` semantics type-safely. + const followedChar = codePointCharAt(src, marker.length) ?? ''; // not followed by Unicode whitespace, if (UNICODE_WHITESPACE_REG.test(followedChar)) return false; @@ -168,11 +235,14 @@ function canOpenEmphasis(src: string, marker: string, pending: string) { // and either (2a) not followed by a punctuation character, // or (2b) followed by a punctuation character and preceded by Unicode whitespace or a punctuation character. // For purposes of this definition, the beginning and the end of the line count as Unicode whitespace. + // CJK widening (see CJK_REG above) — additive: a preceding CJK character is + // accepted as a boundary on top of the CommonMark whitespace/punctuation set. if ( PUNCTUATION_REG.test(followedChar) && !( UNICODE_WHITESPACE_REG.test(precededChar) || PUNCTUATION_REG.test(precededChar) + || CJK_REG.test(precededChar) ) ) { return false; @@ -183,6 +253,7 @@ function canOpenEmphasis(src: string, marker: string, pending: string) { && !( UNICODE_WHITESPACE_REG.test(precededChar) || PUNCTUATION_REG.test(precededChar) + || CJK_REG.test(precededChar) ) ) { return false; @@ -192,19 +263,22 @@ function canOpenEmphasis(src: string, marker: string, pending: string) { } function canCloseEmphasis(src: string, offset: number, marker: string) { - const precededChar = src[offset - marker.length - 1]; - const followedChar = src[offset] || '\n'; + const precededChar = lastCodePointChar(src.substring(0, offset - marker.length)); + const followedChar = codePointCharAt(src, offset) || '\n'; // not preceded by Unicode whitespace, if (UNICODE_WHITESPACE_REG.test(precededChar)) return false; // either (2a) not preceded by a punctuation character, // or (2b) preceded by a punctuation character and followed by Unicode whitespace or a punctuation character. + // CJK widening: symmetric to canOpenEmphasis — a following CJK character is + // accepted as a boundary on top of the CommonMark whitespace/punctuation set. if ( PUNCTUATION_REG.test(precededChar) && !( UNICODE_WHITESPACE_REG.test(followedChar) || PUNCTUATION_REG.test(followedChar) + || CJK_REG.test(followedChar) ) ) { return false; @@ -215,6 +289,7 @@ function canCloseEmphasis(src: string, offset: number, marker: string) { && !( UNICODE_WHITESPACE_REG.test(followedChar) || PUNCTUATION_REG.test(followedChar) + || CJK_REG.test(followedChar) ) ) { return false; diff --git a/packages/muya/src/state/__tests__/strongCjkFlanking.spec.ts b/packages/muya/src/state/__tests__/strongCjkFlanking.spec.ts index 2722d920fb..d70e2fab47 100644 --- a/packages/muya/src/state/__tests__/strongCjkFlanking.spec.ts +++ b/packages/muya/src/state/__tests__/strongCjkFlanking.spec.ts @@ -1,6 +1,8 @@ // @vitest-environment happy-dom +import type { Token } from '../../inlineRenderer/types'; import { describe, expect, it } from 'vitest'; +import { tokenizer } from '../../inlineRenderer/lexer'; import { renderToStaticHTML } from '../renderToStaticHTML'; // Regression coverage ported from marktext#4307 (legacy desktop spec @@ -9,73 +11,132 @@ import { renderToStaticHTML } from '../renderToStaticHTML'; // recognised, even when the emphasised content begins/ends with a punctuation // character (a quote, bracket, or paren). // -// WHY THE CJK CASES FAIL ON @muyajs/core (documented engine gap): -// The new engine tokenises inline markdown with marked@16, which implements -// the CommonMark emphasis "flanking" rule literally. CommonMark classifies -// every character as whitespace, (Unicode) punctuation, or "other"; CJK -// ideographs and Hangul are "other". For a left-flanking `**` run, clause -// (2b) requires the character *before* the run to be whitespace or -// punctuation whenever the character *after* the run is punctuation. In +// THE ENGINE GAP, AND HOW IT IS NOW CLOSED: +// muya tokenises inline markdown through TWO paths, and CommonMark's +// emphasis "flanking" rule denies bold in both for CJK-bounded runs. The +// spec classifies every character as whitespace, (Unicode) punctuation, or +// "other"; CJK ideographs and Hangul are "other". For a left-flanking `**` +// run, clause (2b) requires the character *before* the run to be whitespace +// or punctuation whenever the character *after* the run is punctuation. In // `例子例子**"加粗"**例子例子` the char after the opening `**` is `"` -// (punctuation) and the char before it is `子` (a CJK ideograph → "other", -// neither whitespace nor punctuation), so the run is not left-flanking and -// marked emits the literal `**`. The same happens at the closing run. +// (punctuation) and the char before it is `子` (a CJK ideograph → "other"), +// so the run is not left-flanking and the `**` stays literal. // -// Legacy marktext shipped its own inline tokenizer (muyajs -// `lib/parser/render`) whose `canOpen/canCloseEmphasis` flanking helpers -// treat CJK characters as punctuation, so `**` adjacent to a CJK char with -// punctuation-bounded inner content opens/closes emphasis. marked has no -// such patch, and fixing it requires either patching the dependency or -// shipping a custom inline-emphasis tokenizer extension — out of scope for a -// tests-only fidelity-verification PR. The CJK cases below assert the -// CORRECT (legacy) behavior and are wrapped in `it.fails`, so: -// - the suite stays green while the gap exists, AND -// - the moment the engine starts recognising these (e.g. a marked upgrade -// or a flanking patch lands) the `it.fails` flips red, forcing this file -// to be promoted to a plain `it`. Fidelity can only go up. +// Legacy marktext (muyajs) shipped a custom inline tokenizer whose +// `canOpen/canCloseEmphasis` flanking helpers treat CJK as punctuation. We +// restore that, additively, in both of muya's paths: +// - Static / export path — marked@16, via the `cjkEmStrong` tokenizer +// override registered in `getHighlightHtml` / `getClipboardHtml`. +// - Live editor path — muya's own inline lexer, via the CJK widening +// in `inlineRenderer/utils.ts` (`canOpen/canCloseEmphasis`). // -// This gap is documented in the PR body for #4307 follow-up. +// The widening is additive: it never bolds anything CommonMark accepts as +// non-emphasis, so the CommonMark / GFM conformance suites are unaffected. +// +// See marktext/marktext#4307. + +const STATIC_OPTIONS = { + footnote: false, + math: false, + superSubScript: false, + isGitlabCompatibilityEnabled: false, + frontMatter: false, + sanitize: false, +} as const; + +// The four #4307 examples — CJK ideographs, Kana, Hangul, and a non-BMP CJK +// Ext-B example — that legacy bolded and muya now bolds too. +const CJK_CASES = [ + '例子例子**"加粗"**例子例子', // CJK ideographs, fullwidth quotes + '日本語**(強調)**日本語', // Kana/ideographs, fullwidth parens + '한국어**[강조]**한국어', // Hangul syllables, brackets + // Non-BMP CJK (CJK Ext-B): 𠀀 is U+20000, stored as a surrogate pair. The + // flanking boundary check must read the full code point. + '𠀀𠀁**"加粗"**𠀀𠀁', +]; + +// Cases that already work — they lock in the pre-existing behavior so the fix +// can't regress them. Each emphasised run is bounded by a CJK ideograph or +// whitespace on the inner side, satisfying flanking without the CJK widening. +const SANITY_CASES = [ + 'before **"normal"** after', + 'before**normal**after', + '中文**加粗**中文', +]; + +// Cases that MUST NOT bold — the additive CJK widening must leave these as +// CommonMark rejects them. +const NEGATIVE_CASES = [ + 'a * foo bar*', // space after opening `*` — not left-flanking + 'a_foo bar_', // intraword `_` emphasis is disallowed + '*(*foo)', // inner `(` makes the run both-flanking, can't open +]; function rendersStrong(src: string): boolean { - const html = renderToStaticHTML(src, { sanitize: false }); - return //.test(html); + return //.test(renderToStaticHTML(src, STATIC_OPTIONS)); } -describe('strong emphasis with CJK boundaries (#4307)', () => { - // Cases that already work on @muyajs/core — they lock in the pre-existing - // behavior so any fix to the CJK gap can't regress them. Each emphasised - // run here is bounded by a CJK ideograph or whitespace on the inner side, - // so the flanking rule is satisfied without the legacy CJK-as-punctuation - // patch. - const sanityCases = [ - 'before **"normal"** after', - 'before**normal**after', - '中文**加粗**中文', - ]; - - for (const src of sanityCases) { - it(`recognises strong in: ${src}`, () => { - expect(rendersStrong(src)).toBe(true); - }); - } +function rendersEm(src: string): boolean { + return //.test(renderToStaticHTML(src, STATIC_OPTIONS)); +} - // CJK-boundary cases. These are the #4307 regression cases the legacy - // marktext tokenizer fixed. @muyajs/core (via marked) does NOT recognise - // them — documented engine gap (see file header). The assertion states the - // CORRECT expected behavior; `it.fails` keeps the suite green until the gap - // is closed, at which point it must be converted to a plain `it`. - const cjkGapCases = [ - '例子例子**"加粗"**例子例子', - '日本語**(強調)**日本語', - '한국어**[강조]**한국어', - // Non-BMP CJK (CJK Ext-B): 𠀀 is U+20000, stored as a surrogate pair. - // The flanking boundary check must read the full code point. - '𠀀𠀁**"加粗"**𠀀𠀁', - ]; - - for (const src of cjkGapCases) { - it.fails(`[GAP #4307] should recognise strong in CJK context: ${src}`, () => { - expect(rendersStrong(src)).toBe(true); - }); +function collectTypes(tokens: Token[], out: string[] = []): string[] { + for (const token of tokens) { + out.push(token.type); + if ('children' in token && Array.isArray(token.children)) + collectTypes(token.children, out); } + return out; +} + +function tokenizesEmphasis(src: string): boolean { + const types = collectTypes(tokenizer(src, { hasBeginRules: false }) as Token[]); + return types.includes('strong') || types.includes('em'); +} + +describe('strong emphasis with CJK boundaries (#4307)', () => { + describe('static / export path — renderToStaticHTML (marked@16)', () => { + for (const src of SANITY_CASES) { + it(`recognises strong in: ${src}`, () => { + expect(rendersStrong(src)).toBe(true); + }); + } + + for (const src of CJK_CASES) { + it(`recognises strong in CJK context: ${src}`, () => { + expect(rendersStrong(src)).toBe(true); + }); + } + + for (const src of NEGATIVE_CASES) { + it(`does not bold or italicise: ${src}`, () => { + // Assert NEITHER NOR : because this PR widens the + // emphasis/strong flanking logic, a regression could surface as + // unexpected output while still passing a -only + // check. Guard both tags so the negative cases stay meaningful. + expect(rendersStrong(src), src).toBe(false); + expect(rendersEm(src), src).toBe(false); + }); + } + }); + + describe('live editor path — inlineRenderer tokenizer', () => { + for (const src of SANITY_CASES) { + it(`tokenizes strong/em in: ${src}`, () => { + expect(tokenizesEmphasis(src), src).toBe(true); + }); + } + + for (const src of CJK_CASES) { + it(`tokenizes strong/em in CJK context: ${src}`, () => { + expect(tokenizesEmphasis(src), src).toBe(true); + }); + } + + for (const src of NEGATIVE_CASES) { + it(`does not tokenize strong/em in: ${src}`, () => { + expect(tokenizesEmphasis(src), src).toBe(false); + }); + } + }); }); diff --git a/packages/muya/src/utils/marked/extensions/cjkEmStrong.ts b/packages/muya/src/utils/marked/extensions/cjkEmStrong.ts new file mode 100644 index 0000000000..dda024e514 --- /dev/null +++ b/packages/muya/src/utils/marked/extensions/cjkEmStrong.ts @@ -0,0 +1,251 @@ +import type { MarkedExtension, TokenizerObject, Tokens } from 'marked'; + +// NON-STANDARD EXTENSION — a deliberate divergence from CommonMark, ported +// from the legacy muyajs tokenizer (packages/muyajs/lib/parser/utils.js). +// +// CommonMark §6.2 only counts Unicode whitespace and Unicode punctuation as +// emphasis flanking boundaries. CJK ideographs / Hangul / Kana are Lo +// (Letter, other) — neither whitespace nor punctuation — so under a literal +// reading of the spec `中文**"加粗"**中文` MUST NOT open a strong run. marked@16 +// is spec-conformant here and refuses to bold it. +// +// But CJK scripts do not use spaces between words, so that rule denies +// emphasis to virtually any CJK paragraph that wraps the `**` run with +// punctuation (quotes, parentheses, brackets, …). Typora, VSCode +// markdownlint, Joplin and most CJK-oriented Markdown tools — and the legacy +// muyajs engine MarkText shipped — widen the flanking check so CJK counts as +// a boundary. This extension restores that behavior for the marked-based +// static / export render path. +// +// The widening is ADDITIVE: it only ever lets emphasis open/close where +// CommonMark refused, never the reverse, so spec-conformant Latin inputs +// parse identically (verified by the CommonMark / GFM conformance suites). +// +// Tracking: marktext/marktext#4307. + +// CJK ranges treated as punctuation for flanking (matches legacy CJK_REG): +// U+3040–U+30FF Hiragana + Katakana +// U+3400–U+4DBF CJK Unified Ideographs Extension A +// U+4E00–U+9FFF CJK Unified Ideographs +// U+F900–U+FAFF CJK Compatibility Ideographs +// U+AC00–U+D7AF Hangul Syllables +// U+FF66–U+FF9D Halfwidth Katakana +const CJK = '\\u3040-\\u30FF\\u3400-\\u4DBF\\u4E00-\\u9FFF\\uF900-\\uFAFF\\uAC00-\\uD7AF\\uFF66-\\uFF9D'; +// CJK Unified Ideographs Extension B (non-BMP, U+20000–U+2A6DF) — matched as +// a full code point under the `u` flag inside the delimiter regexes. +const CJK_NON_BMP = '\\u{20000}-\\u{2A6DF}'; +// A lone low surrogate can only be the trailing UTF-16 unit of a non-BMP code +// point. marked's lexer hands `emStrong` a single-unit `prevChar` (the last +// code unit of the preceding text token), so for an Ext-B ideograph that's a +// lone low surrogate. Accept it in the single-unit boundary test below. +const LOW_SURROGATE = '\\uDC00-\\uDFFF'; + +// Rebuild marked@16's emphasis flanking regexes (marked.cjs `emStrongLDelim`, +// `emStrongRDelimAst`, `emStrongRDelimUnd`, `punctuation`, and the `other` +// `unicodeAlphaNumeric` rule) with CJK folded into the punctuation class and +// removed from the alphanumeric class. Sources mirror marked@16.4.2 verbatim +// apart from substituting the CJK-widened character classes for `punct`, +// `punctSpace`, `notPunctSpace`. Under the `u` flag a single character class +// can mix BMP ranges with the non-BMP Ext-B range, so no surrogate-pair +// alternation is needed inside these classes. +const PUNCT = `[\\p{P}\\p{S}${CJK}${CJK_NON_BMP}]`; +const PUNCT_SPACE = `[\\s\\p{P}\\p{S}${CJK}${CJK_NON_BMP}]`; +const NOT_PUNCT_SPACE = `[^\\s\\p{P}\\p{S}${CJK}${CJK_NON_BMP}]`; + +// These three patterns are verbatim copies of marked@16.4.2's emStrong +// delimiter regexes (only the punct/punctSpace/notPunctSpace classes are +// CJK-widened). The non-capturing groups and lazy quantifiers are marked's own +// shape — keeping them identical is the whole point, so the regexp/* lints that +// would "simplify" them are disabled to preserve upstream fidelity. +/* eslint-disable regexp/no-useless-non-capturing-group, regexp/no-useless-lazy */ +const emStrongLDelim = new RegExp( + `^(?:\\*+(?:((?!\\*)${PUNCT})|[^\\s*]))|^_+(?:((?!_)${PUNCT})|([^\\s_]))`, + 'u', +); +const emStrongRDelimAst = new RegExp( + `^[^_*]*?__[^_*]*?\\*[^_*]*?(?=__)|[^*]+(?=[^*])|(?!\\*)${PUNCT}(\\*+)(?=\\s|$)|${NOT_PUNCT_SPACE}(\\*+)(?!\\*)(?=${PUNCT_SPACE}|$)|(?!\\*)${PUNCT_SPACE}(\\*+)(?=${NOT_PUNCT_SPACE})|\\s(\\*+)(?!\\*)(?=${PUNCT})|(?!\\*)${PUNCT}(\\*+)(?!\\*)(?=${PUNCT})|${NOT_PUNCT_SPACE}(\\*+)(?=${NOT_PUNCT_SPACE})`, + 'gu', +); +const emStrongRDelimUnd = new RegExp( + `^[^_*]*?\\*\\*[^_*]*?_[^_*]*?(?=\\*\\*)|[^_]+(?=[^_])|(?!_)${PUNCT}(_+)(?=\\s|$)|${NOT_PUNCT_SPACE}(_+)(?!_)(?=${PUNCT_SPACE}|$)|(?!_)${PUNCT_SPACE}(_+)(?=${NOT_PUNCT_SPACE})|\\s(_+)(?!_)(?=${PUNCT})|(?!_)${PUNCT}(_+)(?!_)(?=${PUNCT})`, + 'gu', +); +/* eslint-enable regexp/no-useless-non-capturing-group, regexp/no-useless-lazy */ +// `punctuation` and `unicodeAlphaNumeric` are tested against the single-unit +// `prevChar`, so they additionally fold in a lone low surrogate (the trailing +// half of an Ext-B ideograph) — see LOW_SURROGATE above. +const punctuation = new RegExp( + `^(?![*_])[\\s\\p{P}\\p{S}${CJK}${CJK_NON_BMP}${LOW_SURROGATE}]`, + 'u', +); +// "Alphanumeric but not CJK" — a set difference, so it stays a lookahead-then- +// class rather than a single character class. Removing CJK from the +// alphanumeric set is what lets a CJK-preceded `**punct…` run open emphasis +// (marked's emStrong rejects an alphanumeric-preceded both-flanking opener). +const unicodeAlphaNumeric = new RegExp( + `(?![${CJK}${CJK_NON_BMP}${LOW_SURROGATE}])[\\p{L}\\p{N}]`, + 'u', +); + +interface IEmStrongRules { + emStrongLDelim: RegExp; + emStrongRDelimAst: RegExp; + emStrongRDelimUnd: RegExp; + punctuation: RegExp; +} + +// `this` inside a marked TokenizerObject method is the internal `_Tokenizer`, +// which exposes `.rules` and `.lexer`. marked doesn't export that class, so we +// describe the slice we touch. +interface ITokenizerThis { + rules: { + inline: IEmStrongRules; + other: { unicodeAlphaNumeric: RegExp }; + }; + lexer: { inlineTokens: (src: string) => Tokens.Generic[] }; +} + +/** + * A faithful re-implementation of marked@16's `Tokenizer.emStrong`, identical + * to the upstream body apart from swapping in the CJK-widened flanking rules + * for the duration of the call. The rules are restored in a `finally` so the + * shared `_Tokenizer.rules` object is never left mutated for other tokenizers. + */ +// eslint-disable-next-line complexity -- verbatim copy of marked's emStrong body +function cjkAwareEmStrong( + this: ITokenizerThis, + src: string, + maskedSrc: string, + prevChar = '', +): Tokens.Em | Tokens.Strong | undefined { + const inline = this.rules.inline; + const other = this.rules.other; + const saved = { + emStrongLDelim: inline.emStrongLDelim, + emStrongRDelimAst: inline.emStrongRDelimAst, + emStrongRDelimUnd: inline.emStrongRDelimUnd, + punctuation: inline.punctuation, + unicodeAlphaNumeric: other.unicodeAlphaNumeric, + }; + + inline.emStrongLDelim = emStrongLDelim; + inline.emStrongRDelimAst = emStrongRDelimAst; + inline.emStrongRDelimUnd = emStrongRDelimUnd; + inline.punctuation = punctuation; + other.unicodeAlphaNumeric = unicodeAlphaNumeric; + + try { + let match = inline.emStrongLDelim.exec(src); + if (!match) + return undefined; + + // CommonMark §6.4: a `**`/`__` run that is both left- and right-flanking + // can only open emphasis when preceded by punctuation. With CJK now in + // the punctuation class, `unicodeAlphaNumeric` excludes CJK so this + // guard no longer rejects CJK-preceded openers. + if (match[3] && prevChar.match(other.unicodeAlphaNumeric)) + return undefined; + + const nextChar = match[1] || match[2] || ''; + + if (!nextChar || !prevChar || inline.punctuation.exec(prevChar)) { + // Unicode codepoints can be 1 or 2 chars wide. + const lLength = [...match[0]].length - 1; + let rDelim; + let rLength; + let delimTotal = lLength; + let midDelimTotal = 0; + + const endReg + = match[0][0] === '*' + ? inline.emStrongRDelimAst + : inline.emStrongRDelimUnd; + endReg.lastIndex = 0; + + // Clip maskedSrc to the opener so the right-delimiter scan starts + // immediately after the opening run (marked passes the masked, + // already-skipped variant of `src`). + maskedSrc = maskedSrc.slice(-1 * src.length + lLength); + + // eslint-disable-next-line no-cond-assign + while ((match = endReg.exec(maskedSrc)) != null) { + rDelim + = match[1] + || match[2] + || match[3] + || match[4] + || match[5] + || match[6]; + + if (!rDelim) + continue; + + rLength = [...rDelim].length; + + if (match[3] || match[4]) { + // Found another opener — push the requirement deeper. + delimTotal += rLength; + continue; + } + else if ((match[5] || match[6]) && lLength % 3 && !((lLength + rLength) % 3)) { + // Rule of 3 — a delimiter run usable as both opener and + // closer can't close here. + midDelimTotal += rLength; + continue; + } + + delimTotal -= rLength; + if (delimTotal > 0) + continue; + + rLength = Math.min(rLength, rLength + delimTotal + midDelimTotal); + + const lastCharLength = [...match[0]][0].length; + const raw = src.slice(0, lLength + match.index + lastCharLength + rLength); + + if (Math.min(lLength, rLength) % 2) { + const text = raw.slice(1, -1); + return { + type: 'em', + raw, + text, + tokens: this.lexer.inlineTokens(text), + } as Tokens.Em; + } + + const text = raw.slice(2, -2); + return { + type: 'strong', + raw, + text, + tokens: this.lexer.inlineTokens(text), + } as Tokens.Strong; + } + } + + return undefined; + } + finally { + inline.emStrongLDelim = saved.emStrongLDelim; + inline.emStrongRDelimAst = saved.emStrongRDelimAst; + inline.emStrongRDelimUnd = saved.emStrongRDelimUnd; + inline.punctuation = saved.punctuation; + other.unicodeAlphaNumeric = saved.unicodeAlphaNumeric; + } +} + +/** + * marked extension that makes the emphasis/strong flanking check treat CJK + * characters as punctuation. Register via `marked.use(cjkEmStrongExtension())` + * on every Marked instance that renders inline emphasis. + */ +export default function cjkEmStrongExtension(): MarkedExtension { + // marked's TokenizerObject.emStrong has `this: _Tokenizer` (a class marked + // doesn't export). Our ITokenizerThis describes the slice we touch; the + // cast bridges to marked's public TokenizerObject type. + const tokenizer = { + emStrong: cjkAwareEmStrong as unknown, + } as TokenizerObject; + + return { tokenizer }; +} diff --git a/packages/muya/src/utils/marked/getClipboardHtml.ts b/packages/muya/src/utils/marked/getClipboardHtml.ts index 036dbc2e87..0d9562f376 100644 --- a/packages/muya/src/utils/marked/getClipboardHtml.ts +++ b/packages/muya/src/utils/marked/getClipboardHtml.ts @@ -1,5 +1,6 @@ import type { ILexOption } from './types'; import { Marked } from 'marked'; +import cjkEmStrongExtension from './extensions/cjkEmStrong'; import mathExtension from './extensions/math'; import superSubScriptExtension from './extensions/superSubscript'; import fm, { frontMatterRender } from './frontMatter'; @@ -22,6 +23,10 @@ export function getClipBoardHtml(src: string, options: ILexOption = {}) { walkTokens: walkTokens({ math, isGitlabCompatibilityEnabled }), }); + // CJK-as-punctuation emphasis flanking (marktext/marktext#4307); keeps the + // clipboard HTML consistent with the static / export render path. + marked.use(cjkEmStrongExtension()); + if (math) { marked.use( mathExtension({ diff --git a/packages/muya/src/utils/marked/getHighlightHtml.ts b/packages/muya/src/utils/marked/getHighlightHtml.ts index ae7f3707c4..e6fe0811ff 100644 --- a/packages/muya/src/utils/marked/getHighlightHtml.ts +++ b/packages/muya/src/utils/marked/getHighlightHtml.ts @@ -2,6 +2,7 @@ import type { ILexOption } from './types'; import { Marked } from 'marked'; import { markedHighlight } from 'marked-highlight'; import Prism from 'prismjs'; +import cjkEmStrongExtension from './extensions/cjkEmStrong'; import emojiExtension from './extensions/emoji'; import footnoteExtension from './extensions/footnote'; import mathExtension from './extensions/math'; @@ -49,6 +50,11 @@ export function getHighlightHtml(src: string, options: ILexOption = {}) { walkTokens: walkTokens({ math, isGitlabCompatibilityEnabled }), }); + // Treat CJK characters as punctuation for emphasis/strong flanking so + // `中文**"加粗"**中文` bolds (marktext/marktext#4307). Additive override — + // never regresses spec-conformant Latin emphasis. + marked.use(cjkEmStrongExtension()); + marked.use(emojiExtension({ isRenderEmoji: true })); if (math) { From 0706ee59f07f0beacb47ac7901d35da8ab9cd7c4 Mon Sep 17 00:00:00 2001 From: Ran Luo Date: Mon, 8 Jun 2026 17:54:35 +0800 Subject: [PATCH 2/5] feat(desktop): consume @muyajs/core in util files (markdownToHtml/pdf/dompurify/printService/sourceCode/icon) (#4402) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(desktop): wire @muyajs/core into the desktop package Begin migrating the desktop renderer off the legacy `@marktext/muyajs` engine and onto the TypeScript rewrite published as `@muyajs/core` (packages/muya). This first step only adds the dependency and the type plumbing; the `muya` alias (→ packages/muyajs) is left intact so editor.vue keeps working while the migration proceeds file by file. - Add `@muyajs/core: workspace:*` to packages/desktop dependencies. - `@muyajs/core` ships no built `lib/types`; its package `exports` map points `.` at `./src/index.ts`, so vue-tsc would otherwise descend into the entire muya source tree and report errors that only exist because muya's own `src/types/global.d.ts` globals aren't loaded under the desktop program. Add a hand-written `src/types/muya-core.d.ts` declaring just the surface the desktop consumes, and a `paths` entry redirecting type resolution to it. Runtime resolution still goes through the package `exports` map via Vite/electron-vite (verified by build:unpack and a vitest import smoke test). Co-Authored-By: Claude Opus 4.8 (1M context) * feat(desktop): consume @muyajs/core in markdownToHtml/pdf/sourceCode Swap the directly-equivalent muyajs imports for their `@muyajs/core` counterparts: - util/markdownToHtml.ts: `new ExportHtml(md).renderHtml()` → `new MarkdownToHtml(md).renderHtml()`. The new `renderHtml()` already wraps its output in `
…
` (muyajs did not), so the desktop no longer adds the wrapper — output is byte-identical. - util/pdf.ts: `escapeHTML`/`unescapeHTML` now come from `@muyajs/core` (identical implementations). The `Slugger` import is intentionally left on muyajs: the TOC anchors built here must match the heading `id`s emitted by the muyajs export renderer (`Muya#exportStyledHTML`, still used by editor.vue), which use the SAME Slugger. Swapping to @muyajs/core's `generateGithubSlug` (a different algorithm, no dedup/unicode downcoding) would break in-document TOC links; it migrates together with the editor.vue swap. - components/editorWithTabs/sourceCode.vue: `wordCount` now comes from `@muyajs/core`. Same `{ word, paragraph, character, all }` return shape; the token-split regex differs only cosmetically (`/\s+/` vs `/[\s\n]+/`, equivalent since `\s` covers `\n`). Co-Authored-By: Claude Opus 4.8 (1M context) * refactor(desktop): vendor dompurify directly in util/dompurify Replace the `muya/lib/utils/dompurify` import (whose default export was simply `DOMPurify.sanitize`) with `dompurify` directly — already a direct desktop dependency. `@muyajs/core`'s exported `sanitize(html, opts, disableHtml)` has a different signature (it escapes HTML internally), whereas the desktop needs the RAW sanitizer that does not escape (pdf.ts escapes/unescapes around it). Behavior is identical to the legacy path. Cast through `unknown` to bridge DOMPurify's `string | TrustedHTML` overload union; both configs set `RETURN_TRUSTED_TYPE: false`, so the result is always a string. Co-Authored-By: Claude Opus 4.8 (1M context) * refactor(desktop): use @marktext/file-icons directly in sideBar/icon Drop the `muya/lib/ui/fileIcons` import in favor of the underlying `@marktext/file-icons` package (already a direct desktop dependency). The muyajs wrapper only added a `getClassByName(name)` helper around the package's `matchName(name)?.getClass(0, false)`; inline that helper in the component and import the package's CSS (`build/index.css`) directly, as the wrapper did. Add an ambient module declaration for `@marktext/file-icons` (it ships no types) to shims.d.ts. Co-Authored-By: Claude Opus 4.8 (1M context) * docs(desktop): document printService getImageInfo migration blocker services/printService.ts calls `getImageInfo(rawSrc: string).src` to normalise an 's src attribute into a displayable URL before printing/PDF export (GH#678). `@muyajs/core` has no behaviour-equivalent string helper for this: - its `getImageInfo(image: HTMLElement)` takes a DOM element, not a string; - its `getImageSrc(src: string)` double-prefixes already-resolved `file://` URLs (`file://file://…`) and blanks `data:` URLs — and the muyajs export renderer already emits absolute `file://` srcs into the HTML this consumes, so it would regress every image. Keep the muyajs import for now and document why; it migrates once the export render path (editor.vue / Muya#exportStyledHTML) moves to @muyajs/core. No behavioral change. Co-Authored-By: Claude Opus 4.8 (1M context) --------- Co-authored-by: Claude Opus 4.8 (1M context) --- packages/desktop/package.json | 1 + .../components/editorWithTabs/sourceCode.vue | 2 +- .../renderer/src/components/sideBar/icon.vue | 15 ++++++-- .../src/renderer/src/services/printService.ts | 10 ++++++ .../src/renderer/src/util/dompurify.ts | 11 ++++-- .../src/renderer/src/util/markdownToHtml.ts | 7 ++-- packages/desktop/src/renderer/src/util/pdf.ts | 11 +++++- packages/desktop/src/types/muya-core.d.ts | 34 +++++++++++++++++++ packages/desktop/src/types/shims.d.ts | 12 +++++++ packages/desktop/tsconfig.base.json | 1 + pnpm-lock.yaml | 3 ++ 11 files changed, 97 insertions(+), 10 deletions(-) create mode 100644 packages/desktop/src/types/muya-core.d.ts diff --git a/packages/desktop/package.json b/packages/desktop/package.json index 6b63f55180..c67f1e36d7 100644 --- a/packages/desktop/package.json +++ b/packages/desktop/package.json @@ -59,6 +59,7 @@ "@hfelix/electron-localshortcut": "^4.0.1", "@marktext/file-icons": "^1.0.6", "@marktext/muyajs": "workspace:*", + "@muyajs/core": "workspace:*", "@popperjs/core": "^2.11.8", "@vscode/ripgrep": "^1.18.0", "arg": "^5.0.2", diff --git a/packages/desktop/src/renderer/src/components/editorWithTabs/sourceCode.vue b/packages/desktop/src/renderer/src/components/editorWithTabs/sourceCode.vue index 58d85dc3c7..f9fbeea66c 100644 --- a/packages/desktop/src/renderer/src/components/editorWithTabs/sourceCode.vue +++ b/packages/desktop/src/renderer/src/components/editorWithTabs/sourceCode.vue @@ -11,7 +11,7 @@ import { useEditorStore } from '@/store/editor' import { usePreferencesStore } from '@/store/preferences' import { storeToRefs } from 'pinia' import codeMirror, { setCursorAtFirstLine, setTextDirection } from '../../codeMirror' -import { wordCount as getWordCount } from 'muya/lib/utils' +import { wordCount as getWordCount } from '@muyajs/core' import { adjustCursor } from '../../util' import bus from '../../bus' import { oneDarkThemes, railscastsThemes } from '@/config' diff --git a/packages/desktop/src/renderer/src/components/sideBar/icon.vue b/packages/desktop/src/renderer/src/components/sideBar/icon.vue index d9f6764833..76efa467bb 100644 --- a/packages/desktop/src/renderer/src/components/sideBar/icon.vue +++ b/packages/desktop/src/renderer/src/components/sideBar/icon.vue @@ -1,19 +1,28 @@