From 6048bda6191bb12569c390b4463bb75747ad7690 Mon Sep 17 00:00:00 2001 From: Lazizbek Ergashev Date: Wed, 2 Sep 2026 04:36:43 +0500 Subject: [PATCH 01/11] fix(extension-table): escape pipe characters in serialized table cells --- .../2026-09-02-table-cell-pipe-escape.md | 5 +++ .../__tests__/tableMarkdown.spec.ts | 31 +++++++++++++++++++ .../src/table/utilities/markdown.ts | 17 +++++++--- 3 files changed, 48 insertions(+), 5 deletions(-) create mode 100644 .changeset/2026-09-02-table-cell-pipe-escape.md diff --git a/.changeset/2026-09-02-table-cell-pipe-escape.md b/.changeset/2026-09-02-table-cell-pipe-escape.md new file mode 100644 index 0000000000..1a40a7e1c9 --- /dev/null +++ b/.changeset/2026-09-02-table-cell-pipe-escape.md @@ -0,0 +1,5 @@ +--- +'@tiptap/extension-table': patch +--- + +Table cells exported to Markdown now escape literal pipe characters, so the cell content survives when the output is read back. diff --git a/packages/extension-table/__tests__/tableMarkdown.spec.ts b/packages/extension-table/__tests__/tableMarkdown.spec.ts index ec2d45eceb..b9714a8008 100644 --- a/packages/extension-table/__tests__/tableMarkdown.spec.ts +++ b/packages/extension-table/__tests__/tableMarkdown.spec.ts @@ -298,3 +298,34 @@ describe('escapeTableCellPipes — pipe escaping inside code spans', () => { expect(escapeTableCellPipes('| `|\\||` |')).toBe('| `\\|\\|\\|` |') }) }) + +describe('table markdown — pipe escaping when serializing cells', () => { + const manager = new MarkdownManager({ + extensions: [Document, Paragraph, Text, Code, TableKit], + }) + + it('escapes a bare pipe in cell text', () => { + const markdown = '| Col | Value |\n| --- | --- |\n| a | uses \\| pipe |' + const serialized = manager.serialize(manager.parse(markdown)) + + expect(serialized).toContain('uses \\| pipe') + }) + + it('round trips a cell with an escaped pipe', () => { + const markdown = '| Col | Value |\n| --- | --- |\n| a | uses \\| pipe |' + const pass1 = manager.serialize(manager.parse(markdown)) + const pass2 = manager.serialize(manager.parse(pass1)) + + expect(pass2).toBe(pass1) + + const cell = manager.parse(pass1).content?.[0]?.content?.[1]?.content?.[1] + expect(cell?.content?.[0]?.content?.[0]?.text).toBe('uses | pipe') + }) + + it('escapes pipes inside a code span in a cell', () => { + const markdown = '| H |\n| - |\n| `a || b` |' + const serialized = manager.serialize(manager.parse(markdown)) + + expect(serialized).toContain('`a \\|\\| b`') + }) +}) diff --git a/packages/extension-table/src/table/utilities/markdown.ts b/packages/extension-table/src/table/utilities/markdown.ts index 0928f66a81..5bc1774b47 100644 --- a/packages/extension-table/src/table/utilities/markdown.ts +++ b/packages/extension-table/src/table/utilities/markdown.ts @@ -85,6 +85,11 @@ function collapseWhitespace(s: string) { return (s || '').replace(/\s+/g, ' ').trim() } +function escapeCellPipes(text: string) { + // Stepping over backslash escapes keeps an already escaped pipe untouched. + return text.replace(/\\.|\|/g, match => (match === '|' ? '\\|' : match)) +} + export function renderTableToMarkdown( node: JSONContent, h: MarkdownRendererHelpers, @@ -120,11 +125,13 @@ export function renderTableToMarkdown( // Cells have to stay on a single line, so line breaks become
tags. // The parser already turns
back into hard breaks, so this round trips. - const text = collapseWhitespace( - raw - .split(cellSep) - .join('\n') - .replace(/[ \t]*\r?\n[ \t]*/g, '
'), + const text = escapeCellPipes( + collapseWhitespace( + raw + .split(cellSep) + .join('\n') + .replace(/[ \t]*\r?\n[ \t]*/g, '
'), + ), ) const isHeader = cellNode.type === 'tableHeader' const align = normalizeTableCellAlignFromAttributes(cellNode.attrs) From aa5307a7a8e739156d5a882e0b9dc82faa6c18b9 Mon Sep 17 00:00:00 2001 From: Lazizbek Ergashev Date: Wed, 2 Sep 2026 04:43:34 +0500 Subject: [PATCH 02/11] refactor(extension-table): clarify pipe escaping helper name --- .../__tests__/tableMarkdown.spec.ts | 16 ++++++++-------- .../src/table/utilities/markdown.ts | 17 ++++++++--------- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/packages/extension-table/__tests__/tableMarkdown.spec.ts b/packages/extension-table/__tests__/tableMarkdown.spec.ts index b9714a8008..c4d042a18a 100644 --- a/packages/extension-table/__tests__/tableMarkdown.spec.ts +++ b/packages/extension-table/__tests__/tableMarkdown.spec.ts @@ -304,27 +304,27 @@ describe('table markdown — pipe escaping when serializing cells', () => { extensions: [Document, Paragraph, Text, Code, TableKit], }) + const markdown = '| Col | Value |\n| --- | --- |\n| a | uses \\| pipe |' + it('escapes a bare pipe in cell text', () => { - const markdown = '| Col | Value |\n| --- | --- |\n| a | uses \\| pipe |' const serialized = manager.serialize(manager.parse(markdown)) expect(serialized).toContain('uses \\| pipe') }) it('round trips a cell with an escaped pipe', () => { - const markdown = '| Col | Value |\n| --- | --- |\n| a | uses \\| pipe |' - const pass1 = manager.serialize(manager.parse(markdown)) - const pass2 = manager.serialize(manager.parse(pass1)) + const serialized = manager.serialize(manager.parse(markdown)) + const reparsed = manager.parse(serialized) - expect(pass2).toBe(pass1) + expect(manager.serialize(reparsed)).toBe(serialized) - const cell = manager.parse(pass1).content?.[0]?.content?.[1]?.content?.[1] + const cell = reparsed.content?.[0]?.content?.[1]?.content?.[1] expect(cell?.content?.[0]?.content?.[0]?.text).toBe('uses | pipe') }) it('escapes pipes inside a code span in a cell', () => { - const markdown = '| H |\n| - |\n| `a || b` |' - const serialized = manager.serialize(manager.parse(markdown)) + const codeSpanMarkdown = '| H |\n| - |\n| `a || b` |' + const serialized = manager.serialize(manager.parse(codeSpanMarkdown)) expect(serialized).toContain('`a \\|\\| b`') }) diff --git a/packages/extension-table/src/table/utilities/markdown.ts b/packages/extension-table/src/table/utilities/markdown.ts index 5bc1774b47..70c4ae96f6 100644 --- a/packages/extension-table/src/table/utilities/markdown.ts +++ b/packages/extension-table/src/table/utilities/markdown.ts @@ -85,8 +85,8 @@ function collapseWhitespace(s: string) { return (s || '').replace(/\s+/g, ' ').trim() } -function escapeCellPipes(text: string) { - // Stepping over backslash escapes keeps an already escaped pipe untouched. +function escapePipesInCellText(text: string) { + // Keep already escaped pipes as they are return text.replace(/\\.|\|/g, match => (match === '|' ? '\\|' : match)) } @@ -125,14 +125,13 @@ export function renderTableToMarkdown( // Cells have to stay on a single line, so line breaks become
tags. // The parser already turns
back into hard breaks, so this round trips. - const text = escapeCellPipes( - collapseWhitespace( - raw - .split(cellSep) - .join('\n') - .replace(/[ \t]*\r?\n[ \t]*/g, '
'), - ), + const singleLine = collapseWhitespace( + raw + .split(cellSep) + .join('\n') + .replace(/[ \t]*\r?\n[ \t]*/g, '
'), ) + const text = escapePipesInCellText(singleLine) const isHeader = cellNode.type === 'tableHeader' const align = normalizeTableCellAlignFromAttributes(cellNode.attrs) From 504260bb19a9ea4dcae5d1c960593a2f3f8cc749 Mon Sep 17 00:00:00 2001 From: Solomon Date: Thu, 3 Sep 2026 09:26:16 +0530 Subject: [PATCH 03/11] perf(extension-list): stop scanning the whole document for every ordered list Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01TgWFnwc9wduBkoq2RTpTJR --- ...09-03-ordered-list-markdown-linear-scan.md | 5 ++ .../orderedListLargeDocument.spec.ts | 73 +++++++++++++++++++ .../collectOrderedListItemsFromSource.ts | 58 +++++++++++++++ .../src/ordered-list/ordered-list.ts | 11 +-- 4 files changed, 139 insertions(+), 8 deletions(-) create mode 100644 .changeset/2026-09-03-ordered-list-markdown-linear-scan.md create mode 100644 packages/extension-list/__tests__/orderedListLargeDocument.spec.ts create mode 100644 packages/extension-list/src/ordered-list/collectOrderedListItemsFromSource.ts diff --git a/.changeset/2026-09-03-ordered-list-markdown-linear-scan.md b/.changeset/2026-09-03-ordered-list-markdown-linear-scan.md new file mode 100644 index 0000000000..d5457e962c --- /dev/null +++ b/.changeset/2026-09-03-ordered-list-markdown-linear-scan.md @@ -0,0 +1,5 @@ +--- +'@tiptap/extension-list': patch +--- + +Parsing Markdown is no longer super-linear in document size. The ordered list tokenizer read the whole remaining document at every block, so a 578KB file took about 19s and now takes about 160ms. diff --git a/packages/extension-list/__tests__/orderedListLargeDocument.spec.ts b/packages/extension-list/__tests__/orderedListLargeDocument.spec.ts new file mode 100644 index 0000000000..65c87146e1 --- /dev/null +++ b/packages/extension-list/__tests__/orderedListLargeDocument.spec.ts @@ -0,0 +1,73 @@ +import Blockquote from '@tiptap/extension-blockquote' +import Document from '@tiptap/extension-document' +import Heading from '@tiptap/extension-heading' +import Paragraph from '@tiptap/extension-paragraph' +import Text from '@tiptap/extension-text' +import { MarkdownManager } from '@tiptap/markdown' +import { describe, expect, it } from 'vite-plus/test' + +import { BulletList, ListItem, OrderedList } from '../src/index.js' + +const markdownManager = new MarkdownManager({ + extensions: [Document, Paragraph, Text, Heading, Blockquote, BulletList, OrderedList, ListItem], +}) + +const buildDocument = (sections: number) => + Array.from( + { length: sections }, + (_, index) => + `## Section ${index}\n\nParagraph ${index} text.\n\n` + + `- bullet ${index}\n\n1. first ${index}\n2. second ${index}\n\n> quote ${index}`, + ).join('\n\n') + +describe('ordered lists in large documents', () => { + it('reads a list that runs past the initial scan window', () => { + const items = 200 + const markdown = Array.from( + { length: items }, + (_, index) => `${index + 1}. item ${index}`, + ).join('\n') + + const json = markdownManager.parse(markdown) + + expect(json.content).toHaveLength(1) + expect(json.content?.[0].type).toBe('orderedList') + expect(json.content?.[0].content).toHaveLength(items) + }) + + it('stops the list at the following paragraph', () => { + const markdown = `${Array.from({ length: 120 }, (_, index) => `${index + 1}. item ${index}`).join('\n')}\n\nAfter the list.` + + const json = markdownManager.parse(markdown) + + expect(json.content).toHaveLength(2) + expect(json.content?.[0].content).toHaveLength(120) + expect(json.content?.[1]).toEqual({ + type: 'paragraph', + content: [{ type: 'text', text: 'After the list.' }], + }) + }) + + it('matches a single parse of the same list split across the window boundary', () => { + const short = markdownManager.parse('1. one\n2. two') + const long = markdownManager.parse( + Array.from({ length: 70 }, (_, index) => `${index + 1}. item ${index}`).join('\n'), + ) + + expect(short.content?.[0].type).toBe('orderedList') + expect(long.content?.[0].content).toHaveLength(70) + }) + + it('parses a large document in roughly linear time', () => { + const markdown = buildDocument(6000) + + expect(Buffer.byteLength(markdown)).toBeGreaterThan(400_000) + + const startedAt = performance.now() + markdownManager.parse(markdown) + const duration = performance.now() - startedAt + + // Quadratic scanning took ~19s for this document. + expect(duration).toBeLessThan(5000) + }, 60000) +}) diff --git a/packages/extension-list/src/ordered-list/collectOrderedListItemsFromSource.ts b/packages/extension-list/src/ordered-list/collectOrderedListItemsFromSource.ts new file mode 100644 index 0000000000..d7e8de4e14 --- /dev/null +++ b/packages/extension-list/src/ordered-list/collectOrderedListItemsFromSource.ts @@ -0,0 +1,58 @@ +import type { OrderedListItem } from './utils.js' +import { collectOrderedListItems, ORDERED_LIST_ITEM_REGEX } from './utils.js' + +const ORDERED_LIST_SCAN_WINDOW = 64 + +// Splits at most `max` lines. A shorter result means `source` ended. +function takeLines(source: string, max: number): string[] { + const lines: string[] = [] + let start = 0 + + while (lines.length < max) { + const lineEnd = source.indexOf('\n', start) + + if (lineEnd === -1) { + lines.push(source.slice(start)) + return lines + } + + lines.push(source.slice(start, lineEnd)) + start = lineEnd + 1 + } + + return lines +} + +/** + * Collect ordered list items from the start of `source`, reading only the lines the list needs. + * + * @param source Markdown beginning at a block boundary. + * @returns The items, how many lines they consumed, and the lines that were read. + * @example + * collectOrderedListItemsFromSource('1. one\n2. two\n\nAfter') + * // → [[item, item], 3, ['1. one', '2. two', '', 'After']] + */ +export function collectOrderedListItemsFromSource( + source: string, +): [OrderedListItem[], number, string[]] { + const firstLineEnd = source.indexOf('\n') + const firstLine = firstLineEnd === -1 ? source : source.slice(0, firstLineEnd) + + // Every block start is offered here, so reject non-lists before reading further. + if (!ORDERED_LIST_ITEM_REGEX.test(firstLine)) { + return [[], 0, []] + } + + let windowSize = ORDERED_LIST_SCAN_WINDOW + let lines = takeLines(source, windowSize) + let collected = collectOrderedListItems(lines) + + // Consuming the whole window means the list may continue past it. + while (collected[1] === lines.length && lines.length === windowSize) { + windowSize *= 2 + lines = takeLines(source, windowSize) + collected = collectOrderedListItems(lines) + } + + return [collected[0], collected[1], lines] +} diff --git a/packages/extension-list/src/ordered-list/ordered-list.ts b/packages/extension-list/src/ordered-list/ordered-list.ts index 13a079ec41..cba88d8273 100644 --- a/packages/extension-list/src/ordered-list/ordered-list.ts +++ b/packages/extension-list/src/ordered-list/ordered-list.ts @@ -3,12 +3,8 @@ import { Plugin } from '@tiptap/pm/state' import { mergeAttributes, Node, wrappingInputRule } from '@tiptap/core' -import { - buildNestedStructure, - collectOrderedListItems, - parseListItems, - parsePlainTextOrderedListPaste, -} from './utils.js' +import { collectOrderedListItemsFromSource } from './collectOrderedListItemsFromSource.js' +import { buildNestedStructure, parseListItems, parsePlainTextOrderedListPaste } from './utils.js' const ListItemName = 'listItem' const TextStyleName = 'textStyle' @@ -236,8 +232,7 @@ export const OrderedList = Node.create({ // `tokenize`, which would re-introduce the mid-line split. start: () => -1, tokenize: (src: string, _tokens, lexer) => { - const lines = src.split('\n') - const [listItems, consumed] = collectOrderedListItems(lines) + const [listItems, consumed, lines] = collectOrderedListItemsFromSource(src) if (listItems.length === 0) { return undefined From 281290782016c45c97b69df01ba180eff3d7d8ed Mon Sep 17 00:00:00 2001 From: Solomon Date: Thu, 3 Sep 2026 14:15:41 +0530 Subject: [PATCH 04/11] chore: format changeset frontmatter so vp check passes Duplicates #8303. Carried here so this PR's CI can go green before that lands; the change is identical, so it collapses to nothing on merge. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01TgWFnwc9wduBkoq2RTpTJR --- .changeset/fix-ios-ignore-mutation-contentdom.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/fix-ios-ignore-mutation-contentdom.md b/.changeset/fix-ios-ignore-mutation-contentdom.md index 06c8dc1920..ba55bd0702 100644 --- a/.changeset/fix-ios-ignore-mutation-contentdom.md +++ b/.changeset/fix-ios-ignore-mutation-contentdom.md @@ -1,5 +1,5 @@ --- -"@tiptap/core": patch +'@tiptap/core': patch --- Fix freezes in framework-based node views on iOS and Android From 399dc72aac30703009cd0b6e5827645578c38cc7 Mon Sep 17 00:00:00 2001 From: Solomon Date: Fri, 4 Sep 2026 09:35:29 +0530 Subject: [PATCH 05/11] refactor(extension-list): address review on the ordered list scan Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01TgWFnwc9wduBkoq2RTpTJR --- .changeset/2026-09-03-ordered-list-markdown-linear-scan.md | 2 +- .../extension-list/__tests__/orderedListLargeDocument.spec.ts | 4 ++-- .../src/ordered-list/collectOrderedListItemsFromSource.ts | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.changeset/2026-09-03-ordered-list-markdown-linear-scan.md b/.changeset/2026-09-03-ordered-list-markdown-linear-scan.md index d5457e962c..13ef4ea3da 100644 --- a/.changeset/2026-09-03-ordered-list-markdown-linear-scan.md +++ b/.changeset/2026-09-03-ordered-list-markdown-linear-scan.md @@ -2,4 +2,4 @@ '@tiptap/extension-list': patch --- -Parsing Markdown is no longer super-linear in document size. The ordered list tokenizer read the whole remaining document at every block, so a 578KB file took about 19s and now takes about 160ms. +Parsing large Markdown documents that contain ordered lists is much faster. A 578KB document went from about 19s to about 160ms. diff --git a/packages/extension-list/__tests__/orderedListLargeDocument.spec.ts b/packages/extension-list/__tests__/orderedListLargeDocument.spec.ts index 65c87146e1..bc3b0051b5 100644 --- a/packages/extension-list/__tests__/orderedListLargeDocument.spec.ts +++ b/packages/extension-list/__tests__/orderedListLargeDocument.spec.ts @@ -67,7 +67,7 @@ describe('ordered lists in large documents', () => { markdownManager.parse(markdown) const duration = performance.now() - startedAt - // Quadratic scanning took ~19s for this document. - expect(duration).toBeLessThan(5000) + // Quadratic scanning took ~37s here; ~250ms now. Loose enough for CI. + expect(duration).toBeLessThan(2000) }, 60000) }) diff --git a/packages/extension-list/src/ordered-list/collectOrderedListItemsFromSource.ts b/packages/extension-list/src/ordered-list/collectOrderedListItemsFromSource.ts index d7e8de4e14..976085ebfb 100644 --- a/packages/extension-list/src/ordered-list/collectOrderedListItemsFromSource.ts +++ b/packages/extension-list/src/ordered-list/collectOrderedListItemsFromSource.ts @@ -3,7 +3,7 @@ import { collectOrderedListItems, ORDERED_LIST_ITEM_REGEX } from './utils.js' const ORDERED_LIST_SCAN_WINDOW = 64 -// Splits at most `max` lines. A shorter result means `source` ended. +// Fewer than `max` lines means `source` ended. function takeLines(source: string, max: number): string[] { const lines: string[] = [] let start = 0 From cd7d7abe3fad4b74db96a8e3ca47bb286c8037a9 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 6 Sep 2026 16:10:22 +0000 Subject: [PATCH 06/11] fix(extension-table): stop leaking U+001F into table markdown Join multi-block cells with newlines (then
) instead of the vestigial UNIT SEPARATOR, and strip any leftover U+001F before emit. Fixes ueberdosis/tiptap#8152 Co-authored-by: Bennett Payoyo --- ...026-09-06-table-markdown-cell-separator.md | 5 ++ .../src/table/utilities/markdown.test.ts | 77 ++++++++++++++++++- .../src/table/utilities/markdown.ts | 18 ++--- .../tests/integration/tableMarkdown.test.ts | 11 ++- 4 files changed, 95 insertions(+), 16 deletions(-) create mode 100644 .changeset/2026-09-06-table-markdown-cell-separator.md diff --git a/.changeset/2026-09-06-table-markdown-cell-separator.md b/.changeset/2026-09-06-table-markdown-cell-separator.md new file mode 100644 index 0000000000..56abbbb0a0 --- /dev/null +++ b/.changeset/2026-09-06-table-markdown-cell-separator.md @@ -0,0 +1,5 @@ +--- +'@tiptap/extension-table': patch +--- + +Multi-block table cells no longer leak a U+001F control character into Markdown. diff --git a/packages/extension-table/src/table/utilities/markdown.test.ts b/packages/extension-table/src/table/utilities/markdown.test.ts index c1b9020477..4eb5ed0b94 100644 --- a/packages/extension-table/src/table/utilities/markdown.test.ts +++ b/packages/extension-table/src/table/utilities/markdown.test.ts @@ -1,6 +1,31 @@ +import type { JSONContent, MarkdownRendererHelpers } from '@tiptap/core' import { describe, expect, it } from 'vite-plus/test' -import { escapeTableCellPipes } from './markdown.js' +import { + DEFAULT_CELL_LINE_SEPARATOR, + escapeTableCellPipes, + renderTableToMarkdown, +} from './markdown.js' + +function renderInline(node: JSONContent): string { + if (node.type === 'text') { + return node.text ?? '' + } + + return (node.content ?? []).map(child => renderInline(child)).join('') +} + +const markdownHelpers: MarkdownRendererHelpers = { + renderChildren: nodes => { + if (Array.isArray(nodes)) { + return nodes.map(node => renderInline(node)).join('') + } + + return renderInline(nodes) + }, + wrapInBlock: (prefix, content) => prefix + content, + indent: content => content, +} describe('escapeTableCellPipes', () => { it('escapes a bare pipe inside a code span', () => { @@ -23,3 +48,53 @@ describe('escapeTableCellPipes', () => { expect(escapeTableCellPipes('| `|\\||` |')).toBe('| `\\|\\|\\|` |') }) }) + +describe('renderTableToMarkdown', () => { + const paragraph = (text: string): JSONContent => ({ + type: 'paragraph', + content: [{ type: 'text', text }], + }) + + it('does not emit U+001F when a cell has multiple block children', () => { + const markdown = renderTableToMarkdown( + { + type: 'table', + content: [ + { + type: 'tableRow', + content: [ + { type: 'tableCell', content: [paragraph('line one'), paragraph('line two')] }, + { type: 'tableCell', content: [paragraph('B')] }, + ], + }, + ], + }, + markdownHelpers, + ) + + expect(markdown).not.toContain(DEFAULT_CELL_LINE_SEPARATOR) + expect(markdown).not.toContain('\u001F') + expect(markdown).toContain('line one
line two') + }) + + it('strips a leftover cell-line separator even when it is passed as an option', () => { + const markdown = renderTableToMarkdown( + { + type: 'table', + content: [ + { + type: 'tableRow', + content: [ + { type: 'tableCell', content: [paragraph('line one'), paragraph('line two')] }, + ], + }, + ], + }, + markdownHelpers, + { cellLineSeparator: DEFAULT_CELL_LINE_SEPARATOR }, + ) + + expect(markdown).not.toContain('\u001F') + expect(markdown).toContain('line one
line two') + }) +}) diff --git a/packages/extension-table/src/table/utilities/markdown.ts b/packages/extension-table/src/table/utilities/markdown.ts index 0928f66a81..88e9d74d2e 100644 --- a/packages/extension-table/src/table/utilities/markdown.ts +++ b/packages/extension-table/src/table/utilities/markdown.ts @@ -82,7 +82,7 @@ export function preprocessTablePipes(src: string): string { } function collapseWhitespace(s: string) { - return (s || '').replace(/\s+/g, ' ').trim() + return (s || '').replaceAll(DEFAULT_CELL_LINE_SEPARATOR, ' ').replace(/\s+/g, ' ').trim() } export function renderTableToMarkdown( @@ -90,8 +90,6 @@ export function renderTableToMarkdown( h: MarkdownRendererHelpers, options: { cellLineSeparator?: string } = {}, ) { - const cellSep = options.cellLineSeparator ?? DEFAULT_CELL_LINE_SEPARATOR - if (!node || !node.content || node.content.length === 0) { return '' } @@ -107,25 +105,23 @@ export function renderTableToMarkdown( let raw = '' if (cellNode.content && Array.isArray(cellNode.content) && cellNode.content.length > 1) { - // Render each direct child separately and join with separator so we can split again later const parts = cellNode.content.map(child => h.renderChildren(child as unknown as JSONContent), ) - raw = parts.join(cellSep) + raw = parts.join('\n') } else { raw = cellNode.content ? h.renderChildren(cellNode.content as unknown as JSONContent[]) : '' } + if (options.cellLineSeparator) { + raw = raw.split(options.cellLineSeparator).join('\n') + } + // Cells have to stay on a single line, so line breaks become
tags. // The parser already turns
back into hard breaks, so this round trips. - const text = collapseWhitespace( - raw - .split(cellSep) - .join('\n') - .replace(/[ \t]*\r?\n[ \t]*/g, '
'), - ) + const text = collapseWhitespace(raw.replace(/[ \t]*\r?\n[ \t]*/g, '
')) const isHeader = cellNode.type === 'tableHeader' const align = normalizeTableCellAlignFromAttributes(cellNode.attrs) diff --git a/packages/extension-table/tests/integration/tableMarkdown.test.ts b/packages/extension-table/tests/integration/tableMarkdown.test.ts index 2d1f9cf63e..778dfc0ad3 100644 --- a/packages/extension-table/tests/integration/tableMarkdown.test.ts +++ b/packages/extension-table/tests/integration/tableMarkdown.test.ts @@ -248,17 +248,20 @@ describe('table markdown line breaks', () => { expect(serialized).toContain('foo
bar') }) - it('joins multiple paragraphs in a cell with
', () => { + it('joins multiple paragraphs in a cell with
and does not emit U+001F', () => { const doc = tableDoc([ cell([ - { type: 'paragraph', content: [{ type: 'text', text: 'one' }] }, - { type: 'paragraph', content: [{ type: 'text', text: 'two' }] }, + { type: 'paragraph', content: [{ type: 'text', text: 'line one' }] }, + { type: 'paragraph', content: [{ type: 'text', text: 'line two' }] }, ]), + cell([{ type: 'paragraph', content: [{ type: 'text', text: 'B' }] }]), ]) const serialized = markdownManager.serialize(doc) - expect(serialized).toContain('one
two') + expect(serialized).toContain('line one
line two') + expect(serialized).toContain('| B') + expect(serialized.includes(String.fromCharCode(31))).toBe(false) expect(serialized).not.toContain('\u001F') }) From 7c8ad8774a27b3c3c0e0a76ae550f4c14a5dffe7 Mon Sep 17 00:00:00 2001 From: Pierre LHOSTE Date: Wed, 2 Sep 2026 13:48:43 +0200 Subject: [PATCH 07/11] fix(markdown): choose an inline code fence the content cannot close --- ...2026-09-02-fix-code-span-backtick-fence.md | 6 ++ packages/extension-code/src/code.ts | 22 +++++- packages/markdown/src/MarkdownManager.ts | 39 +++++++-- .../src/utils/collectMarkSpanTexts.ts | 40 ++++++++++ .../integration/code-span-backticks.test.ts | 79 +++++++++++++++++++ 5 files changed, 178 insertions(+), 8 deletions(-) create mode 100644 .changeset/2026-09-02-fix-code-span-backtick-fence.md create mode 100644 packages/markdown/src/utils/collectMarkSpanTexts.ts create mode 100644 packages/markdown/tests/integration/code-span-backticks.test.ts diff --git a/.changeset/2026-09-02-fix-code-span-backtick-fence.md b/.changeset/2026-09-02-fix-code-span-backtick-fence.md new file mode 100644 index 0000000000..f1370ad995 --- /dev/null +++ b/.changeset/2026-09-02-fix-code-span-backtick-fence.md @@ -0,0 +1,6 @@ +--- +'@tiptap/markdown': patch +'@tiptap/extension-code': patch +--- + +Fix Markdown serialization of inline code containing backticks diff --git a/packages/extension-code/src/code.ts b/packages/extension-code/src/code.ts index 1f317bfbfc..74f3d505b2 100644 --- a/packages/extension-code/src/code.ts +++ b/packages/extension-code/src/code.ts @@ -93,6 +93,18 @@ export const pasteRegexMatch = (text: string): PasteRuleMatch[] => { return matches } +// A code span must be fenced by a backtick run its own content does not contain +function shortestUnusedBacktickRun(text: string): number { + const runs = new Set((text.match(/`+/g) || []).map(run => run.length)) + let length = 1 + + while (runs.has(length)) { + length += 1 + } + + return length +} + /** * This extension allows you to mark text as inline code. * @see https://tiptap.dev/api/marks/code @@ -128,12 +140,18 @@ export const Code = Mark.create({ return helpers.applyMark('code', [{ type: 'text', text: token.text || '' }]) }, - renderMarkdown: (node, h) => { + renderMarkdown: (node, h, context) => { if (!node.content) { return '' } - return `\`${h.renderChildren(node.content)}\`` + // Children are a placeholder here, so the fence comes from the full span text + const markText = typeof context?.meta?.markText === 'string' ? context.meta.markText : '' + const fence = '`'.repeat(shortestUnusedBacktickRun(markText)) + // CommonMark strips one space on each side, so padding keeps backticks apart + const padding = markText.startsWith('`') || markText.endsWith('`') ? ' ' : '' + + return `${fence}${padding}${h.renderChildren(node.content)}${padding}${fence}` }, addCommands() { diff --git a/packages/markdown/src/MarkdownManager.ts b/packages/markdown/src/MarkdownManager.ts index 7a287409dd..a65ff4c67d 100644 --- a/packages/markdown/src/MarkdownManager.ts +++ b/packages/markdown/src/MarkdownManager.ts @@ -33,6 +33,7 @@ import { reopenMarksAfterNode, wrapInMarkdownBlock, } from './utils.js' +import { collectMarkSpanTexts } from './utils/collectMarkSpanTexts.js' import { htmlContainsUnrecognizedTag } from './utils/htmlTagDetection.js' export class MarkdownManager { @@ -1271,6 +1272,8 @@ export class MarkdownManager { const activeMarks: Map = new Map() const reopenWithHtmlOnNextOpen = new Set() const markOpeningModes = new Map() + // Delimiters can depend on the whole mark run, not just one text node + const markSpanTexts = collectMarkSpanTexts(nodes) nodes.forEach((node, i) => { // Lookahead to the next node to determine if marks need to be closed const nextNode = i < nodes.length - 1 ? nodes[i + 1] : null @@ -1344,6 +1347,7 @@ export class MarkdownManager { markType, mark, markOpeningModes.get(markType), + markSpanTexts[i]?.get(markType), ) if (closeMarkdown) { textContent += closeMarkdown @@ -1372,7 +1376,12 @@ export class MarkdownManager { // so new marks correctly see them as active context. marksToOpen.forEach(({ type, mark }) => { const openingMode = reopenWithHtmlOnNextOpen.has(type) ? 'html' : 'markdown' - const openMarkdown = this.getMarkOpening(type, mark, openingMode) + const openMarkdown = this.getMarkOpening( + type, + mark, + openingMode, + markSpanTexts[i]?.get(type), + ) if (openMarkdown) { textContent = openMarkdown + textContent } @@ -1439,7 +1448,12 @@ export class MarkdownManager { marksToCloseAtEnd.forEach(markType => { const mark = activeMarks.get(markType) ?? currentMarks.get(markType) - const closeMarkdown = this.getMarkClosing(markType, mark, markOpeningModes.get(markType)) + const closeMarkdown = this.getMarkClosing( + markType, + mark, + markOpeningModes.get(markType), + markSpanTexts[i]?.get(markType), + ) if (closeMarkdown) { textContent += closeMarkdown } @@ -1466,8 +1480,14 @@ export class MarkdownManager { }) // Close all marks before the node + // The run ends on the previous node and restarts on the next one const beforeMarkdown = closeMarksBeforeNode(activeMarks, (markType, mark) => { - return this.getMarkClosing(markType, mark, markOpeningModes.get(markType)) + return this.getMarkClosing( + markType, + mark, + markOpeningModes.get(markType), + markSpanTexts[i - 1]?.get(markType), + ) }) markOpeningModes.clear() @@ -1482,7 +1502,12 @@ export class MarkdownManager { : reopenMarksAfterNode(marksToReopen, activeMarks, (markType, mark) => { const openingMode = openingModesToReopen.get(markType) ?? 'markdown' markOpeningModes.set(markType, openingMode) - return this.getMarkOpening(markType, mark, openingMode) + return this.getMarkOpening( + markType, + mark, + openingMode, + markSpanTexts[i + 1]?.get(markType), + ) }) result.push(beforeMarkdown + nodeContent + afterMarkdown) @@ -1499,6 +1524,7 @@ export class MarkdownManager { markType: string, mark: any, openingMode: 'markdown' | 'html' = 'markdown', + markText?: string, ): string { if (openingMode === 'html') { return this.getHtmlReopenTags(markType)?.open || '' @@ -1529,7 +1555,7 @@ export class MarkdownManager { indent: (content: string) => content, wrapInBlock: (prefix: string, content: string) => prefix + content, }, - { index: 0, level: 0, parentType: 'text', meta: {} }, + { index: 0, level: 0, parentType: 'text', meta: { markText } }, ) // Extract the opening part (everything before placeholder) @@ -1547,6 +1573,7 @@ export class MarkdownManager { markType: string, mark: any, openingMode: 'markdown' | 'html' = 'markdown', + markText?: string, ): string { if (openingMode === 'html') { return this.getHtmlReopenTags(markType)?.close || '' @@ -1576,7 +1603,7 @@ export class MarkdownManager { indent: (content: string) => content, wrapInBlock: (prefix: string, content: string) => prefix + content, }, - { index: 0, level: 0, parentType: 'text', meta: {} }, + { index: 0, level: 0, parentType: 'text', meta: { markText } }, ) // Extract the closing part (everything after placeholder) diff --git a/packages/markdown/src/utils/collectMarkSpanTexts.ts b/packages/markdown/src/utils/collectMarkSpanTexts.ts new file mode 100644 index 0000000000..eb7392d9ab --- /dev/null +++ b/packages/markdown/src/utils/collectMarkSpanTexts.ts @@ -0,0 +1,40 @@ +import type { JSONContent } from '@tiptap/core' + +// Gives each node the full text of its mark run, so a delimiter can fit the whole span +export function collectMarkSpanTexts(nodes: JSONContent[]): Map[] { + const spanTexts: Map[] = nodes.map(() => new Map()) + const openRuns = new Map() + + const flush = (markType: string) => { + const run = openRuns.get(markType) + if (!run) { + return + } + run.indexes.forEach(index => spanTexts[index].set(markType, run.text)) + openRuns.delete(markType) + } + + nodes.forEach((node, index) => { + if (node.type !== 'text') { + Array.from(openRuns.keys()).forEach(flush) + return + } + + const markTypes = new Set((node.marks || []).map(mark => mark.type)) + + Array.from(openRuns.keys()) + .filter(markType => !markTypes.has(markType)) + .forEach(flush) + + markTypes.forEach(markType => { + const run = openRuns.get(markType) || { indexes: [], text: '' } + run.indexes.push(index) + run.text += node.text || '' + openRuns.set(markType, run) + }) + }) + + Array.from(openRuns.keys()).forEach(flush) + + return spanTexts +} diff --git a/packages/markdown/tests/integration/code-span-backticks.test.ts b/packages/markdown/tests/integration/code-span-backticks.test.ts new file mode 100644 index 0000000000..8ce31d0590 --- /dev/null +++ b/packages/markdown/tests/integration/code-span-backticks.test.ts @@ -0,0 +1,79 @@ +import { Code } from '@tiptap/extension-code' +import { Document } from '@tiptap/extension-document' +import { Image } from '@tiptap/extension-image' +import { Paragraph } from '@tiptap/extension-paragraph' +import { Text } from '@tiptap/extension-text' +import { describe, expect, it } from 'vite-plus/test' + +import { MarkdownManager } from '../../src/MarkdownManager.js' + +describe('Code spans containing backticks', () => { + const markdownManager = new MarkdownManager({ extensions: [Document, Paragraph, Text, Code] }) + + const codeSpan = (text: string) => ({ + type: 'doc', + content: [{ type: 'paragraph', content: [{ type: 'text', text, marks: [{ type: 'code' }] }] }], + }) + + it.each([ + ['hello', '`hello`'], + ['hello ` world', '``hello ` world``'], + ['a ` b `` c', '```a ` b `` c```'], + ['`leading', '`` `leading ``'], + ['trailing`', '`` trailing` ``'], + ['`', '`` ` ``'], + ['``', '` `` `'], + ])('serializes %j with a fence the content cannot close', (text, expected) => { + expect(markdownManager.serialize(codeSpan(text))).toBe(expected) + }) + + it.each(['hello', 'hello ` world', 'a ` b `` c', '`leading', 'trailing`', '`', '``'])( + 'round-trips %j through serialize and parse', + text => { + const parsed = markdownManager.parse(markdownManager.serialize(codeSpan(text))) + + expect(parsed.content?.[0].content).toEqual([ + { type: 'text', text, marks: [{ type: 'code' }] }, + ]) + }, + ) + + it('keeps a code span with a backtick separate from its surrounding text', () => { + const doc = { + type: 'doc', + content: [ + { + type: 'paragraph', + content: [ + { type: 'text', text: 'Press ' }, + { type: 'text', text: '`', marks: [{ type: 'code' }] }, + { type: 'text', text: ' to open.' }, + ], + }, + ], + } + + const markdown = markdownManager.serialize(doc) + + expect(markdown).toBe('Press `` ` `` to open.') + expect(markdownManager.parse(markdown).content?.[0].content).toEqual(doc.content[0].content) + }) + + it('fences both sides of a code mark that spans an inline atom', () => { + const withImage = new MarkdownManager({ + extensions: [Document, Paragraph, Text, Code, Image.configure({ inline: true })], + }) + const content = [ + { type: 'text', text: 'a`b', marks: [{ type: 'code' }] }, + { type: 'image', attrs: { src: 'x.png' }, marks: [{ type: 'code' }] }, + { type: 'text', text: 'c`d', marks: [{ type: 'code' }] }, + ] + + const markdown = withImage.serialize({ + type: 'doc', + content: [{ type: 'paragraph', content }], + }) + + expect(markdown).toBe('``a`b``![](x.png)``c`d``') + }) +}) From a90921e7cf6369c14f881691d5c7ed360f5cd1b0 Mon Sep 17 00:00:00 2001 From: HwangRock Date: Sun, 20 Sep 2026 16:37:09 +0900 Subject: [PATCH 08/11] fix(core): decode numeric HTML entities when parsing markdown Numeric character references like ' and ' were left as literal text when parsing markdown, because decodeHtmlEntities only handled the four named entities it emits itself. Decode numeric and hex references too, and leave out-of-range code points untouched instead of throwing. --- ...2026-09-20-decode-numeric-html-entities.md | 5 ++++ .../core/src/utilities/htmlEntities.test.ts | 24 +++++++++++++++++++ packages/core/src/utilities/htmlEntities.ts | 7 ++++++ .../tests/integration/conversion.test.ts | 14 +++++++++++ 4 files changed, 50 insertions(+) create mode 100644 .changeset/2026-09-20-decode-numeric-html-entities.md diff --git a/.changeset/2026-09-20-decode-numeric-html-entities.md b/.changeset/2026-09-20-decode-numeric-html-entities.md new file mode 100644 index 0000000000..47a16c9148 --- /dev/null +++ b/.changeset/2026-09-20-decode-numeric-html-entities.md @@ -0,0 +1,5 @@ +--- +'@tiptap/core': patch +--- + +Numeric HTML entities (`'`, `'`) are now decoded when parsing markdown, instead of showing up as literal text in the editor. diff --git a/packages/core/src/utilities/htmlEntities.test.ts b/packages/core/src/utilities/htmlEntities.test.ts index fad9d57fd4..63d0cc3ee1 100644 --- a/packages/core/src/utilities/htmlEntities.test.ts +++ b/packages/core/src/utilities/htmlEntities.test.ts @@ -19,6 +19,30 @@ describe('decodeHtmlEntities', () => { expect(decodeHtmlEntities('&lt;')).toBe('<') }) + it('decodes decimal numeric entities like '', () => { + expect(decodeHtmlEntities('it's')).toBe("it's") + }) + + it('decodes hex numeric entities like '', () => { + expect(decodeHtmlEntities(''')).toBe("'") + }) + + it('decodes decimal numeric entities for non-ASCII characters like —', () => { + expect(decodeHtmlEntities('—')).toBe('—') + }) + + it('handles doubly-encoded numeric entities like &#39;', () => { + expect(decodeHtmlEntities('&#39;')).toBe(''') + }) + + it('leaves out-of-range decimal numeric entities untouched', () => { + expect(decodeHtmlEntities('�')).toBe('�') + }) + + it('leaves out-of-range hex numeric entities untouched', () => { + expect(decodeHtmlEntities('�')).toBe('�') + }) + it('returns plain text unchanged', () => { expect(decodeHtmlEntities('hello world')).toBe('hello world') }) diff --git a/packages/core/src/utilities/htmlEntities.ts b/packages/core/src/utilities/htmlEntities.ts index ee04bf5e71..2bf42886fb 100644 --- a/packages/core/src/utilities/htmlEntities.ts +++ b/packages/core/src/utilities/htmlEntities.ts @@ -9,9 +9,16 @@ export function decodeHtmlEntities(text: string): string { .replace(/</g, '<') .replace(/>/g, '>') .replace(/"/g, '"') + .replace(/&#(\d+);/g, (match, dec) => decodeCodePoint(match, Number(dec))) + .replace(/&#x([0-9a-f]+);/gi, (match, hex) => decodeCodePoint(match, parseInt(hex, 16))) .replace(/&/g, '&') } +// Leave out-of-range numeric entities untouched instead of throwing. +function decodeCodePoint(match: string, codePoint: number): string { + return codePoint <= 0x10ffff ? String.fromCodePoint(codePoint) : match +} + /** * Encode HTML special characters so they roundtrip safely through markdown. * `&` is encoded **first** to avoid double-encoding the ampersand in other diff --git a/packages/markdown/tests/integration/conversion.test.ts b/packages/markdown/tests/integration/conversion.test.ts index 7ba0bb6ba1..818166f179 100644 --- a/packages/markdown/tests/integration/conversion.test.ts +++ b/packages/markdown/tests/integration/conversion.test.ts @@ -538,6 +538,20 @@ describe('Markdown Conversion Tests', () => { expect(serialized).toBe('foo &lt; bar') }) + it('should decode decimal numeric entities like ' when parsing', () => { + const markdown = 'it's a test' + const json = markdownManager.parse(markdown) + + expect(json.content[0].content[0].text).toBe("it's a test") + }) + + it('should decode hex numeric entities like ' when parsing', () => { + const markdown = 'it's a test' + const json = markdownManager.parse(markdown) + + expect(json.content[0].content[0].text).toBe("it's a test") + }) + it('should preserve   empty paragraph behavior', () => { const markdown = 'Line1\n\n \n\nLine2' const json = markdownManager.parse(markdown) From c828d74a141d5cb9d4c30d0aa35c1b68b9f9b14d Mon Sep 17 00:00:00 2001 From: HwangRock Date: Mon, 21 Sep 2026 20:37:48 +0900 Subject: [PATCH 09/11] fix(core): decode HTML entities in a single pass The chained replaces re-scanned their own output, so a decimal entity that produced '&' could be decoded again (e.g. &#x27; became '). Use one regex over a single pass so a replacement is never decoded twice. --- .../core/src/utilities/htmlEntities.test.ts | 12 +++++++++ packages/core/src/utilities/htmlEntities.ts | 26 ++++++++++++------- 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/packages/core/src/utilities/htmlEntities.test.ts b/packages/core/src/utilities/htmlEntities.test.ts index 63d0cc3ee1..ed6b2ef31f 100644 --- a/packages/core/src/utilities/htmlEntities.test.ts +++ b/packages/core/src/utilities/htmlEntities.test.ts @@ -43,6 +43,18 @@ describe('decodeHtmlEntities', () => { expect(decodeHtmlEntities('�')).toBe('�') }) + it('does not re-scan replaced output for a second round of decoding', () => { + expect(decodeHtmlEntities('&#x27;')).toBe(''') + }) + + it('does not turn a decoded & into a second-pass < decode', () => { + expect(decodeHtmlEntities('&lt;')).toBe('<') + }) + + it('does not turn a decoded & into a second-pass & decode', () => { + expect(decodeHtmlEntities('&amp;')).toBe('&') + }) + it('returns plain text unchanged', () => { expect(decodeHtmlEntities('hello world')).toBe('hello world') }) diff --git a/packages/core/src/utilities/htmlEntities.ts b/packages/core/src/utilities/htmlEntities.ts index 2bf42886fb..8ba1a40b90 100644 --- a/packages/core/src/utilities/htmlEntities.ts +++ b/packages/core/src/utilities/htmlEntities.ts @@ -1,17 +1,23 @@ +const NAMED_ENTITIES: Record = { lt: '<', gt: '>', quot: '"', amp: '&' } + /** * Decode common HTML entities in text content so they display as literal - * characters inside the editor. The decode order matters: `&` must be - * decoded **last** so that doubly-encoded sequences like `&lt;` first - * survive the `<` pass and then correctly become `<` (not `<`). + * characters inside the editor. Runs as a single pass so a decoded entity + * (e.g. `&` → `&`) is never re-scanned and decoded again. */ export function decodeHtmlEntities(text: string): string { - return text - .replace(/</g, '<') - .replace(/>/g, '>') - .replace(/"/g, '"') - .replace(/&#(\d+);/g, (match, dec) => decodeCodePoint(match, Number(dec))) - .replace(/&#x([0-9a-f]+);/gi, (match, hex) => decodeCodePoint(match, parseInt(hex, 16))) - .replace(/&/g, '&') + return text.replace( + /&(?:#(\d+)|#[xX]([0-9a-fA-F]+)|(lt|gt|quot|amp));/g, + (match, dec, hex, name) => { + if (dec !== undefined) { + return decodeCodePoint(match, Number(dec)) + } + if (hex !== undefined) { + return decodeCodePoint(match, parseInt(hex, 16)) + } + return NAMED_ENTITIES[name] + }, + ) } // Leave out-of-range numeric entities untouched instead of throwing. From 916d1359d2c4a9500a72d8bbc207dea37c9732b7 Mon Sep 17 00:00:00 2001 From: Bennett Payoyo Date: Mon, 21 Sep 2026 13:06:40 +0000 Subject: [PATCH 10/11] fix(extension-table): keep empty cell paragraphs Replace cellLineSeparator inside each rendered child before joining so a blank paragraph stays a blank line when the separator is two newlines. Co-authored-by: Bennett Payoyo --- .../src/table/utilities/markdown.test.ts | 23 ++++++++++++++++++ .../src/table/utilities/markdown.ts | 24 ++++++++++++------- 2 files changed, 39 insertions(+), 8 deletions(-) diff --git a/packages/extension-table/src/table/utilities/markdown.test.ts b/packages/extension-table/src/table/utilities/markdown.test.ts index 4eb5ed0b94..b57c9ef887 100644 --- a/packages/extension-table/src/table/utilities/markdown.test.ts +++ b/packages/extension-table/src/table/utilities/markdown.test.ts @@ -97,4 +97,27 @@ describe('renderTableToMarkdown', () => { expect(markdown).not.toContain('\u001F') expect(markdown).toContain('line one
line two') }) + + it('preserves an empty paragraph when cellLineSeparator is two newlines', () => { + const markdown = renderTableToMarkdown( + { + type: 'table', + content: [ + { + type: 'tableRow', + content: [ + { + type: 'tableCell', + content: [paragraph('one'), { type: 'paragraph', content: [] }, paragraph('two')], + }, + ], + }, + ], + }, + markdownHelpers, + { cellLineSeparator: '\n\n' }, + ) + + expect(markdown).toContain('one

two') + }) }) diff --git a/packages/extension-table/src/table/utilities/markdown.ts b/packages/extension-table/src/table/utilities/markdown.ts index 88e9d74d2e..fe2d5a76d2 100644 --- a/packages/extension-table/src/table/utilities/markdown.ts +++ b/packages/extension-table/src/table/utilities/markdown.ts @@ -85,6 +85,14 @@ function collapseWhitespace(s: string) { return (s || '').replaceAll(DEFAULT_CELL_LINE_SEPARATOR, ' ').replace(/\s+/g, ' ').trim() } +function replaceCellLineSeparator(value: string, cellLineSeparator?: string) { + if (!cellLineSeparator) { + return value + } + + return value.split(cellLineSeparator).join('\n') +} + export function renderTableToMarkdown( node: JSONContent, h: MarkdownRendererHelpers, @@ -106,17 +114,17 @@ export function renderTableToMarkdown( if (cellNode.content && Array.isArray(cellNode.content) && cellNode.content.length > 1) { const parts = cellNode.content.map(child => - h.renderChildren(child as unknown as JSONContent), + replaceCellLineSeparator( + h.renderChildren(child as unknown as JSONContent), + options.cellLineSeparator, + ), ) raw = parts.join('\n') } else { - raw = cellNode.content - ? h.renderChildren(cellNode.content as unknown as JSONContent[]) - : '' - } - - if (options.cellLineSeparator) { - raw = raw.split(options.cellLineSeparator).join('\n') + raw = replaceCellLineSeparator( + cellNode.content ? h.renderChildren(cellNode.content as unknown as JSONContent[]) : '', + options.cellLineSeparator, + ) } // Cells have to stay on a single line, so line breaks become
tags. From 5c45a8646ebc6509f11ae80a12fc83edc29f8824 Mon Sep 17 00:00:00 2001 From: HwangRock Date: Tue, 22 Sep 2026 00:56:03 +0900 Subject: [PATCH 11/11] fix(core): make numeric HTML entity decoding CommonMark-conformant Invalid code points (zero, surrogates, above U+10FFFF) become U+FFFD, over-long references (more than 7 decimal or 6 hex digits) stay literal, and the C1 range (0x80-0x9F) is remapped to Windows-1252. --- .../core/src/utilities/htmlEntities.test.ts | 76 ++++++++++++++++++- packages/core/src/utilities/htmlEntities.ts | 47 ++++++++++-- 2 files changed, 113 insertions(+), 10 deletions(-) diff --git a/packages/core/src/utilities/htmlEntities.test.ts b/packages/core/src/utilities/htmlEntities.test.ts index ed6b2ef31f..0b0c6dc45b 100644 --- a/packages/core/src/utilities/htmlEntities.test.ts +++ b/packages/core/src/utilities/htmlEntities.test.ts @@ -28,21 +28,89 @@ describe('decodeHtmlEntities', () => { }) it('decodes decimal numeric entities for non-ASCII characters like —', () => { - expect(decodeHtmlEntities('—')).toBe('—') + expect(decodeHtmlEntities('—')).toBe('\u2014') }) it('handles doubly-encoded numeric entities like &#39;', () => { expect(decodeHtmlEntities('&#39;')).toBe(''') }) - it('leaves out-of-range decimal numeric entities untouched', () => { + it('leaves decimal references over 7 digits as a literal (exceeds CommonMark digit limit)', () => { expect(decodeHtmlEntities('�')).toBe('�') }) - it('leaves out-of-range hex numeric entities untouched', () => { + it('leaves hex references over 6 digits as a literal (exceeds CommonMark digit limit)', () => { expect(decodeHtmlEntities('�')).toBe('�') }) + it('leaves an 8-digit decimal reference as a literal (format exceeds the limit)', () => { + expect(decodeHtmlEntities('�')).toBe('�') + }) + + it('replaces a 7-digit decimal reference whose value exceeds 0x10FFFF with U+FFFD', () => { + expect(decodeHtmlEntities('�')).toBe('\uFFFD') + }) + + it('replaces a 6-digit hex reference whose value exceeds 0x10FFFF with U+FFFD', () => { + expect(decodeHtmlEntities('�')).toBe('\uFFFD') + }) + + it('matches a 7-digit decimal reference greedily and leaves the trailing digit as a literal', () => { + expect(decodeHtmlEntities('�8')).toBe('\uFFFD8') + }) + + it('replaces � (NUL) with U+FFFD', () => { + expect(decodeHtmlEntities('�')).toBe('\uFFFD') + }) + + it('replaces a lone low surrogate � with U+FFFD', () => { + expect(decodeHtmlEntities('�')).toBe('\uFFFD') + }) + + it('replaces a lone high surrogate � with U+FFFD', () => { + expect(decodeHtmlEntities('�')).toBe('\uFFFD') + }) + + it('decodes ퟿ normally (just below the surrogate range)', () => { + expect(decodeHtmlEntities('퟿')).toBe(String.fromCodePoint(0xd7ff)) + }) + + it('decodes  normally (just above the surrogate range)', () => { + expect(decodeHtmlEntities('')).toBe(String.fromCodePoint(0xe000)) + }) + + it('decodes 􏿿 normally (the maximum valid code point)', () => { + expect(decodeHtmlEntities('􏿿')).toBe(String.fromCodePoint(0x10ffff)) + }) + + it('remaps € (0x80) to the Windows-1252 euro sign', () => { + expect(decodeHtmlEntities('€')).toBe('\u20AC') + }) + + it('remaps — (0x97) to the Windows-1252 em dash', () => { + expect(decodeHtmlEntities('—')).toBe('\u2014') + }) + + it('remaps “ (0x93) to the Windows-1252 left double quote', () => { + expect(decodeHtmlEntities('“')).toBe('\u201C') + }) + + it('remaps ” (0x94) to the Windows-1252 right double quote', () => { + expect(decodeHtmlEntities('”')).toBe('\u201D') + }) + + it('remaps € (hex form) to the Windows-1252 euro sign', () => { + expect(decodeHtmlEntities('€')).toBe('\u20AC') + }) + + it('leaves  (0x81) as the raw C1 control character (not in the Windows-1252 table)', () => { + expect(decodeHtmlEntities('')).toBe('\u0081') + }) + + it('leaves  (0x8D) as the raw C1 control character (not in the Windows-1252 table)', () => { + expect(decodeHtmlEntities('')).toBe('\u008D') + }) + it('does not re-scan replaced output for a second round of decoding', () => { expect(decodeHtmlEntities('&#x27;')).toBe(''') }) @@ -83,7 +151,7 @@ describe('roundtrip', () => { expect(decodeHtmlEntities(encodeHtmlEntities(input))).toBe(input) }) - it('decode is a superset of encode – " decodes but " is not encoded', () => { + it('decode is a superset of encode - " decodes but " is not encoded', () => { // " passes through encode unchanged, " decodes to " expect(encodeHtmlEntities('"hello"')).toBe('"hello"') expect(decodeHtmlEntities('"hello"')).toBe('"hello"') diff --git a/packages/core/src/utilities/htmlEntities.ts b/packages/core/src/utilities/htmlEntities.ts index 8ba1a40b90..788701633f 100644 --- a/packages/core/src/utilities/htmlEntities.ts +++ b/packages/core/src/utilities/htmlEntities.ts @@ -6,23 +6,58 @@ const NAMED_ENTITIES: Record = { lt: '<', gt: '>', quot: '"', am * (e.g. `&` → `&`) is never re-scanned and decoded again. */ export function decodeHtmlEntities(text: string): string { + // Digit limits per CommonMark: decimal refs are 1-7 digits, hex refs are 1-6. return text.replace( - /&(?:#(\d+)|#[xX]([0-9a-fA-F]+)|(lt|gt|quot|amp));/g, + /&(?:#(\d{1,7})|#[xX]([0-9a-fA-F]{1,6})|(lt|gt|quot|amp));/g, (match, dec, hex, name) => { if (dec !== undefined) { - return decodeCodePoint(match, Number(dec)) + return decodeCodePoint(Number(dec)) } if (hex !== undefined) { - return decodeCodePoint(match, parseInt(hex, 16)) + return decodeCodePoint(parseInt(hex, 16)) } return NAMED_ENTITIES[name] }, ) } -// Leave out-of-range numeric entities untouched instead of throwing. -function decodeCodePoint(match: string, codePoint: number): string { - return codePoint <= 0x10ffff ? String.fromCodePoint(codePoint) : match +// HTML5 maps numeric references in the C1 range (0x80-0x9F) to Windows-1252 characters. +const WINDOWS_1252 = new Map([ + [0x80, 0x20ac], + [0x82, 0x201a], + [0x83, 0x0192], + [0x84, 0x201e], + [0x85, 0x2026], + [0x86, 0x2020], + [0x87, 0x2021], + [0x88, 0x02c6], + [0x89, 0x2030], + [0x8a, 0x0160], + [0x8b, 0x2039], + [0x8c, 0x0152], + [0x8e, 0x017d], + [0x91, 0x2018], + [0x92, 0x2019], + [0x93, 0x201c], + [0x94, 0x201d], + [0x95, 0x2022], + [0x96, 0x2013], + [0x97, 0x2014], + [0x98, 0x02dc], + [0x99, 0x2122], + [0x9a, 0x0161], + [0x9b, 0x203a], + [0x9c, 0x0153], + [0x9e, 0x017e], + [0x9f, 0x0178], +]) + +// Replace invalid code points with U+FFFD, per the CommonMark spec. +function decodeCodePoint(codePoint: number): string { + if (codePoint === 0 || codePoint > 0x10ffff || (codePoint >= 0xd800 && codePoint <= 0xdfff)) { + return '\uFFFD' + } + return String.fromCodePoint(WINDOWS_1252.get(codePoint) ?? codePoint) } /**