diff --git a/.changeset/2026-09-02-fix-code-span-backtick-fence.md b/.changeset/2026-09-02-fix-code-span-backtick-fence.md new file mode 100644 index 0000000000..f1370ad995 --- /dev/null +++ b/.changeset/2026-09-02-fix-code-span-backtick-fence.md @@ -0,0 +1,6 @@ +--- +'@tiptap/markdown': patch +'@tiptap/extension-code': patch +--- + +Fix Markdown serialization of inline code containing backticks diff --git a/.changeset/2026-09-02-table-cell-pipe-escape.md b/.changeset/2026-09-02-table-cell-pipe-escape.md new file mode 100644 index 0000000000..1a40a7e1c9 --- /dev/null +++ b/.changeset/2026-09-02-table-cell-pipe-escape.md @@ -0,0 +1,5 @@ +--- +'@tiptap/extension-table': patch +--- + +Table cells exported to Markdown now escape literal pipe characters, so the cell content survives when the output is read back. diff --git a/.changeset/2026-09-03-ordered-list-markdown-linear-scan.md b/.changeset/2026-09-03-ordered-list-markdown-linear-scan.md new file mode 100644 index 0000000000..13ef4ea3da --- /dev/null +++ b/.changeset/2026-09-03-ordered-list-markdown-linear-scan.md @@ -0,0 +1,5 @@ +--- +'@tiptap/extension-list': patch +--- + +Parsing large Markdown documents that contain ordered lists is much faster. A 578KB document went from about 19s to about 160ms. diff --git a/.changeset/2026-09-06-table-markdown-cell-separator.md b/.changeset/2026-09-06-table-markdown-cell-separator.md new file mode 100644 index 0000000000..56abbbb0a0 --- /dev/null +++ b/.changeset/2026-09-06-table-markdown-cell-separator.md @@ -0,0 +1,5 @@ +--- +'@tiptap/extension-table': patch +--- + +Multi-block table cells no longer leak a U+001F control character into Markdown. diff --git a/.changeset/2026-09-20-decode-numeric-html-entities.md b/.changeset/2026-09-20-decode-numeric-html-entities.md new file mode 100644 index 0000000000..47a16c9148 --- /dev/null +++ b/.changeset/2026-09-20-decode-numeric-html-entities.md @@ -0,0 +1,5 @@ +--- +'@tiptap/core': patch +--- + +Numeric HTML entities (`'`, `'`) are now decoded when parsing markdown, instead of showing up as literal text in the editor. diff --git a/packages/core/src/utilities/htmlEntities.test.ts b/packages/core/src/utilities/htmlEntities.test.ts index fad9d57fd4..0b0c6dc45b 100644 --- a/packages/core/src/utilities/htmlEntities.test.ts +++ b/packages/core/src/utilities/htmlEntities.test.ts @@ -19,6 +19,110 @@ describe('decodeHtmlEntities', () => { expect(decodeHtmlEntities('&lt;')).toBe('<') }) + it('decodes decimal numeric entities like '', () => { + expect(decodeHtmlEntities('it's')).toBe("it's") + }) + + it('decodes hex numeric entities like '', () => { + expect(decodeHtmlEntities(''')).toBe("'") + }) + + it('decodes decimal numeric entities for non-ASCII characters like —', () => { + expect(decodeHtmlEntities('—')).toBe('\u2014') + }) + + it('handles doubly-encoded numeric entities like &#39;', () => { + expect(decodeHtmlEntities('&#39;')).toBe(''') + }) + + it('leaves decimal references over 7 digits as a literal (exceeds CommonMark digit limit)', () => { + expect(decodeHtmlEntities('�')).toBe('�') + }) + + it('leaves hex references over 6 digits as a literal (exceeds CommonMark digit limit)', () => { + expect(decodeHtmlEntities('�')).toBe('�') + }) + + it('leaves an 8-digit decimal reference as a literal (format exceeds the limit)', () => { + expect(decodeHtmlEntities('�')).toBe('�') + }) + + it('replaces a 7-digit decimal reference whose value exceeds 0x10FFFF with U+FFFD', () => { + expect(decodeHtmlEntities('�')).toBe('\uFFFD') + }) + + it('replaces a 6-digit hex reference whose value exceeds 0x10FFFF with U+FFFD', () => { + expect(decodeHtmlEntities('�')).toBe('\uFFFD') + }) + + it('matches a 7-digit decimal reference greedily and leaves the trailing digit as a literal', () => { + expect(decodeHtmlEntities('�8')).toBe('\uFFFD8') + }) + + it('replaces � (NUL) with U+FFFD', () => { + expect(decodeHtmlEntities('�')).toBe('\uFFFD') + }) + + it('replaces a lone low surrogate � with U+FFFD', () => { + expect(decodeHtmlEntities('�')).toBe('\uFFFD') + }) + + it('replaces a lone high surrogate � with U+FFFD', () => { + expect(decodeHtmlEntities('�')).toBe('\uFFFD') + }) + + it('decodes ퟿ normally (just below the surrogate range)', () => { + expect(decodeHtmlEntities('퟿')).toBe(String.fromCodePoint(0xd7ff)) + }) + + it('decodes  normally (just above the surrogate range)', () => { + expect(decodeHtmlEntities('')).toBe(String.fromCodePoint(0xe000)) + }) + + it('decodes 􏿿 normally (the maximum valid code point)', () => { + expect(decodeHtmlEntities('􏿿')).toBe(String.fromCodePoint(0x10ffff)) + }) + + it('remaps € (0x80) to the Windows-1252 euro sign', () => { + expect(decodeHtmlEntities('€')).toBe('\u20AC') + }) + + it('remaps — (0x97) to the Windows-1252 em dash', () => { + expect(decodeHtmlEntities('—')).toBe('\u2014') + }) + + it('remaps “ (0x93) to the Windows-1252 left double quote', () => { + expect(decodeHtmlEntities('“')).toBe('\u201C') + }) + + it('remaps ” (0x94) to the Windows-1252 right double quote', () => { + expect(decodeHtmlEntities('”')).toBe('\u201D') + }) + + it('remaps € (hex form) to the Windows-1252 euro sign', () => { + expect(decodeHtmlEntities('€')).toBe('\u20AC') + }) + + it('leaves  (0x81) as the raw C1 control character (not in the Windows-1252 table)', () => { + expect(decodeHtmlEntities('')).toBe('\u0081') + }) + + it('leaves  (0x8D) as the raw C1 control character (not in the Windows-1252 table)', () => { + expect(decodeHtmlEntities('')).toBe('\u008D') + }) + + it('does not re-scan replaced output for a second round of decoding', () => { + expect(decodeHtmlEntities('&#x27;')).toBe(''') + }) + + it('does not turn a decoded & into a second-pass < decode', () => { + expect(decodeHtmlEntities('&lt;')).toBe('<') + }) + + it('does not turn a decoded & into a second-pass & decode', () => { + expect(decodeHtmlEntities('&amp;')).toBe('&') + }) + it('returns plain text unchanged', () => { expect(decodeHtmlEntities('hello world')).toBe('hello world') }) @@ -47,7 +151,7 @@ describe('roundtrip', () => { expect(decodeHtmlEntities(encodeHtmlEntities(input))).toBe(input) }) - it('decode is a superset of encode – " decodes but " is not encoded', () => { + it('decode is a superset of encode - " decodes but " is not encoded', () => { // " passes through encode unchanged, " decodes to " expect(encodeHtmlEntities('"hello"')).toBe('"hello"') expect(decodeHtmlEntities('"hello"')).toBe('"hello"') diff --git a/packages/core/src/utilities/htmlEntities.ts b/packages/core/src/utilities/htmlEntities.ts index ee04bf5e71..788701633f 100644 --- a/packages/core/src/utilities/htmlEntities.ts +++ b/packages/core/src/utilities/htmlEntities.ts @@ -1,15 +1,63 @@ +const NAMED_ENTITIES: Record = { lt: '<', gt: '>', quot: '"', amp: '&' } + /** * Decode common HTML entities in text content so they display as literal - * characters inside the editor. The decode order matters: `&` must be - * decoded **last** so that doubly-encoded sequences like `&lt;` first - * survive the `<` pass and then correctly become `<` (not `<`). + * characters inside the editor. Runs as a single pass so a decoded entity + * (e.g. `&` → `&`) is never re-scanned and decoded again. */ export function decodeHtmlEntities(text: string): string { - return text - .replace(/</g, '<') - .replace(/>/g, '>') - .replace(/"/g, '"') - .replace(/&/g, '&') + // Digit limits per CommonMark: decimal refs are 1-7 digits, hex refs are 1-6. + return text.replace( + /&(?:#(\d{1,7})|#[xX]([0-9a-fA-F]{1,6})|(lt|gt|quot|amp));/g, + (match, dec, hex, name) => { + if (dec !== undefined) { + return decodeCodePoint(Number(dec)) + } + if (hex !== undefined) { + return decodeCodePoint(parseInt(hex, 16)) + } + return NAMED_ENTITIES[name] + }, + ) +} + +// HTML5 maps numeric references in the C1 range (0x80-0x9F) to Windows-1252 characters. +const WINDOWS_1252 = new Map([ + [0x80, 0x20ac], + [0x82, 0x201a], + [0x83, 0x0192], + [0x84, 0x201e], + [0x85, 0x2026], + [0x86, 0x2020], + [0x87, 0x2021], + [0x88, 0x02c6], + [0x89, 0x2030], + [0x8a, 0x0160], + [0x8b, 0x2039], + [0x8c, 0x0152], + [0x8e, 0x017d], + [0x91, 0x2018], + [0x92, 0x2019], + [0x93, 0x201c], + [0x94, 0x201d], + [0x95, 0x2022], + [0x96, 0x2013], + [0x97, 0x2014], + [0x98, 0x02dc], + [0x99, 0x2122], + [0x9a, 0x0161], + [0x9b, 0x203a], + [0x9c, 0x0153], + [0x9e, 0x017e], + [0x9f, 0x0178], +]) + +// Replace invalid code points with U+FFFD, per the CommonMark spec. +function decodeCodePoint(codePoint: number): string { + if (codePoint === 0 || codePoint > 0x10ffff || (codePoint >= 0xd800 && codePoint <= 0xdfff)) { + return '\uFFFD' + } + return String.fromCodePoint(WINDOWS_1252.get(codePoint) ?? codePoint) } /** diff --git a/packages/extension-code/src/code.ts b/packages/extension-code/src/code.ts index 1f317bfbfc..74f3d505b2 100644 --- a/packages/extension-code/src/code.ts +++ b/packages/extension-code/src/code.ts @@ -93,6 +93,18 @@ export const pasteRegexMatch = (text: string): PasteRuleMatch[] => { return matches } +// A code span must be fenced by a backtick run its own content does not contain +function shortestUnusedBacktickRun(text: string): number { + const runs = new Set((text.match(/`+/g) || []).map(run => run.length)) + let length = 1 + + while (runs.has(length)) { + length += 1 + } + + return length +} + /** * This extension allows you to mark text as inline code. * @see https://tiptap.dev/api/marks/code @@ -128,12 +140,18 @@ export const Code = Mark.create({ return helpers.applyMark('code', [{ type: 'text', text: token.text || '' }]) }, - renderMarkdown: (node, h) => { + renderMarkdown: (node, h, context) => { if (!node.content) { return '' } - return `\`${h.renderChildren(node.content)}\`` + // Children are a placeholder here, so the fence comes from the full span text + const markText = typeof context?.meta?.markText === 'string' ? context.meta.markText : '' + const fence = '`'.repeat(shortestUnusedBacktickRun(markText)) + // CommonMark strips one space on each side, so padding keeps backticks apart + const padding = markText.startsWith('`') || markText.endsWith('`') ? ' ' : '' + + return `${fence}${padding}${h.renderChildren(node.content)}${padding}${fence}` }, addCommands() { diff --git a/packages/extension-list/src/ordered-list/collectOrderedListItemsFromSource.ts b/packages/extension-list/src/ordered-list/collectOrderedListItemsFromSource.ts new file mode 100644 index 0000000000..976085ebfb --- /dev/null +++ b/packages/extension-list/src/ordered-list/collectOrderedListItemsFromSource.ts @@ -0,0 +1,58 @@ +import type { OrderedListItem } from './utils.js' +import { collectOrderedListItems, ORDERED_LIST_ITEM_REGEX } from './utils.js' + +const ORDERED_LIST_SCAN_WINDOW = 64 + +// Fewer than `max` lines means `source` ended. +function takeLines(source: string, max: number): string[] { + const lines: string[] = [] + let start = 0 + + while (lines.length < max) { + const lineEnd = source.indexOf('\n', start) + + if (lineEnd === -1) { + lines.push(source.slice(start)) + return lines + } + + lines.push(source.slice(start, lineEnd)) + start = lineEnd + 1 + } + + return lines +} + +/** + * Collect ordered list items from the start of `source`, reading only the lines the list needs. + * + * @param source Markdown beginning at a block boundary. + * @returns The items, how many lines they consumed, and the lines that were read. + * @example + * collectOrderedListItemsFromSource('1. one\n2. two\n\nAfter') + * // → [[item, item], 3, ['1. one', '2. two', '', 'After']] + */ +export function collectOrderedListItemsFromSource( + source: string, +): [OrderedListItem[], number, string[]] { + const firstLineEnd = source.indexOf('\n') + const firstLine = firstLineEnd === -1 ? source : source.slice(0, firstLineEnd) + + // Every block start is offered here, so reject non-lists before reading further. + if (!ORDERED_LIST_ITEM_REGEX.test(firstLine)) { + return [[], 0, []] + } + + let windowSize = ORDERED_LIST_SCAN_WINDOW + let lines = takeLines(source, windowSize) + let collected = collectOrderedListItems(lines) + + // Consuming the whole window means the list may continue past it. + while (collected[1] === lines.length && lines.length === windowSize) { + windowSize *= 2 + lines = takeLines(source, windowSize) + collected = collectOrderedListItems(lines) + } + + return [collected[0], collected[1], lines] +} diff --git a/packages/extension-list/src/ordered-list/ordered-list.ts b/packages/extension-list/src/ordered-list/ordered-list.ts index 13a079ec41..cba88d8273 100644 --- a/packages/extension-list/src/ordered-list/ordered-list.ts +++ b/packages/extension-list/src/ordered-list/ordered-list.ts @@ -3,12 +3,8 @@ import { Plugin } from '@tiptap/pm/state' import { mergeAttributes, Node, wrappingInputRule } from '@tiptap/core' -import { - buildNestedStructure, - collectOrderedListItems, - parseListItems, - parsePlainTextOrderedListPaste, -} from './utils.js' +import { collectOrderedListItemsFromSource } from './collectOrderedListItemsFromSource.js' +import { buildNestedStructure, parseListItems, parsePlainTextOrderedListPaste } from './utils.js' const ListItemName = 'listItem' const TextStyleName = 'textStyle' @@ -236,8 +232,7 @@ export const OrderedList = Node.create({ // `tokenize`, which would re-introduce the mid-line split. start: () => -1, tokenize: (src: string, _tokens, lexer) => { - const lines = src.split('\n') - const [listItems, consumed] = collectOrderedListItems(lines) + const [listItems, consumed, lines] = collectOrderedListItemsFromSource(src) if (listItems.length === 0) { return undefined diff --git a/packages/extension-list/tests/integration/orderedListLargeDocument.test.ts b/packages/extension-list/tests/integration/orderedListLargeDocument.test.ts new file mode 100644 index 0000000000..0437b7b3d5 --- /dev/null +++ b/packages/extension-list/tests/integration/orderedListLargeDocument.test.ts @@ -0,0 +1,73 @@ +import Blockquote from '@tiptap/extension-blockquote' +import Document from '@tiptap/extension-document' +import Heading from '@tiptap/extension-heading' +import Paragraph from '@tiptap/extension-paragraph' +import Text from '@tiptap/extension-text' +import { MarkdownManager } from '@tiptap/markdown' +import { describe, expect, it } from 'vite-plus/test' + +import { BulletList, ListItem, OrderedList } from '../../src/index.js' + +const markdownManager = new MarkdownManager({ + extensions: [Document, Paragraph, Text, Heading, Blockquote, BulletList, OrderedList, ListItem], +}) + +const buildDocument = (sections: number) => + Array.from( + { length: sections }, + (_, index) => + `## Section ${index}\n\nParagraph ${index} text.\n\n` + + `- bullet ${index}\n\n1. first ${index}\n2. second ${index}\n\n> quote ${index}`, + ).join('\n\n') + +describe('ordered lists in large documents', () => { + it('reads a list that runs past the initial scan window', () => { + const items = 200 + const markdown = Array.from( + { length: items }, + (_, index) => `${index + 1}. item ${index}`, + ).join('\n') + + const json = markdownManager.parse(markdown) + + expect(json.content).toHaveLength(1) + expect(json.content?.[0].type).toBe('orderedList') + expect(json.content?.[0].content).toHaveLength(items) + }) + + it('stops the list at the following paragraph', () => { + const markdown = `${Array.from({ length: 120 }, (_, index) => `${index + 1}. item ${index}`).join('\n')}\n\nAfter the list.` + + const json = markdownManager.parse(markdown) + + expect(json.content).toHaveLength(2) + expect(json.content?.[0].content).toHaveLength(120) + expect(json.content?.[1]).toEqual({ + type: 'paragraph', + content: [{ type: 'text', text: 'After the list.' }], + }) + }) + + it('matches a single parse of the same list split across the window boundary', () => { + const short = markdownManager.parse('1. one\n2. two') + const long = markdownManager.parse( + Array.from({ length: 70 }, (_, index) => `${index + 1}. item ${index}`).join('\n'), + ) + + expect(short.content?.[0].type).toBe('orderedList') + expect(long.content?.[0].content).toHaveLength(70) + }) + + it('parses a large document in roughly linear time', () => { + const markdown = buildDocument(6000) + + expect(Buffer.byteLength(markdown)).toBeGreaterThan(400_000) + + const startedAt = performance.now() + markdownManager.parse(markdown) + const duration = performance.now() - startedAt + + // Quadratic scanning took ~37s here; ~250ms now. Loose enough for CI. + expect(duration).toBeLessThan(2000) + }, 60000) +}) diff --git a/packages/extension-table/src/table/utilities/markdown.test.ts b/packages/extension-table/src/table/utilities/markdown.test.ts index c1b9020477..b57c9ef887 100644 --- a/packages/extension-table/src/table/utilities/markdown.test.ts +++ b/packages/extension-table/src/table/utilities/markdown.test.ts @@ -1,6 +1,31 @@ +import type { JSONContent, MarkdownRendererHelpers } from '@tiptap/core' import { describe, expect, it } from 'vite-plus/test' -import { escapeTableCellPipes } from './markdown.js' +import { + DEFAULT_CELL_LINE_SEPARATOR, + escapeTableCellPipes, + renderTableToMarkdown, +} from './markdown.js' + +function renderInline(node: JSONContent): string { + if (node.type === 'text') { + return node.text ?? '' + } + + return (node.content ?? []).map(child => renderInline(child)).join('') +} + +const markdownHelpers: MarkdownRendererHelpers = { + renderChildren: nodes => { + if (Array.isArray(nodes)) { + return nodes.map(node => renderInline(node)).join('') + } + + return renderInline(nodes) + }, + wrapInBlock: (prefix, content) => prefix + content, + indent: content => content, +} describe('escapeTableCellPipes', () => { it('escapes a bare pipe inside a code span', () => { @@ -23,3 +48,76 @@ describe('escapeTableCellPipes', () => { expect(escapeTableCellPipes('| `|\\||` |')).toBe('| `\\|\\|\\|` |') }) }) + +describe('renderTableToMarkdown', () => { + const paragraph = (text: string): JSONContent => ({ + type: 'paragraph', + content: [{ type: 'text', text }], + }) + + it('does not emit U+001F when a cell has multiple block children', () => { + const markdown = renderTableToMarkdown( + { + type: 'table', + content: [ + { + type: 'tableRow', + content: [ + { type: 'tableCell', content: [paragraph('line one'), paragraph('line two')] }, + { type: 'tableCell', content: [paragraph('B')] }, + ], + }, + ], + }, + markdownHelpers, + ) + + expect(markdown).not.toContain(DEFAULT_CELL_LINE_SEPARATOR) + expect(markdown).not.toContain('\u001F') + expect(markdown).toContain('line one
line two') + }) + + it('strips a leftover cell-line separator even when it is passed as an option', () => { + const markdown = renderTableToMarkdown( + { + type: 'table', + content: [ + { + type: 'tableRow', + content: [ + { type: 'tableCell', content: [paragraph('line one'), paragraph('line two')] }, + ], + }, + ], + }, + markdownHelpers, + { cellLineSeparator: DEFAULT_CELL_LINE_SEPARATOR }, + ) + + expect(markdown).not.toContain('\u001F') + expect(markdown).toContain('line one
line two') + }) + + it('preserves an empty paragraph when cellLineSeparator is two newlines', () => { + const markdown = renderTableToMarkdown( + { + type: 'table', + content: [ + { + type: 'tableRow', + content: [ + { + type: 'tableCell', + content: [paragraph('one'), { type: 'paragraph', content: [] }, paragraph('two')], + }, + ], + }, + ], + }, + markdownHelpers, + { cellLineSeparator: '\n\n' }, + ) + + expect(markdown).toContain('one

two') + }) +}) diff --git a/packages/extension-table/src/table/utilities/markdown.ts b/packages/extension-table/src/table/utilities/markdown.ts index 0928f66a81..7304aaa7e2 100644 --- a/packages/extension-table/src/table/utilities/markdown.ts +++ b/packages/extension-table/src/table/utilities/markdown.ts @@ -82,7 +82,20 @@ export function preprocessTablePipes(src: string): string { } function collapseWhitespace(s: string) { - return (s || '').replace(/\s+/g, ' ').trim() + return (s || '').replaceAll(DEFAULT_CELL_LINE_SEPARATOR, ' ').replace(/\s+/g, ' ').trim() +} + +function replaceCellLineSeparator(value: string, cellLineSeparator?: string) { + if (!cellLineSeparator) { + return value + } + + return value.split(cellLineSeparator).join('\n') +} + +function escapePipesInCellText(text: string) { + // Keep already escaped pipes as they are + return text.replace(/\\.|\|/g, match => (match === '|' ? '\\|' : match)) } export function renderTableToMarkdown( @@ -90,8 +103,6 @@ export function renderTableToMarkdown( h: MarkdownRendererHelpers, options: { cellLineSeparator?: string } = {}, ) { - const cellSep = options.cellLineSeparator ?? DEFAULT_CELL_LINE_SEPARATOR - if (!node || !node.content || node.content.length === 0) { return '' } @@ -107,25 +118,24 @@ export function renderTableToMarkdown( let raw = '' if (cellNode.content && Array.isArray(cellNode.content) && cellNode.content.length > 1) { - // Render each direct child separately and join with separator so we can split again later const parts = cellNode.content.map(child => - h.renderChildren(child as unknown as JSONContent), + replaceCellLineSeparator( + h.renderChildren(child as unknown as JSONContent), + options.cellLineSeparator, + ), ) - raw = parts.join(cellSep) + raw = parts.join('\n') } else { - raw = cellNode.content - ? h.renderChildren(cellNode.content as unknown as JSONContent[]) - : '' + raw = replaceCellLineSeparator( + cellNode.content ? h.renderChildren(cellNode.content as unknown as JSONContent[]) : '', + options.cellLineSeparator, + ) } // Cells have to stay on a single line, so line breaks become
tags. // The parser already turns
back into hard breaks, so this round trips. - const text = collapseWhitespace( - raw - .split(cellSep) - .join('\n') - .replace(/[ \t]*\r?\n[ \t]*/g, '
'), - ) + const singleLine = collapseWhitespace(raw.replace(/[ \t]*\r?\n[ \t]*/g, '
')) + const text = escapePipesInCellText(singleLine) const isHeader = cellNode.type === 'tableHeader' const align = normalizeTableCellAlignFromAttributes(cellNode.attrs) diff --git a/packages/extension-table/tests/integration/tableMarkdown.test.ts b/packages/extension-table/tests/integration/tableMarkdown.test.ts index 2d1f9cf63e..9a3306e8dd 100644 --- a/packages/extension-table/tests/integration/tableMarkdown.test.ts +++ b/packages/extension-table/tests/integration/tableMarkdown.test.ts @@ -248,17 +248,20 @@ describe('table markdown line breaks', () => { expect(serialized).toContain('foo
bar') }) - it('joins multiple paragraphs in a cell with
', () => { + it('joins multiple paragraphs in a cell with
and does not emit U+001F', () => { const doc = tableDoc([ cell([ - { type: 'paragraph', content: [{ type: 'text', text: 'one' }] }, - { type: 'paragraph', content: [{ type: 'text', text: 'two' }] }, + { type: 'paragraph', content: [{ type: 'text', text: 'line one' }] }, + { type: 'paragraph', content: [{ type: 'text', text: 'line two' }] }, ]), + cell([{ type: 'paragraph', content: [{ type: 'text', text: 'B' }] }]), ]) const serialized = markdownManager.serialize(doc) - expect(serialized).toContain('one
two') + expect(serialized).toContain('line one
line two') + expect(serialized).toContain('| B') + expect(serialized.includes(String.fromCharCode(31))).toBe(false) expect(serialized).not.toContain('\u001F') }) @@ -273,3 +276,33 @@ describe('table markdown line breaks', () => { expect(serialized).not.toContain('
') }) }) +describe('table markdown — pipe escaping when serializing cells', () => { + const manager = new MarkdownManager({ + extensions: [Document, Paragraph, Text, Code, TableKit], + }) + + const markdown = '| Col | Value |\n| --- | --- |\n| a | uses \\| pipe |' + + it('escapes a bare pipe in cell text', () => { + const serialized = manager.serialize(manager.parse(markdown)) + + expect(serialized).toContain('uses \\| pipe') + }) + + it('round trips a cell with an escaped pipe', () => { + const serialized = manager.serialize(manager.parse(markdown)) + const reparsed = manager.parse(serialized) + + expect(manager.serialize(reparsed)).toBe(serialized) + + const cell = reparsed.content?.[0]?.content?.[1]?.content?.[1] + expect(cell?.content?.[0]?.content?.[0]?.text).toBe('uses | pipe') + }) + + it('escapes pipes inside a code span in a cell', () => { + const codeSpanMarkdown = '| H |\n| - |\n| `a || b` |' + const serialized = manager.serialize(manager.parse(codeSpanMarkdown)) + + expect(serialized).toContain('`a \\|\\| b`') + }) +}) diff --git a/packages/markdown/src/MarkdownManager.ts b/packages/markdown/src/MarkdownManager.ts index 8ca86d3e78..21d28735c0 100644 --- a/packages/markdown/src/MarkdownManager.ts +++ b/packages/markdown/src/MarkdownManager.ts @@ -33,6 +33,7 @@ import { reopenMarksAfterNode, wrapInMarkdownBlock, } from './utils.js' +import { collectMarkSpanTexts } from './utils/collectMarkSpanTexts.js' import { htmlContainsUnrecognizedTag } from './utils/htmlTagDetection.js' export class MarkdownManager { @@ -1277,6 +1278,8 @@ export class MarkdownManager { const activeMarks: Map = new Map() const reopenWithHtmlOnNextOpen = new Set() const markOpeningModes = new Map() + // Delimiters can depend on the whole mark run, not just one text node + const markSpanTexts = collectMarkSpanTexts(nodes) nodes.forEach((node, i) => { // Lookahead to the next node to determine if marks need to be closed const nextNode = i < nodes.length - 1 ? nodes[i + 1] : null @@ -1350,6 +1353,7 @@ export class MarkdownManager { markType, mark, markOpeningModes.get(markType), + markSpanTexts[i]?.get(markType), ) if (closeMarkdown) { textContent += closeMarkdown @@ -1378,7 +1382,12 @@ export class MarkdownManager { // so new marks correctly see them as active context. marksToOpen.forEach(({ type, mark }) => { const openingMode = reopenWithHtmlOnNextOpen.has(type) ? 'html' : 'markdown' - const openMarkdown = this.getMarkOpening(type, mark, openingMode) + const openMarkdown = this.getMarkOpening( + type, + mark, + openingMode, + markSpanTexts[i]?.get(type), + ) if (openMarkdown) { textContent = openMarkdown + textContent } @@ -1445,7 +1454,12 @@ export class MarkdownManager { marksToCloseAtEnd.forEach(markType => { const mark = activeMarks.get(markType) ?? currentMarks.get(markType) - const closeMarkdown = this.getMarkClosing(markType, mark, markOpeningModes.get(markType)) + const closeMarkdown = this.getMarkClosing( + markType, + mark, + markOpeningModes.get(markType), + markSpanTexts[i]?.get(markType), + ) if (closeMarkdown) { textContent += closeMarkdown } @@ -1472,8 +1486,14 @@ export class MarkdownManager { }) // Close all marks before the node + // The run ends on the previous node and restarts on the next one const beforeMarkdown = closeMarksBeforeNode(activeMarks, (markType, mark) => { - return this.getMarkClosing(markType, mark, markOpeningModes.get(markType)) + return this.getMarkClosing( + markType, + mark, + markOpeningModes.get(markType), + markSpanTexts[i - 1]?.get(markType), + ) }) markOpeningModes.clear() @@ -1488,7 +1508,12 @@ export class MarkdownManager { : reopenMarksAfterNode(marksToReopen, activeMarks, (markType, mark) => { const openingMode = openingModesToReopen.get(markType) ?? 'markdown' markOpeningModes.set(markType, openingMode) - return this.getMarkOpening(markType, mark, openingMode) + return this.getMarkOpening( + markType, + mark, + openingMode, + markSpanTexts[i + 1]?.get(markType), + ) }) result.push(beforeMarkdown + nodeContent + afterMarkdown) @@ -1505,6 +1530,7 @@ export class MarkdownManager { markType: string, mark: any, openingMode: 'markdown' | 'html' = 'markdown', + markText?: string, ): string { if (openingMode === 'html') { return this.getHtmlReopenTags(markType)?.open || '' @@ -1535,7 +1561,7 @@ export class MarkdownManager { indent: (content: string) => content, wrapInBlock: (prefix: string, content: string) => prefix + content, }, - { index: 0, level: 0, parentType: 'text', meta: {} }, + { index: 0, level: 0, parentType: 'text', meta: { markText } }, ) // Extract the opening part (everything before placeholder) @@ -1553,6 +1579,7 @@ export class MarkdownManager { markType: string, mark: any, openingMode: 'markdown' | 'html' = 'markdown', + markText?: string, ): string { if (openingMode === 'html') { return this.getHtmlReopenTags(markType)?.close || '' @@ -1582,7 +1609,7 @@ export class MarkdownManager { indent: (content: string) => content, wrapInBlock: (prefix: string, content: string) => prefix + content, }, - { index: 0, level: 0, parentType: 'text', meta: {} }, + { index: 0, level: 0, parentType: 'text', meta: { markText } }, ) // Extract the closing part (everything after placeholder) diff --git a/packages/markdown/src/utils/collectMarkSpanTexts.ts b/packages/markdown/src/utils/collectMarkSpanTexts.ts new file mode 100644 index 0000000000..eb7392d9ab --- /dev/null +++ b/packages/markdown/src/utils/collectMarkSpanTexts.ts @@ -0,0 +1,40 @@ +import type { JSONContent } from '@tiptap/core' + +// Gives each node the full text of its mark run, so a delimiter can fit the whole span +export function collectMarkSpanTexts(nodes: JSONContent[]): Map[] { + const spanTexts: Map[] = nodes.map(() => new Map()) + const openRuns = new Map() + + const flush = (markType: string) => { + const run = openRuns.get(markType) + if (!run) { + return + } + run.indexes.forEach(index => spanTexts[index].set(markType, run.text)) + openRuns.delete(markType) + } + + nodes.forEach((node, index) => { + if (node.type !== 'text') { + Array.from(openRuns.keys()).forEach(flush) + return + } + + const markTypes = new Set((node.marks || []).map(mark => mark.type)) + + Array.from(openRuns.keys()) + .filter(markType => !markTypes.has(markType)) + .forEach(flush) + + markTypes.forEach(markType => { + const run = openRuns.get(markType) || { indexes: [], text: '' } + run.indexes.push(index) + run.text += node.text || '' + openRuns.set(markType, run) + }) + }) + + Array.from(openRuns.keys()).forEach(flush) + + return spanTexts +} diff --git a/packages/markdown/tests/integration/code-span-backticks.test.ts b/packages/markdown/tests/integration/code-span-backticks.test.ts new file mode 100644 index 0000000000..8ce31d0590 --- /dev/null +++ b/packages/markdown/tests/integration/code-span-backticks.test.ts @@ -0,0 +1,79 @@ +import { Code } from '@tiptap/extension-code' +import { Document } from '@tiptap/extension-document' +import { Image } from '@tiptap/extension-image' +import { Paragraph } from '@tiptap/extension-paragraph' +import { Text } from '@tiptap/extension-text' +import { describe, expect, it } from 'vite-plus/test' + +import { MarkdownManager } from '../../src/MarkdownManager.js' + +describe('Code spans containing backticks', () => { + const markdownManager = new MarkdownManager({ extensions: [Document, Paragraph, Text, Code] }) + + const codeSpan = (text: string) => ({ + type: 'doc', + content: [{ type: 'paragraph', content: [{ type: 'text', text, marks: [{ type: 'code' }] }] }], + }) + + it.each([ + ['hello', '`hello`'], + ['hello ` world', '``hello ` world``'], + ['a ` b `` c', '```a ` b `` c```'], + ['`leading', '`` `leading ``'], + ['trailing`', '`` trailing` ``'], + ['`', '`` ` ``'], + ['``', '` `` `'], + ])('serializes %j with a fence the content cannot close', (text, expected) => { + expect(markdownManager.serialize(codeSpan(text))).toBe(expected) + }) + + it.each(['hello', 'hello ` world', 'a ` b `` c', '`leading', 'trailing`', '`', '``'])( + 'round-trips %j through serialize and parse', + text => { + const parsed = markdownManager.parse(markdownManager.serialize(codeSpan(text))) + + expect(parsed.content?.[0].content).toEqual([ + { type: 'text', text, marks: [{ type: 'code' }] }, + ]) + }, + ) + + it('keeps a code span with a backtick separate from its surrounding text', () => { + const doc = { + type: 'doc', + content: [ + { + type: 'paragraph', + content: [ + { type: 'text', text: 'Press ' }, + { type: 'text', text: '`', marks: [{ type: 'code' }] }, + { type: 'text', text: ' to open.' }, + ], + }, + ], + } + + const markdown = markdownManager.serialize(doc) + + expect(markdown).toBe('Press `` ` `` to open.') + expect(markdownManager.parse(markdown).content?.[0].content).toEqual(doc.content[0].content) + }) + + it('fences both sides of a code mark that spans an inline atom', () => { + const withImage = new MarkdownManager({ + extensions: [Document, Paragraph, Text, Code, Image.configure({ inline: true })], + }) + const content = [ + { type: 'text', text: 'a`b', marks: [{ type: 'code' }] }, + { type: 'image', attrs: { src: 'x.png' }, marks: [{ type: 'code' }] }, + { type: 'text', text: 'c`d', marks: [{ type: 'code' }] }, + ] + + const markdown = withImage.serialize({ + type: 'doc', + content: [{ type: 'paragraph', content }], + }) + + expect(markdown).toBe('``a`b``![](x.png)``c`d``') + }) +}) diff --git a/packages/markdown/tests/integration/conversion.test.ts b/packages/markdown/tests/integration/conversion.test.ts index 7ba0bb6ba1..818166f179 100644 --- a/packages/markdown/tests/integration/conversion.test.ts +++ b/packages/markdown/tests/integration/conversion.test.ts @@ -538,6 +538,20 @@ describe('Markdown Conversion Tests', () => { expect(serialized).toBe('foo &lt; bar') }) + it('should decode decimal numeric entities like ' when parsing', () => { + const markdown = 'it's a test' + const json = markdownManager.parse(markdown) + + expect(json.content[0].content[0].text).toBe("it's a test") + }) + + it('should decode hex numeric entities like ' when parsing', () => { + const markdown = 'it's a test' + const json = markdownManager.parse(markdown) + + expect(json.content[0].content[0].text).toBe("it's a test") + }) + it('should preserve   empty paragraph behavior', () => { const markdown = 'Line1\n\n \n\nLine2' const json = markdownManager.parse(markdown)