From 2ff818c1fc64c28ef017d6f2a731df274971b18e Mon Sep 17 00:00:00 2001 From: onmax Date: Sun, 1 Nov 2026 12:15:30 +0100 Subject: [PATCH 01/15] fix(html): read braced attribute values, --> comments and the first duplicate attribute MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A JSX-style value (`
`) failed CommonMark's attribute grammar, so the tag became text. The inline rule and the raw-block splitter now share one tag grammar that accepts balanced `{…}` values and comments ending in `--->`. A repeated attribute keeps its first value, as in HTML. --- .../SPEC/HTML/attribute-jsx-expression.md | 45 +++++++++++++++++ .../comark/SPEC/HTML/duplicate-attributes.md | 38 ++++++++++++++ .../comark/src/internal/parse/html-tag.ts | 27 ++++++++++ packages/comark/src/internal/parse/utils.ts | 14 ++++-- packages/comark/src/plugins/html.ts | 47 ++++++++--------- packages/comark/test/html-tags.test.ts | 50 +++++++++++++++++++ 6 files changed, 192 insertions(+), 29 deletions(-) create mode 100644 packages/comark/SPEC/HTML/attribute-jsx-expression.md create mode 100644 packages/comark/SPEC/HTML/duplicate-attributes.md create mode 100644 packages/comark/src/internal/parse/html-tag.ts create mode 100644 packages/comark/test/html-tags.test.ts diff --git a/packages/comark/SPEC/HTML/attribute-jsx-expression.md b/packages/comark/SPEC/HTML/attribute-jsx-expression.md new file mode 100644 index 000000000..292f5fb57 --- /dev/null +++ b/packages/comark/SPEC/HTML/attribute-jsx-expression.md @@ -0,0 +1,45 @@ +## Input + +```md +
+ +x +``` + +## AST + +```json +{ + "frontmatter": {}, + "meta": {}, + "nodes": [ + [ + "div", + { + "$": { + "html": 1, + "block": 1 + }, + "style": "{{a: 1}}" + }, + "x" + ] + ] +} +``` + +## HTML + +```html +
+ x +
+``` + +## Markdown + +```md +
+x +
+``` diff --git a/packages/comark/SPEC/HTML/duplicate-attributes.md b/packages/comark/SPEC/HTML/duplicate-attributes.md new file mode 100644 index 000000000..085e569ed --- /dev/null +++ b/packages/comark/SPEC/HTML/duplicate-attributes.md @@ -0,0 +1,38 @@ +## Input + +```md +first +``` + +## AST + +```json +{ + "frontmatter": {}, + "meta": {}, + "nodes": [ + [ + "img", + { + "$": { + "html": 1, + "block": 1 + }, + "alt": "first" + } + ] + ] +} +``` + +## HTML + +```html +first +``` + +## Markdown + +```md +first +``` diff --git a/packages/comark/src/internal/parse/html-tag.ts b/packages/comark/src/internal/parse/html-tag.ts new file mode 100644 index 000000000..ee3a5fc4d --- /dev/null +++ b/packages/comark/src/internal/parse/html-tag.ts @@ -0,0 +1,27 @@ +// Adapted from markdown-exit's html_re, with two changes: attribute values may be +// a balanced `{…}` expression (JSX style), and a comment may end in `--->` as in HTML. + +const attr_name = '[a-zA-Z_:][a-zA-Z0-9:._-]*' + +const braces = (depth: number): string => (depth === 0 ? '[^{}]*' : `(?:[^{}]|\\{${braces(depth - 1)}\\})*`) + +/** Source of a `{…}` attribute value. Nesting is limited to four levels. */ +export const braced_value = `\\{${braces(3)}\\}` + +const unquoted = '[^"\'=<>`\\x00-\\x20]+' +const single_quoted = "'[^']*'" +const double_quoted = '"[^"]*"' + +const attr_value = `(?:${braced_value}|${unquoted}|${single_quoted}|${double_quoted})` + +const attribute = `(?:\\s+${attr_name}(?:\\s*=\\s*${attr_value})?)` + +const open_tag = `<[A-Za-z][A-Za-z0-9\\-]*${attribute}*\\s*\\/?>` + +const close_tag = '<\\/[A-Za-z][A-Za-z0-9\\-]*\\s*>' +const comment = ')' +const processing = '<[?][\\s\\S]*?[?]>' +const declaration = ']*>' +const cdata = '' + +export const HTML_TAG_RE = new RegExp(`^(?:${open_tag}|${close_tag}|${comment}|${processing}|${declaration}|${cdata})`) diff --git a/packages/comark/src/internal/parse/utils.ts b/packages/comark/src/internal/parse/utils.ts index 8bcc6a825..7ede1e272 100644 --- a/packages/comark/src/internal/parse/utils.ts +++ b/packages/comark/src/internal/parse/utils.ts @@ -1,4 +1,5 @@ import type { Node, ElementNode, ElementNodeAttributes, MarkdownDocument } from 'comark' +import { braced_value } from './html-tag.ts' import { decodeHTML, isHtmlVoidElement, TREE_WALK_MAX_DEPTH } from 'comark/utils' export type ParsedHtmlTag = @@ -187,14 +188,19 @@ export function parseHtmlInline(content: string): ParsedHtmlTag { * Names are lowercased, as HTML attribute names are case-insensitive. */ export function parseHtmlAttributes(attrsStr: string): Record { +const ATTRIBUTE_RE = new RegExp( + `([:\\w.-]+)(?:\\s*=\\s*(?:"([^"]*)"|'([^']*)'|(${braced_value})|([^\\s"'=<>\`]+)))?`, + 'g' +) + const attrs: Record = {} if (!attrsStr || !attrsStr.trim()) return attrs - const re = /([:\w.-]+)(?:\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'=<>`]+)))?/g - let match: RegExpExecArray | null - while ((match = re.exec(attrsStr)) !== null) { + for (const match of attrsStr.matchAll(ATTRIBUTE_RE)) { const key = match[1].toLowerCase() - const value = match[2] ?? match[3] ?? match[4] + // The first occurrence wins, as in HTML. + if (Object.hasOwn(attrs, key)) continue + const value = match[2] ?? match[3] ?? match[4] ?? match[5] attrs[key] = value === undefined ? true : decodeHTML(value) } diff --git a/packages/comark/src/plugins/html.ts b/packages/comark/src/plugins/html.ts index 28ec8835a..c2cb9721c 100644 --- a/packages/comark/src/plugins/html.ts +++ b/packages/comark/src/plugins/html.ts @@ -35,9 +35,10 @@ * ``` */ -import { type StateBlock, Token, type MarkdownExit } from 'markdown-exit' +import { type StateBlock, type StateInline, Token, type MarkdownExit } from 'markdown-exit' import type { ComarkParseTokensState } from '../types.ts' import { defineComarkPlugin } from '../utils/helpers.ts' +import { HTML_TAG_RE } from '../internal/parse/html-tag.ts' export interface HtmlPluginOptions { /** @@ -59,6 +60,26 @@ export default defineComarkPlugin((opts: HtmlPluginOptions = {}) => { function markdownItHtmlWithoutMarkdown(md: MarkdownExit) { md.set({ html: true }) + md.inline.ruler.at('html_inline', htmlInline) +} + +// markdown-exit's rule with Comark's tag grammar, so inline and block scanning agree. +function htmlInline(state: StateInline, silent: boolean): boolean { + if (!state.md.options.html) return false + const pos = state.pos + if (state.src.charCodeAt(pos) !== 0x3c /* < */ || pos + 2 >= state.posMax) return false + const next = state.src.charCodeAt(pos + 1) + if (next !== 0x21 /* ! */ && next !== 0x3f /* ? */ && next !== 0x2f /* / */ && !isLetter(next)) return false + const match = HTML_TAG_RE.exec(state.src.slice(pos, state.posMax)) + if (!match) return false + if (!silent) { + const token = state.push('html_inline', '', 0) + token.content = match[0] + if (/^\s]/i.test(match[0])) state.linkLevel++ + if (/^<\/a\s*>/i.test(match[0])) state.linkLevel-- + } + state.pos += match[0].length + return true } function markdownItHtmlWithMarkdown(md: MarkdownExit) { @@ -103,30 +124,6 @@ function markdownItPost(state: ComarkParseTokensState) { } } -// region - https://github.com/markdown-it/markdown-it/blob/master/src/common/html_re.ts#L21 - -const attr_name = '[a-zA-Z_:][a-zA-Z0-9:._-]*' - -const unquoted = '[^"\'=<>`\\x00-\\x20]+' -const single_quoted = "'[^']*'" -const double_quoted = '"[^"]*"' - -const attr_value = `(?:${unquoted}|${single_quoted}|${double_quoted})` - -const attribute = `(?:\\s+${attr_name}(?:\\s*=\\s*${attr_value})?)` - -const open_tag = `<[A-Za-z][A-Za-z0-9\\-]*${attribute}*\\s*\\/?>` - -const close_tag = '<\\/[A-Za-z][A-Za-z0-9\\-]*\\s*>' -const comment = '' -const processing = '<[?][\\s\\S]*?[?]>' -const declaration = ']*>' -const cdata = '' - -const HTML_TAG_RE = new RegExp(`^(?:${open_tag}|${close_tag}|${comment}|${processing}|${declaration}|${cdata})`) - -// end region - function isLetter(code: number): boolean { return (code >= 0x41 && code <= 0x5a) || (code >= 0x61 && code <= 0x7a) } diff --git a/packages/comark/test/html-tags.test.ts b/packages/comark/test/html-tags.test.ts new file mode 100644 index 000000000..083c28c0c --- /dev/null +++ b/packages/comark/test/html-tags.test.ts @@ -0,0 +1,50 @@ +import { describe, expect, it } from 'vitest' +import { createMarkdownParser, parseMarkdown } from '../src/index' +import html from '../src/plugins/html' +import { renderMarkdown } from '../src/render' +import { renderHtmlForTest } from './utils/render-html' + +const block = { html: 1, block: 1 } +const inline = { html: 1, block: 0 } + +describe('HTML attributes', () => { + it('keeps a JSX-style braced value on an open tag', async () => { + const result = await parseMarkdown('
x
') + + expect(result.nodes).toEqual([['div', { $: block, style: '{{a: 1}}' }, 'x']]) + }) + + it('keeps a braced value with nested braces, quotes and spaces on an inline tag', async () => { + const result = await parseMarkdown('a y') + + expect(result.nodes).toEqual([ + ['p', {}, 'a ', ['b', { $: inline, data: '{`x ${1}`}', on: '{{ k: { v: 1 } }}', d: true }, 'y']], + ]) + }) + + it('keeps the first of duplicate attributes', async () => { + const result = await parseMarkdown('first\n\nt') + + expect(result.nodes).toEqual([ + ['img', { $: block, alt: 'first' }], + ['a', { $: inline, href: 'x', title: true }, 't'], + ]) + }) + + it('keeps an attribute named like an Object.prototype member', async () => { + const result = await parseMarkdown('
x
') + + expect(result.nodes).toEqual([['div', { $: block, constructor: 'a' }, 'x']]) + }) +}) + +describe('HTML comments', () => { + it('reads as a comment inline and as a block', async () => { + const result = await parseMarkdown('a c\n\n') + + expect(result.nodes).toEqual([ + ['p', {}, 'a ', [null, {}, '- b -'], ' c'], + [null, {}, '- d -'], + ]) + }) +}) From c23231c1028ca9a96e78e9aed5170b2929eeedc2 Mon Sep 17 00:00:00 2001 From: onmax Date: Sun, 1 Nov 2026 12:20:22 +0100 Subject: [PATCH 02/15] fix(html): pair implied end tags and keep inline content like browsers A stray `
` was dropped and a second `

` nested in the first; `
` is now a `
` and `

` closes an open `

`, except across table, cell, caption, button, template, object, marquee and applet boundaries. Inside an element that collects blocks, an image, `
`, component or custom element ended its paragraph; inline nodes now stay together and a lone author-written element is not wrapped. An open tag that follows text in its paragraph (`GET /apps/`, `- **a** c`) ends with its paragraph unless a later paragraph closes it, instead of swallowing later blocks or reordering a list item. --- .../HTML/block-paragraph-inline-content.md | 78 +++++++++ packages/comark/SPEC/HTML/close-br-tag.md | 43 +++++ packages/comark/SPEC/HTML/p-closes-p.md | 54 +++++++ .../comark/SPEC/HTML/unclosed-after-text.md | 54 +++++++ packages/comark/SPEC/HTML/unclosed-in-list.md | 56 +++++++ packages/comark/src/internal/parse/tree.ts | 105 ++++++++++-- packages/comark/test/html-tags.test.ts | 151 ++++++++++++++++++ 7 files changed, 526 insertions(+), 15 deletions(-) create mode 100644 packages/comark/SPEC/HTML/block-paragraph-inline-content.md create mode 100644 packages/comark/SPEC/HTML/close-br-tag.md create mode 100644 packages/comark/SPEC/HTML/p-closes-p.md create mode 100644 packages/comark/SPEC/HTML/unclosed-after-text.md create mode 100644 packages/comark/SPEC/HTML/unclosed-in-list.md diff --git a/packages/comark/SPEC/HTML/block-paragraph-inline-content.md b/packages/comark/SPEC/HTML/block-paragraph-inline-content.md new file mode 100644 index 000000000..33aaa4d0c --- /dev/null +++ b/packages/comark/SPEC/HTML/block-paragraph-inline-content.md @@ -0,0 +1,78 @@ +--- +options: + autoUnwrap: false +--- + +## Input + +```md +

+ +![](x.png) + +a\ +b + +
+``` + +## AST + +```json +{ + "frontmatter": {}, + "meta": {}, + "nodes": [ + [ + "div", + { + "$": { + "html": 1, + "block": 1 + } + }, + [ + "p", + {}, + [ + "img", + { + "src": "x.png" + } + ] + ], + [ + "p", + {}, + "a", + [ + "br", + {} + ], + "b" + ] + ] + ] +} +``` + +## HTML + +```html +
+

+

a
b

+
+``` + +## Markdown + +```md +
+![](x.png) + + +a +b +
+``` diff --git a/packages/comark/SPEC/HTML/close-br-tag.md b/packages/comark/SPEC/HTML/close-br-tag.md new file mode 100644 index 000000000..a7278da50 --- /dev/null +++ b/packages/comark/SPEC/HTML/close-br-tag.md @@ -0,0 +1,43 @@ +## Input + +```md +a
b +``` + +## AST + +```json +{ + "frontmatter": {}, + "meta": {}, + "nodes": [ + [ + "p", + {}, + "a", + [ + "br", + { + "$": { + "html": 1, + "block": 0 + } + } + ], + "b" + ] + ] +} +``` + +## HTML + +```html +

a
b

+``` + +## Markdown + +```md +a
b +``` diff --git a/packages/comark/SPEC/HTML/p-closes-p.md b/packages/comark/SPEC/HTML/p-closes-p.md new file mode 100644 index 000000000..4396fca4c --- /dev/null +++ b/packages/comark/SPEC/HTML/p-closes-p.md @@ -0,0 +1,54 @@ +## Input + +```md +

+one +

+two +``` + +## AST + +```json +{ + "frontmatter": {}, + "meta": {}, + "nodes": [ + [ + "p", + { + "$": { + "html": 1, + "block": 0 + } + }, + "one" + ], + [ + "p", + { + "$": { + "html": 1, + "block": 0 + } + }, + "two" + ] + ] +} +``` + +## HTML + +```html +

one

+

two

+``` + +## Markdown + +```md +

one

+ +

two

+``` diff --git a/packages/comark/SPEC/HTML/unclosed-after-text.md b/packages/comark/SPEC/HTML/unclosed-after-text.md new file mode 100644 index 000000000..202e3c5e6 --- /dev/null +++ b/packages/comark/SPEC/HTML/unclosed-after-text.md @@ -0,0 +1,54 @@ +## Input + +```md +GET /apps/ + +Next paragraph. +``` + +## AST + +```json +{ + "frontmatter": {}, + "meta": {}, + "nodes": [ + [ + "p", + {}, + "GET /apps/", + [ + "id", + { + "$": { + "html": 1, + "block": 0 + } + } + ] + ], + [ + "p", + {}, + "Next paragraph." + ] + ] +} +``` + +## HTML + +```html +

+ GET /apps/ +

+

Next paragraph.

+``` + +## Markdown + +```md +GET /apps/ + +Next paragraph. +``` diff --git a/packages/comark/SPEC/HTML/unclosed-in-list.md b/packages/comark/SPEC/HTML/unclosed-in-list.md new file mode 100644 index 000000000..17148512e --- /dev/null +++ b/packages/comark/SPEC/HTML/unclosed-in-list.md @@ -0,0 +1,56 @@ +## Input + +```md +- **a** c +``` + +## AST + +```json +{ + "frontmatter": {}, + "meta": {}, + "nodes": [ + [ + "ul", + {}, + [ + "li", + {}, + [ + "strong", + {}, + "a" + ], + " ", + [ + "b", + { + "$": { + "html": 1, + "block": 0 + } + }, + "c" + ] + ] + ] + ] +} +``` + +## HTML + +```html +
    +
  • + a c +
  • +
+``` + +## Markdown + +```md +- **a** c +``` diff --git a/packages/comark/src/internal/parse/tree.ts b/packages/comark/src/internal/parse/tree.ts index 62640668d..711af8aca 100644 --- a/packages/comark/src/internal/parse/tree.ts +++ b/packages/comark/src/internal/parse/tree.ts @@ -36,6 +36,8 @@ type HtmlOpenFrame = { block: boolean /** Bare-inline nodes waiting to be flushed as a single

on a block frame. */ pendingInline: Node[] + /** True when markdown or text came before the open tag in its paragraph (`text `). */ + afterText?: boolean } /** Shared mutable context for a whole document walk. */ @@ -76,10 +78,24 @@ function isInlineish(node: Node): boolean { return tag === null || inlineTags.has(tag as string) } +/** Inline-level result of an HTML tag: the tags `isInlineish` knows, `img`, `br`, and custom elements. */ +function isInlineHtml(node: Node): boolean { + if (!Array.isArray(node)) return false + const tag = node[0] as string | null + if (tag === null || inlineTags.has(tag) || tag === 'img' || tag === 'br') return true + return tag.includes('-') && (node[1] as ElementNodeAttributes).$?.block === 0 +} + function flushPendingInline(frame: HtmlOpenFrame) { if (frame.pendingInline.length === 0) return const pending = frame.pendingInline frame.pendingInline = [] + const only = pending.length === 1 ? pending[0] : undefined + // A lone author-written element is a block of its own, not a paragraph. + if (Array.isArray(only) && (only[1] as ElementNodeAttributes)?.$?.html && !isInlineish(only)) { + frame.children.push(only) + return + } frame.children.push(['p', {}, ...pending] as ElementNode) } @@ -119,11 +135,11 @@ function deliverBlock(state: ProcessState, node: Node | undefined): Node | undef } /** Nest an inline/finished node into the open HTML frame during processInline. */ -function deliverInline(state: ProcessState, node: Node | undefined): Node | undefined { +function deliverInline(state: ProcessState, node: Node | undefined, inline?: boolean): Node | undefined { if (node === undefined) return undefined if (state.htmlStack.length === 0) return node const top = state.htmlStack[state.htmlStack.length - 1] - if (top.block && isInlineish(node)) { + if (top.block && (inline ?? isInlineish(node))) { top.pendingInline.push(node) } else { flushPendingInline(top) @@ -164,6 +180,55 @@ function tryCommentTag(content: string, state: ProcessState): ProcessorResult | return null } +/** Pop the frames above and including `idx`; inner ones nest into their parent, the outermost is returned. */ +function closeFramesTo(state: ProcessState, idx: number): Node { + let closed: Node | undefined + while (state.htmlStack.length > idx) { + const node = frameToNode(state.htmlStack.pop()!) + if (state.htmlStack.length > idx) deliverInline(state, node) + else closed = node + } + return closed as Node +} + +// Elements that stop a `

` from being closed by a following `

` (HTML "button scope"). +const P_SCOPE_BOUNDARY = new Set(['table', 'td', 'th', 'caption', 'button', 'template', 'object', 'marquee', 'applet']) + +/** `

` closes an open `

` (HTML implied end tag). Returns the closed paragraph, if any. */ +function closeImpliedParagraph(token: Token, state: ProcessState): Node | undefined { + if (!/^]/i.test(token.content)) return undefined + const parsed = parseHtmlInline(token.content) + if (parsed.kind !== 'open' || parsed.tag !== 'p' || parsed.selfClosing) return undefined + for (let idx = state.htmlStack.length - 1; idx >= 0; idx--) { + const tag = state.htmlStack[idx].tag + if (tag === 'p') return closeFramesTo(state, idx) + if (tag !== null && P_SCOPE_BOUNDARY.has(tag)) return undefined + } + return undefined +} + +/** True when something other than whitespace and HTML tags precedes `tokens[start]` in its paragraph. */ +function followsText(tokens: Token[], start: number): boolean { + for (let k = start - 1; k >= 0; k--) { + const { type, content } = tokens[k] + if (type === 'softbreak' || type === 'html_inline' || (type === 'text' && !content.trim())) continue + return true + } + return false +} + +/** True when a later paragraph holds ``. */ +function hasCloserAhead(tokens: Token[], from: number, tag: string): boolean { + for (let j = from; j < tokens.length; j++) { + for (const child of tokens[j].children ?? []) { + if (child.type !== 'html_inline' || child.content.charCodeAt(1) !== 0x2f /* / */) continue + const parsed = parseHtmlInline(child.content) + if (parsed.kind === 'close' && parsed.tag === tag) return true + } + } + return false +} + function processToken(tokens: Token[], i: number, state: ProcessState): ProcessorResult | undefined { return processors[tokens[i].type]?.(tokens, i, state) } @@ -180,12 +245,23 @@ function processInline(inlineTokens: Token[], state: ProcessState, nestHtml = tr let i = 0 while (i < inlineTokens.length) { + // An inline-level node stays with its paragraph; only an HTML tag is classified by name. + const fromHtml = inlineTokens[i].type === 'html_inline' + if (nestHtml && fromHtml) { + const closed = closeImpliedParagraph(inlineTokens[i], state) + if (closed !== undefined) { + const free = deliverInline(state, closed) + if (free !== undefined) pushNode(nodes, free) + } + } const result = processToken(inlineTokens, i, state) if (!result) { i += 1 continue } - const node = nestHtml ? deliverInline(state, result.node) : result.node + const node = nestHtml + ? deliverInline(state, result.node, !fromHtml || (result.node !== undefined && isInlineHtml(result.node))) + : result.node if (node !== undefined) pushNode(nodes, node) i = result.nextIndex } @@ -235,7 +311,7 @@ function processChildren( if (result) { // Same nesting rule as processInline: free nodes join the open HTML // frame when nestHtml is on (e.g. block children under bare HTML). - const node = nestHtml ? deliverInline(state, result.node) : result.node + const node = nestHtml && !result.siblingOfHtmlStack ? deliverInline(state, result.node) : result.node if (node !== undefined) pushNode(children, node) i = result.nextIndex } else { @@ -484,6 +560,12 @@ const processors: Record = { const depthBefore = state.htmlStack.length const nestHtml = state.insideMarkdownContainer === 0 const { children, nextIndex } = processChildren(tokens, start, 'paragraph_close', state, nestHtml) + // `text ` with no `` anywhere after is prose, not a container: it ends with its paragraph. + const opened = state.htmlStack[depthBefore] + if (nestHtml && opened?.afterText && opened.tag !== null && !hasCloserAhead(tokens, nextIndex, opened.tag)) { + const free = deliverInline(state, closeFramesTo(state, depthBefore), true) + if (free !== undefined) pushNode(children, free) + } const asParagraph = (): ProcessorResult => children.length === 0 ? { nextIndex, node: undefined } @@ -597,6 +679,8 @@ const processors: Record = { } if (parsed.kind === 'close') { + // Browsers read a stray `
` as `
`. + if (parsed.tag === 'br') return { nextIndex: start + 1, node: ['br', makeHtmlAttrs({}, false)] } // Find the matching open frame (case-insensitive), auto-closing any // nested frames above it the way a real HTML parser would. const closeTag = parsed.tag.toLowerCase() @@ -612,17 +696,7 @@ const processors: Record = { return { nextIndex: start + 1, node } } - let closed: Node | undefined - while (state.htmlStack.length > idx) { - const node = frameToNode(state.htmlStack.pop()!) - // Nested auto-closed frames nest into the next outer open frame. - if (state.htmlStack.length > idx) { - deliverInline(state, node) - } else { - closed = node - } - } - return { nextIndex: start + 1, node: closed } + return { nextIndex: start + 1, node: closeFramesTo(state, idx) } } if (parsed.selfClosing) { @@ -638,6 +712,7 @@ const processors: Record = { children: [], block: false, pendingInline: [], + afterText: followsText(tokens, start), }) return { nextIndex: start + 1, node: undefined } }, diff --git a/packages/comark/test/html-tags.test.ts b/packages/comark/test/html-tags.test.ts index 083c28c0c..52701ff96 100644 --- a/packages/comark/test/html-tags.test.ts +++ b/packages/comark/test/html-tags.test.ts @@ -48,3 +48,154 @@ describe('HTML comments', () => { ]) }) }) + +describe('implied end tags', () => { + it('reads a stray
as a line break', async () => { + const result = await parseMarkdown('a
b') + + expect(result.nodes).toEqual([['p', {}, 'a', ['br', { $: inline }], 'b']]) + }) + + it('closes an open

when another

starts', async () => { + const result = await parseMarkdown('

\none\n

\ntwo') + + expect(result.nodes).toEqual([ + ['p', { $: inline }, 'one'], + ['p', { $: inline }, 'two'], + ]) + }) + + it('closes an open

across blocks and inline elements', async () => { + const result = await parseMarkdown('

\n\n

a x\n\n

b\n\n

') + + expect(result.nodes).toEqual([ + ['div', { $: block }, ['p', { $: block }, 'a ', ['b', { $: block }, 'x']], ['p', { $: block }, 'b']], + ]) + }) + + it('keeps a

nested in a table cell inside the outer

', async () => { + const result = await parseMarkdown('

a

b

') + + expect(result.nodes).toEqual([ + [ + 'p', + { $: block }, + 'a', + ['table', { $: inline }, ['tr', { $: inline }, ['td', { $: inline }, ['p', { $: inline }, 'b']]]], + ], + ]) + }) + + it('renders consecutive

tags as siblings', async () => { + const result = await parseMarkdown('

one\n

two') + + expect(await renderHtmlForTest(result)).toBe('

one

\n

two

') + }) +}) + +describe('inline content inside block HTML', () => { + const md = '
\n\nhello ![](x.png) a \nb :badge[x] y
end\n\n
' + + it('keeps images, line breaks, components and custom elements in their paragraph', async () => { + const result = await parseMarkdown(md, { autoUnwrap: false }) + + expect(result.nodes).toEqual([ + [ + 'div', + { $: block }, + [ + 'p', + {}, + 'hello ', + ['img', { src: 'x.png' }], + ' a', + ['br', {}], + 'b ', + ['badge', {}, 'x'], + ' ', + ['my-el', { $: inline }, 'y'], + ' ', + ['br', { $: inline }], + ' end', + ], + ], + ]) + }) + + it('wraps a markdown image on its own in a paragraph', async () => { + const result = await parseMarkdown('
\n\n![](x.png)\n\n![](y.png)\n\n
', { autoUnwrap: false }) + + expect(result.nodes).toEqual([ + ['div', { $: block }, ['p', {}, ['img', { src: 'x.png' }]], ['p', {}, ['img', { src: 'y.png' }]]], + ]) + }) + + it('leaves an author-written element on its own as a block', async () => { + const result = await parseMarkdown('
\n\ny\n\n\n\n
', { autoUnwrap: false }) + + expect(result.nodes).toEqual([ + ['div', { $: block }, ['my-el', { $: inline }, 'y'], ['img', { $: inline, src: 'a' }]], + ]) + }) + + it('keeps the order of a closed element and the one that follows it in a list item', async () => { + const result = await parseMarkdown('-
a
b\n- c') + + expect(result.nodes).toEqual([ + ['ul', {}, ['li', {}, ['div', { $: inline }, 'a'], ['div', { $: inline }, 'b']], ['li', {}, 'c']], + ]) + }) +}) + +describe('an open tag after text', () => { + it('ends with its paragraph when nothing closes it', async () => { + const result = await parseMarkdown('GET /apps/\n\nNext paragraph.') + + expect(result.nodes).toEqual([ + ['p', {}, 'GET /apps/', ['id', { $: inline }]], + ['p', {}, 'Next paragraph.'], + ]) + }) + + it('ends with its paragraph inside a block element', async () => { + const result = await parseMarkdown('
\n\nHello x\n\nmore\n\n
', { autoUnwrap: false }) + + expect(result.nodes).toEqual([ + ['div', { $: block }, ['p', {}, 'Hello ', ['b', { $: inline }, 'x']], ['p', {}, 'more']], + ]) + }) + + it('keeps the text before a tag that a later paragraph closes', async () => { + const result = await parseMarkdown('text \n\n**a**\n\n') + + expect(result.nodes).toEqual(['text ', ['x-card', { $: block }, ['strong', {}, 'a']]]) + }) + + it('still lets a tag at the start of its paragraph collect the blocks after it', async () => { + const result = await parseMarkdown('\n\nNext paragraph.') + + expect(result.nodes).toEqual([['ai-thinking', { $: block }, 'Next paragraph.']]) + }) + + it('keeps markdown before the tag in its list item', async () => { + const result = await parseMarkdown('- **a** c\n- d') + + expect(result.nodes).toEqual([ + ['ul', {}, ['li', {}, ['strong', {}, 'a'], ' ', ['b', { $: inline }, 'c']], ['li', {}, 'd']], + ]) + }) + + it('settles on the paired tree once the closer streams in', async () => { + const source = 'text \n\n**a**\n\n' + const parse = createMarkdownParser() + for (let end = 1; end < source.length; end++) { + await expect(parse(source.slice(0, end))).resolves.toBeDefined() + } + + expect((await parse(source)).nodes).toEqual((await parseMarkdown(source)).nodes) + expect((await parse('text \n\n**a**')).nodes).toEqual([ + ['p', {}, 'text ', ['x-card', { $: inline }]], + ['p', {}, ['strong', {}, 'a']], + ]) + }) +}) From 22db6a1b5f59bc8826b32708b57b2358648bcedf Mon Sep 17 00:00:00 2001 From: onmax Date: Sun, 1 Nov 2026 12:20:28 +0100 Subject: [PATCH 03/15] fix(html): do not wrap a standalone comment in a paragraph An indented standalone comment kept the block's indentation as a text node, so the comment was wrapped in a paragraph instead of being a comment node. --- .../comark/SPEC/HTML/comments-standalone.md | 43 +++++++++++++++++++ packages/comark/src/plugins/html.ts | 3 +- packages/comark/test/html-tags.test.ts | 17 ++++++++ 3 files changed, 62 insertions(+), 1 deletion(-) create mode 100644 packages/comark/SPEC/HTML/comments-standalone.md diff --git a/packages/comark/SPEC/HTML/comments-standalone.md b/packages/comark/SPEC/HTML/comments-standalone.md new file mode 100644 index 000000000..19545d786 --- /dev/null +++ b/packages/comark/SPEC/HTML/comments-standalone.md @@ -0,0 +1,43 @@ +## Input + +```md + + + +``` + +## AST + +```json +{ + "frontmatter": {}, + "meta": {}, + "nodes": [ + [ + null, + {}, + " a " + ], + [ + null, + {}, + "- b -" + ] + ] +} +``` + +## HTML + +```html + + +``` + +## Markdown + +```md + + + +``` diff --git a/packages/comark/src/plugins/html.ts b/packages/comark/src/plugins/html.ts index c2cb9721c..b4c1e1646 100644 --- a/packages/comark/src/plugins/html.ts +++ b/packages/comark/src/plugins/html.ts @@ -109,7 +109,8 @@ function markdownItPost(state: ComarkParseTokensState) { // Expand raw CommonMark html_block into html_inline + text, wrapped as a // paragraph so the tree walk pairs open/close the same way as html_inline. // Body text stays literal (no markdown re-parse). - const children = htmlToTokens(token.content || '') + // The block's own indentation is not content (` `). + const children = htmlToTokens((token.content || '').replace(/^[ \t]+/, '')) const open = new Token('paragraph_open', 'p', 1) const inline = new Token('inline', '', 0) const close = new Token('paragraph_close', 'p', -1) diff --git a/packages/comark/test/html-tags.test.ts b/packages/comark/test/html-tags.test.ts index 52701ff96..ef964dc74 100644 --- a/packages/comark/test/html-tags.test.ts +++ b/packages/comark/test/html-tags.test.ts @@ -199,3 +199,20 @@ describe('an open tag after text', () => { ]) }) }) + +describe('standalone comments', () => { + it('does not wrap an indented comment in a paragraph', async () => { + const result = await parseMarkdown(' \n\n ') + + expect(result.nodes).toEqual([ + [null, {}, ' a '], + [null, {}, '- b -'], + ]) + }) + + it('keeps the comment of a closed block inside its element', async () => { + const result = await parseMarkdown('
\n \n text\n
') + + expect(result.nodes).toEqual([['div', { $: block }, [null, {}, ' x '], 'text']]) + }) +}) From dc5aa4eb28fab37a179fb589ff6c0cf03ff5a52c Mon Sep 17 00:00:00 2001 From: onmax Date: Sun, 1 Nov 2026 12:20:31 +0100 Subject: [PATCH 04/15] fix(html): decode character references and parse tags inside

`
` kept its tags and entities as literal text. In `
` tags are now HTML and character references are decoded, as in `\n\n\n\n'
+    )
+
+    expect(result.nodes).toEqual([
+      ['textarea', { $: block }, 'a < b'],
+      ['script', { $: block }, 'a < b'],
+      ['style', { $: block }, 'a < b'],
+    ])
+  })
+
+  it('writes the decoded text of 
 back as references', async () => {
+    const source = '
a <b>c</b> &amp; <!-- x -->
' + const first = await parseMarkdown(source) + const rendered = await renderMarkdown(first) + + expect(rendered).toBe('
a <b>c</b> &amp; <!-- x -->
') + expect((await parseMarkdown(rendered)).nodes).toEqual(first.nodes) + }) +}) From 3e80f25537b70d5b608346f3e50a3f347206c161 Mon Sep 17 00:00:00 2001 From: onmax Date: Sun, 1 Nov 2026 12:20:35 +0100 Subject: [PATCH 05/15] fix(parse): keep an author-written

when autoUnwrap runs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit autoUnwrap treated the `

` in `

…

` as a wrapper and moved its attributes onto the parent. Only the paragraph markdown adds is unwrapped now. --- .../SPEC/HTML/author-paragraph-auto-unwrap.md | 52 +++++++++++++++++++ packages/comark/src/internal/parse/utils.ts | 18 ++++--- packages/comark/src/types.ts | 1 + packages/comark/test/html-tags.test.ts | 18 +++++++ 4 files changed, 83 insertions(+), 6 deletions(-) create mode 100644 packages/comark/SPEC/HTML/author-paragraph-auto-unwrap.md diff --git a/packages/comark/SPEC/HTML/author-paragraph-auto-unwrap.md b/packages/comark/SPEC/HTML/author-paragraph-auto-unwrap.md new file mode 100644 index 000000000..b4a52d366 --- /dev/null +++ b/packages/comark/SPEC/HTML/author-paragraph-auto-unwrap.md @@ -0,0 +1,52 @@ +## Input + +```md +

Check

+``` + +## AST + +```json +{ + "frontmatter": {}, + "meta": {}, + "nodes": [ + [ + "section", + { + "$": { + "html": 1, + "block": 1 + } + }, + [ + "p", + { + "$": { + "html": 1, + "block": 0 + }, + "class": "c" + }, + "Check" + ] + ] + ] +} +``` + +## HTML + +```html +
+

Check

+
+``` + +## Markdown + +```md +
+

Check

+
+``` diff --git a/packages/comark/src/internal/parse/utils.ts b/packages/comark/src/internal/parse/utils.ts index 7ede1e272..3c6949a26 100644 --- a/packages/comark/src/internal/parse/utils.ts +++ b/packages/comark/src/internal/parse/utils.ts @@ -63,7 +63,13 @@ function applyAutoUnwrapAt(node: Node, depth: number): Node { return node } - if (significantCount === 1 && Array.isArray(significant) && significant[0] === 'p') { + // An author-written `

` (`$.html`) is content, not a wrapper. + if ( + significantCount === 1 && + Array.isArray(significant) && + significant[0] === 'p' && + !(significant[1] as ElementNodeAttributes | undefined)?.$?.html + ) { // Lift the paragraph's attrs onto the parent so trailing `{attr}` survives the unwrap. // Parent attrs take precedence so explicit component props aren't overridden. const paragraphAttrs = significant[1] as Record | undefined @@ -182,17 +188,17 @@ export function parseHtmlInline(content: string): ParsedHtmlTag { return { kind: 'other', content: trimmed } } +const ATTRIBUTE_RE = new RegExp( + `([:\\w.-]+)(?:\\s*=\\s*(?:"([^"]*)"|'([^']*)'|(${braced_value})|([^\\s"'=<>\`]+)))?`, + 'g' +) + /** * Parse HTML attribute string into a key/value map. * Supports `key="value"`, `key='value'`, bare `key=value`, and boolean attributes. * Names are lowercased, as HTML attribute names are case-insensitive. */ export function parseHtmlAttributes(attrsStr: string): Record { -const ATTRIBUTE_RE = new RegExp( - `([:\\w.-]+)(?:\\s*=\\s*(?:"([^"]*)"|'([^']*)'|(${braced_value})|([^\\s"'=<>\`]+)))?`, - 'g' -) - const attrs: Record = {} if (!attrsStr || !attrsStr.trim()) return attrs diff --git a/packages/comark/src/types.ts b/packages/comark/src/types.ts index 36c51e3ae..9b805aa85 100644 --- a/packages/comark/src/types.ts +++ b/packages/comark/src/types.ts @@ -449,6 +449,7 @@ export interface ParserOptions[ * When enabled, if a container component (alert, card, callout, note, warning, tip, info) * has only a single paragraph child, the paragraph wrapper is removed and its children * become direct children of the container. This creates cleaner HTML output. + * A `

` written as HTML is content and is never unwrapped. * * @default true * @example diff --git a/packages/comark/test/html-tags.test.ts b/packages/comark/test/html-tags.test.ts index 2a9d2afa5..314ab7a4d 100644 --- a/packages/comark/test/html-tags.test.ts +++ b/packages/comark/test/html-tags.test.ts @@ -296,3 +296,21 @@ describe('character references and

', () => {
     expect((await parseMarkdown(rendered)).nodes).toEqual(first.nodes)
   })
 })
+
+describe('autoUnwrap', () => {
+  it('keeps a 

the author wrote, with its attributes', async () => { + const source = '

Check

' + + for (const autoUnwrap of [true, false]) { + const result = await parseMarkdown(source, { autoUnwrap }) + + expect(result.nodes).toEqual([['section', { $: block }, ['p', { $: inline, class: 'c' }, 'Check']]]) + } + }) + + it('still unwraps the paragraph markdown wraps around a single line', async () => { + const result = await parseMarkdown('
\n\nCheck\n\n
') + + expect(result.nodes).toEqual([['section', { $: block }, 'Check']]) + }) +}) From 784bd90283537e5bf6389fe06b6faea90a5b345c Mon Sep 17 00:00:00 2001 From: onmax Date: Sun, 1 Nov 2026 12:20:36 +0100 Subject: [PATCH 06/15] docs(html): document the tag grammar, implied end tags and
 contents

Record the behavior changes of the previous commits in the HTML contract and the html plugin page.
---
 packages/comark/SPEC/HTML/README.md | 29 ++++++++++++++++++++++++++++-
 1 file changed, 28 insertions(+), 1 deletion(-)

diff --git a/packages/comark/SPEC/HTML/README.md b/packages/comark/SPEC/HTML/README.md
index 64bc938f7..33f0e406d 100644
--- a/packages/comark/SPEC/HTML/README.md
+++ b/packages/comark/SPEC/HTML/README.md
@@ -15,6 +15,8 @@ HTML tags become element nodes with `$: { html: 1, block: 0 | 1 }`. A node witho
 
 Void tags (`img`, `br`, …) are always `block: 0`.
 
+A non-void open tag that follows text in its paragraph (`GET /apps/`, `- **a** c`) is prose, not a container. It ends with its paragraph, `block: 0`, unless a later paragraph holds its close tag. A tag that starts its paragraph keeps collecting the blocks after it.
+
 Consecutive same-paragraph tags stay inline even if indented on following lines:
 
 ```md
@@ -41,7 +43,13 @@ Blank-line bodies parse as markdown in both modes. CommonMark ends the HTML bloc
 
 `
', + '', + ]) { + const { first, markdown, again } = await roundTrip(source) + + expect(markdown).toBe(source) + expect(again.nodes).toEqual(first.nodes) + } + }) + + it('keeps every character of a comment inside
', async () => {
+    for (const source of ['
 a < b
', '
ac
']) { + const { first, markdown, again, html: output } = await roundTrip(source) + + expect(markdown).toBe(source) + expect(output).toBe(source) + expect(again.nodes).toEqual(first.nodes) + } + }) + + it('keeps a ') + + expect(first.nodes).toEqual([['textarea', { $: block }, 'a & b']]) + expect(markdown).toBe('') + expect(again.nodes).toEqual(first.nodes) + }) + + it('does not read markdown in a
 that follows text', async () => {
+    const source = 'before 
**x** < y
after' + const { first, markdown, again, html: output } = await roundTrip(source) + + expect(first.nodes).toEqual([['p', {}, 'before ', ['pre', { $: inline }, '**x** < y'], ' after']]) + expect(markdown).toBe(source) + expect(output).toBe('

before

**x** < y
after

') + expect(again.nodes).toEqual(first.nodes) + }) + + it('keeps a
 after text unindented in the paragraph', async () => {
+    const { html: output } = await roundTrip('before 
\nline\n  indented
after') + + expect(output).toBe('

before

\nline\n  indented
after

') + }) +}) + describe('linear time', () => { async function median(source: (n: number) => string, n: number) { const runs: number[] = [] From 950533d1df07ffcd86b578ffbdbc4e808dbb68e2 Mon Sep 17 00:00:00 2001 From: onmax Date: Sun, 1 Nov 2026 13:09:37 +0100 Subject: [PATCH 09/15] fix(html): index closing tags once, match raw tags exactly, keep literal text and
 markup as written

Raw-element closers and the prose check before an open tag are looked up in one pass per input instead of rescanned per tag. Closers in html_block expansions keep their container level, so html({ markdown: false }) pairs tags like the default. Raw tag names are read whole (script-x is not script). Literal text stays encoded under nested inline tags, and 
 gets no added separators between sibling tags. Wall-clock ratio tests are replaced by large-input correctness tests.
---
 packages/comark/src/internal/parse/tree.ts    |  15 ++-
 .../src/internal/stringify/handlers/html.ts   |  15 ++-
 .../comark/src/internal/stringify/state.ts    |   7 +-
 packages/comark/src/plugins/html.ts           |  75 ++++++++---
 packages/comark/test/html-tags.test.ts        | 124 +++++++++++++++---
 5 files changed, 193 insertions(+), 43 deletions(-)

diff --git a/packages/comark/src/internal/parse/tree.ts b/packages/comark/src/internal/parse/tree.ts
index 1d80e8f62..a72f531cf 100644
--- a/packages/comark/src/internal/parse/tree.ts
+++ b/packages/comark/src/internal/parse/tree.ts
@@ -209,14 +209,19 @@ function closeImpliedParagraph(token: Token, state: ProcessState): Node | undefi
   return undefined
 }
 
+const firstProse = new WeakMap()
+
 /** True when something other than whitespace and HTML tags precedes `tokens[start]` in its paragraph. */
 function followsText(tokens: Token[], start: number): boolean {
-  for (let k = start - 1; k >= 0; k--) {
-    const { type, content } = tokens[k]
-    if (type === 'softbreak' || type === 'html_inline' || (type === 'text' && !content.trim())) continue
-    return true
+  let first = firstProse.get(tokens)
+  if (first === undefined) {
+    first = tokens.findIndex(
+      ({ type, content }) => !(type === 'softbreak' || type === 'html_inline' || (type === 'text' && !content.trim()))
+    )
+    if (first < 0) first = tokens.length
+    firstProse.set(tokens, first)
   }
-  return false
+  return first < start
 }
 
 /** Where each `` sits, so "is it closed later?" is a lookup and not a scan of the rest of the document. */
diff --git a/packages/comark/src/internal/stringify/handlers/html.ts b/packages/comark/src/internal/stringify/handlers/html.ts
index 5011fe2a5..660c4f4f4 100644
--- a/packages/comark/src/internal/stringify/handlers/html.ts
+++ b/packages/comark/src/internal/stringify/handlers/html.ts
@@ -48,7 +48,9 @@ export async function html(node: ElementNode, state: State, parent?: ElementNode
   if (!oneLiner && inlineTags.has(String(tag)) && hasOnlyTextChildren) {
     oneLiner = true
   }
-  if (tag === 'pre' || tag === 'textarea' || tag === 'script' || tag === 'style') {
+  // Every character in an authored `
` is content, nested tags included.
+  const authoredRaw = state.context.rawText === true && $.html === 1
+  if (tag === 'pre' || tag === 'textarea' || tag === 'script' || tag === 'style' || authoredRaw) {
     oneLiner = true
   }
   // Re-indenting a paragraph would put spaces into the `
` written inside it.
@@ -75,6 +77,8 @@ export async function html(node: ElementNode, state: State, parent?: ElementNode
   const revert = state.applyContext({
     inline: oneLiner,
     rawText: state.context.rawText === true || tag === 'pre' || tag === 'textarea',
+    // Text in raw HTML was kept as written, so it is encoded again at any depth of inline tags.
+    literal: $.html === 1 && ($.block === 1 || state.context.literal === true),
   })
 
   const childrenContent: string[] = []
@@ -83,7 +87,14 @@ export async function html(node: ElementNode, state: State, parent?: ElementNode
   }
 
   // A blank line inside a raw-HTML element would terminate it on reparse
-  const childSeparator = state.context.html ? state.context.blockSeparator : oneLiner ? '' : '\n'
+  const childSeparator =
+    state.context.rawText && $.html === 1
+      ? ''
+      : state.context.html
+        ? state.context.blockSeparator
+        : oneLiner
+          ? ''
+          : '\n'
 
   let content = ''
   let isPrevBlock = true
diff --git a/packages/comark/src/internal/stringify/state.ts b/packages/comark/src/internal/stringify/state.ts
index e3ca17131..16f17bed7 100644
--- a/packages/comark/src/internal/stringify/state.ts
+++ b/packages/comark/src/internal/stringify/state.ts
@@ -54,7 +54,7 @@ export async function one(node: Node, state: State, parent?: ElementNode, atLine
     // syntax inside it must not be escaped (inline HTML, `$.block === 0`, has
     // its markdown parsed into children and is escaped normally). Decoded text
     // that would read as a reference or a tag is encoded again.
-    if (parent?.[1].$?.html === 1 && parent[1].$?.block === 1) {
+    if (parent?.[1].$?.html === 1 && (parent[1].$?.block === 1 || state.context.literal === true)) {
       return node.replace(/&(?=#\d+;|#x[\da-f]+;|[a-z][a-z\d]*;)/gi, '&').replace(/<(?=[a-z/!?])/gi, '<')
     }
     return escapeTextNode(node, atLineStart)
@@ -88,6 +88,10 @@ export async function one(node: Node, state: State, parent?: ElementNode, atLine
     }
   }
 
+  // Text under raw HTML stays literal through its inline elements, but not through markdown nodes.
+  const revertLiteral =
+    state.context.literal === true && node[1].$?.html !== 1 ? state.applyContext({ literal: false }) : undefined
+
   try {
     const userHandler = findHandler(state.context, node)
     if (userHandler) {
@@ -110,6 +114,7 @@ export async function one(node: Node, state: State, parent?: ElementNode, atLine
       : await state.handlers.html(node, state, parent)
   } finally {
     state.renderData = prevRenderData
+    if (revertLiteral) state.applyContext(revertLiteral)
   }
 }
 
diff --git a/packages/comark/src/plugins/html.ts b/packages/comark/src/plugins/html.ts
index 2dda10227..9dc780a55 100644
--- a/packages/comark/src/plugins/html.ts
+++ b/packages/comark/src/plugins/html.ts
@@ -76,7 +76,7 @@ function htmlInline(state: StateInline, silent: boolean): boolean {
   const match = HTML_TAG_RE.exec(state.src.slice(pos, state.posMax))
   if (!match) return false
   // `before 
**x**
`: a raw element's body is not markdown, inline as well as in a block. - const rawEnd = rawElementEnd(state.src.slice(0, state.posMax), pos, match[0]) + const rawEnd = rawElementEnd(state, pos, match[0]) if (rawEnd > 0) { if (!silent) { state.pushPending() @@ -131,6 +131,9 @@ function markdownItPost(state: ComarkParseTokensState) { const open = new Token('paragraph_open', 'p', 1) const inline = new Token('inline', '', 0) const close = new Token('paragraph_close', 'p', -1) + // The paragraph keeps the block's container depth, so closing tags pair within it. + open.level = close.level = token.level + inline.level = token.level + 1 inline.children = children inline.content = token.content || '' if (token.map) { @@ -146,6 +149,11 @@ function isLetter(code: number): boolean { return (code >= 0x41 && code <= 0x5a) || (code >= 0x61 && code <= 0x7a) } +/** The rest of a tag name: letters, digits and `-`, so `script-x` is not `script`. */ +function isNameChar(code: number): boolean { + return isLetter(code) || (code >= 0x30 && code <= 0x39) || code === 0x2d +} + /** CommonMark type-1 raw blocks whose body is kept verbatim. `` does not match. */ const RAW_HTML_TAGS = new Set(['style', 'pre', 'script', 'textarea']) @@ -162,8 +170,8 @@ function readTag(src: string, pos: number): { name: string; close: boolean; afte const close = src.charCodeAt(i) === 0x2f /* / */ if (close) i += 1 const start = i - while (i < src.length && isLetter(src.charCodeAt(i))) i += 1 - if (i === start) return null + if (!isLetter(src.charCodeAt(i))) return null + while (i < src.length && isNameChar(src.charCodeAt(i))) i += 1 return { name: src.slice(start, i).toLowerCase(), close, after: i } } @@ -230,20 +238,50 @@ function rawCloserEnd(src: string, pos: number, name: string): number { return src.charCodeAt(j) === 0x3e /* > */ ? j + 1 : -1 } -/** The first `` at or after `from`. */ -function findRawCloser(str: string, from: number, name: string): { start: number; end: number } | null { - for (let i = str.indexOf('<', from); i >= 0; i = str.indexOf('<', i + 1)) { - const end = rawCloserEnd(str, i, name) - if (end >= 0) return { start: i, end } +interface Closer { + start: number + end: number +} + +/** Where each raw element's `` sits in `src`, found in one pass on first use. */ +function createCloserFinder(src: string): (from: number, name: string, limit?: number) => Closer | null { + let index: Map | undefined + return (from, name, limit = src.length) => { + if (!index) { + index = new Map() + for (let i = src.indexOf('<'); i >= 0; i = src.indexOf('<', i + 1)) { + const tag = readTag(src, i) + if (!tag?.close || !RAW_HTML_TAGS.has(tag.name)) continue + const end = rawCloserEnd(src, i, tag.name) + if (end < 0) continue + const list = index.get(tag.name) + if (list) list.push({ start: i, end }) + else index.set(tag.name, [{ start: i, end }]) + } + } + const list = index.get(name) + if (!list) return null + let lo = 0 + let hi = list.length + while (lo < hi) { + const mid = (lo + hi) >> 1 + if (list[mid].start < from) lo = mid + 1 + else hi = mid + } + return lo < list.length && list[lo].end <= limit ? list[lo] : null } - return null } -/** End of the raw element opened by `openTag` at `pos` when its closer is in `src`, else -1. */ -function rawElementEnd(src: string, pos: number, openTag: string): number { +const inlineClosers = new WeakMap Closer | null>() + +/** End of the raw element opened by `openTag` at `pos` when its closer is before `limit`, else -1. */ +function rawElementEnd(state: StateInline, pos: number, openTag: string): number { const raw = readTag(openTag, 0) - if (!raw || raw.close || !RAW_HTML_TAGS.has(raw.name) || openTag.endsWith('/>')) return -1 - return findRawCloser(src, pos + openTag.length, raw.name)?.end ?? -1 + if (!raw || raw.close || !RAW_HTML_TAGS.has(raw.name) || !isTagBoundary(openTag, raw.after) || openTag.endsWith('/>')) + return -1 + let find = inlineClosers.get(state) + if (!find) inlineClosers.set(state, (find = createCloserFinder(state.src))) + return find(pos + openTag.length, raw.name, state.posMax)?.end ?? -1 } /** @@ -273,6 +311,7 @@ function scan(tokens: Token[], str: string, verbatim: boolean) { let pos = 0 let textStart = 0 const closers = verbatim ? lastClosers(str) : undefined + const findCloser = createCloserFinder(str) while (pos < len) { // Candidate HTML tag starts with '<' followed by letter, '/', '!', or '?' @@ -285,10 +324,16 @@ function scan(tokens: Token[], str: string, verbatim: boolean) { const raw = readTag(match[0], 0) // style / pre / script / textarea: body is source until the closer, inner tags included // (except in `
`, whose inner tags are HTML).
-          if (raw && !raw.close && RAW_HTML_TAGS.has(raw.name) && match[0].charCodeAt(match[0].length - 2) !== 0x2f) {
+          if (
+            raw &&
+            !raw.close &&
+            RAW_HTML_TAGS.has(raw.name) &&
+            isTagBoundary(match[0], raw.after) &&
+            match[0].charCodeAt(match[0].length - 2) !== 0x2f
+          ) {
             pushHtml(tokens, match[0])
             const from = pos + match[0].length
-            const closer = findRawCloser(str, from, raw.name)
+            const closer = findCloser(from, raw.name)
             if (!closer) {
               pushRawBody(tokens, str.slice(from), raw.name)
               return
diff --git a/packages/comark/test/html-tags.test.ts b/packages/comark/test/html-tags.test.ts
index de8669d8b..40bafbb2a 100644
--- a/packages/comark/test/html-tags.test.ts
+++ b/packages/comark/test/html-tags.test.ts
@@ -388,31 +388,115 @@ describe('literal text round trip', () => {
   })
 })
 
-describe('linear time', () => {
-  async function median(source: (n: number) => string, n: number) {
-    const runs: number[] = []
-    for (let i = 0; i < 3; i++) {
-      const start = performance.now()
-      await parseMarkdown(source(n), { autoClose: false })
-      runs.push(performance.now() - start)
+describe('large inputs', () => {
+  // Correctness at a size where a quadratic scan would time out the suite; scaling is measured by hand.
+  it('parses many unpaired raw openers in one paragraph', async () => {
+    const result = await parseMarkdown('a 
+
+
+
+
+```
+
+## AST
+
+```json
+{
+  "frontmatter": {},
+  "meta": {},
+  "nodes": [
+    [
+      "textarea",
+      {
+        "$": {
+          "html": 1,
+          "block": 1
+        }
+      },
+      "a < b"
+    ],
+    [
+      "script",
+      {
+        "$": {
+          "html": 1,
+          "block": 1
+        }
+      },
+      "a < b"
+    ],
+    [
+      "style",
+      {
+        "$": {
+          "html": 1,
+          "block": 1
+        }
+      },
+      "a < b"
+    ]
+  ]
+}
+```
+
+## HTML
+
+```html
+
+
+
+```
+
+## Markdown
+
+```md
+
+
+
+
+
+```
diff --git a/packages/comark/SPEC/HTML/raw-textarea-references.md b/packages/comark/SPEC/HTML/raw-textarea-references.md
new file mode 100644
index 000000000..da8fd3c3d
--- /dev/null
+++ b/packages/comark/SPEC/HTML/raw-textarea-references.md
@@ -0,0 +1,38 @@
+## Input
+
+```md
+
+```
+
+## AST
+
+```json
+{
+  "frontmatter": {},
+  "meta": {},
+  "nodes": [
+    [
+      "textarea",
+      {
+        "$": {
+          "html": 1,
+          "block": 1
+        }
+      },
+      "a & b"
+    ]
+  ]
+}
+```
+
+## HTML
+
+```html
+
+```
+
+## Markdown
+
+```md
+
+```
diff --git a/packages/comark/SPEC/HTML/unwrap-author-paragraph.md b/packages/comark/SPEC/HTML/unwrap-author-paragraph.md
new file mode 100644
index 000000000..b4a52d366
--- /dev/null
+++ b/packages/comark/SPEC/HTML/unwrap-author-paragraph.md
@@ -0,0 +1,52 @@
+## Input
+
+```md
+

Check

+``` + +## AST + +```json +{ + "frontmatter": {}, + "meta": {}, + "nodes": [ + [ + "section", + { + "$": { + "html": 1, + "block": 1 + } + }, + [ + "p", + { + "$": { + "html": 1, + "block": 0 + }, + "class": "c" + }, + "Check" + ] + ] + ] +} +``` + +## HTML + +```html +
+

Check

+
+``` + +## Markdown + +```md +
+

Check

+
+``` diff --git a/packages/comark/SPEC/HTML/unwrap-markdown-paragraph.md b/packages/comark/SPEC/HTML/unwrap-markdown-paragraph.md new file mode 100644 index 000000000..f8a3c0287 --- /dev/null +++ b/packages/comark/SPEC/HTML/unwrap-markdown-paragraph.md @@ -0,0 +1,46 @@ +## Input + +```md +
+ +Check + +
+``` + +## AST + +```json +{ + "frontmatter": {}, + "meta": {}, + "nodes": [ + [ + "section", + { + "$": { + "html": 1, + "block": 1 + } + }, + "Check" + ] + ] +} +``` + +## HTML + +```html +
+ Check +
+``` + +## Markdown + +```md +
+Check +
+``` diff --git a/packages/comark/test/html-tags.test.ts b/packages/comark/test/html-tags.test.ts index f57115956..ab76cf30a 100644 --- a/packages/comark/test/html-tags.test.ts +++ b/packages/comark/test/html-tags.test.ts @@ -8,254 +8,80 @@ const block = { html: 1, block: 1 } const inline = { html: 1, block: 0 } const literal = { plugins: [html({ markdown: false })] } -it.each([ - { - name: 'attribute-jsx-expression', - input: '
\n\nx', - nodes: [['div', { $: block, style: '{{a: 1}}' }, 'x']], - html: '
\n x\n
', - markdown: '
\nx\n
', - }, - { - name: 'author-paragraph-auto-unwrap', - input: '

Check

', - nodes: [['section', { $: block }, ['p', { $: inline, class: 'c' }, 'Check']]], - html: '
\n

Check

\n
', - markdown: '
\n

Check

\n
', - }, - { - name: 'block-paragraph-inline-content', - input: '
\n\n![](x.png)\n\na\\\nb\n\n
', - nodes: [['div', { $: block }, ['p', {}, ['img', { src: 'x.png' }]], ['p', {}, 'a', ['br', {}], 'b']]], - html: '
\n

\n

a
b

\n
', - markdown: '
\n![](x.png)\n\n\na \nb\n
', - autoUnwrap: false, - }, - { - name: 'close-br-tag', - input: 'a
b', - nodes: [['p', {}, 'a', ['br', { $: inline }], 'b']], - html: '

a
b

', - markdown: 'a
b', - }, - { - name: 'comments-standalone', - input: '\n\n', - nodes: [ - [null, {}, ' a '], - [null, {}, '- b -'], - ], - html: '\n', - markdown: '\n\n', - }, - { - name: 'duplicate-attributes', - input: 'first', - nodes: [['img', { $: block, alt: 'first' }]], - html: 'first', - markdown: 'first', - }, - { - name: 'p-closes-p', - input: '

\none\n

\ntwo', - nodes: [ - ['p', { $: inline }, 'one'], - ['p', { $: inline }, 'two'], - ], - html: '

one

\n

two

', - markdown: '

one

\n\n

two

', - }, - { - name: 'pre-code', - input: '
x < y
', - nodes: [['pre', { $: block }, ['code', { $: inline }, 'x < y']]], - html: '
x < y
', - markdown: '
x < y
', - }, - { - name: 'unclosed-after-text', - input: 'GET /apps/\n\nNext paragraph.', - nodes: [ - ['p', {}, 'GET /apps/', ['id', { $: inline }]], - ['p', {}, 'Next paragraph.'], - ], - html: '

\n GET /apps/\n

\n

Next paragraph.

', - markdown: 'GET /apps/\n\nNext paragraph.', - }, - { - name: 'unclosed-in-list', - input: '- **a** c\n- d', - nodes: [['ul', {}, ['li', {}, ['strong', {}, 'a'], ' ', ['b', { $: inline }, 'c']], ['li', {}, 'd']]], - html: '
    \n
  • \n a c\n
  • \n
  • d
  • \n
', - markdown: '- **a** c\n- d', - }, -])('$name', async ({ input, nodes, html, markdown, ...options }) => { - const document = await parseMarkdown(input, options) - expect(document).toEqual({ frontmatter: {}, meta: {}, nodes }) - expect(await renderHtmlForTest(document)).toBe(html) - expect(await renderMarkdown(document)).toBe(markdown) -}) - -describe('HTML attributes', () => { - it('keeps a braced value with nested braces, quotes and spaces on an inline tag', async () => { - const result = await parseMarkdown('a y') - - expect(result.nodes).toEqual([ - ['p', {}, 'a ', ['b', { $: inline, data: '{`x ${1}`}', on: '{{ k: { v: 1 } }}', d: true }, 'y']], - ]) - }) - - it('keeps the first of duplicate attributes whatever their case', async () => { - const result = await parseMarkdown('t') - - expect(result.nodes).toEqual([['a', { $: inline, href: 'x', title: true }, 't']]) - }) +async function roundTrip(source: string, options?: Parameters[1]) { + const first = await parseMarkdown(source, options) + const markdown = await renderMarkdown(first) + return { first, markdown, again: await parseMarkdown(markdown, options), html: await renderHtmlForTest(first) } +} - it('keeps an attribute named like an Object.prototype member', async () => { - const result = await parseMarkdown('
x
') +it('reads as a comment inline', async () => { + const result = await parseMarkdown('a c') - expect(result.nodes).toEqual([['div', { $: block, constructor: 'a' }, 'x']]) - }) + expect(result.nodes).toEqual([['p', {}, 'a ', [null, {}, '- b -'], ' c']]) }) -describe('implied end tags', () => { - it('closes an open

across blocks and inline elements', async () => { - const result = await parseMarkdown('

\n\n

a x\n\n

b\n\n

') - - expect(result.nodes).toEqual([ - ['div', { $: block }, ['p', { $: block }, 'a ', ['b', { $: block }, 'x']], ['p', { $: block }, 'b']], - ]) - }) - - it('keeps a

nested in a table cell inside the outer

', async () => { - const result = await parseMarkdown('

a

b

') +it('settles on the paired tree once the closer streams in', async () => { + const source = 'text \n\n**a**\n\n' + const parse = createMarkdownParser() + for (let end = 1; end < source.length; end++) { + await expect(parse(source.slice(0, end))).resolves.toBeDefined() + } - expect(result.nodes).toEqual([ - [ - 'p', - { $: block }, - 'a', - ['table', { $: inline }, ['tr', { $: inline }, ['td', { $: inline }, ['p', { $: inline }, 'b']]]], - ], - ]) - }) + expect((await parse(source)).nodes).toEqual((await parseMarkdown(source)).nodes) + expect((await parse('text \n\n**a**')).nodes).toEqual([ + ['p', {}, 'text ', ['x-card', { $: inline }]], + ['p', {}, ['strong', {}, 'a']], + ]) }) -describe('inline content inside block HTML', () => { - const md = '

\n\nhello ![](x.png) a \nb :badge[x] y
end\n\n
' - - it('keeps images, line breaks, components and custom elements in their paragraph', async () => { - const result = await parseMarkdown(md, { autoUnwrap: false }) - - expect(result.nodes).toEqual([ - [ - 'div', - { $: block }, - [ - 'p', - {}, - 'hello ', - ['img', { src: 'x.png' }], - ' a', - ['br', {}], - 'b ', - ['badge', {}, 'x'], - ' ', - ['my-el', { $: inline }, 'y'], - ' ', - ['br', { $: inline }], - ' end', - ], - ], - ]) - }) - - it('wraps a markdown image on its own in a paragraph', async () => { - const result = await parseMarkdown('
\n\n![](x.png)\n\n![](y.png)\n\n
', { autoUnwrap: false }) - - expect(result.nodes).toEqual([ - ['div', { $: block }, ['p', {}, ['img', { src: 'x.png' }]], ['p', {}, ['img', { src: 'y.png' }]]], - ]) - }) - - it('leaves an author-written element on its own as a block', async () => { - const result = await parseMarkdown('
\n\ny\n\n\n\n
', { autoUnwrap: false }) - - expect(result.nodes).toEqual([ - ['div', { $: block }, ['my-el', { $: inline }, 'y'], ['img', { $: inline, src: 'a' }]], - ]) - }) - - it('keeps the order of a closed element and the one that follows it in a list item', async () => { - const result = await parseMarkdown('-
a
b\n- c') - - expect(result.nodes).toEqual([ - ['ul', {}, ['li', {}, ['div', { $: inline }, 'a'], ['div', { $: inline }, 'b']], ['li', {}, 'c']], - ]) - }) +it.each([ + '
a <b>c</b> &amp; <!-- x -->
', + '
', + '
', + '', + '
 a < b
', + '
ac
', + '', + 'before
**x** < y
after', +])('parses the saved Markdown of %s back to the same tree', async (source) => { + const { first, again } = await roundTrip(source) + + expect(again.nodes).toEqual(first.nodes) }) -describe('an open tag after text', () => { - it('ends with its paragraph inside a block element', async () => { - const result = await parseMarkdown('
\n\nHello x\n\nmore\n\n
', { autoUnwrap: false }) - - expect(result.nodes).toEqual([ - ['div', { $: block }, ['p', {}, 'Hello ', ['b', { $: inline }, 'x']], ['p', {}, 'more']], - ]) - }) - - it('keeps the text before a tag that a later paragraph closes', async () => { - const result = await parseMarkdown('text \n\n**a**\n\n') - - expect(result.nodes).toEqual(['text ', ['x-card', { $: block }, ['strong', {}, 'a']]]) - }) - - it('still lets a tag at the start of its paragraph collect the blocks after it', async () => { - const result = await parseMarkdown('\n\nNext paragraph.') - - expect(result.nodes).toEqual([['ai-thinking', { $: block }, 'Next paragraph.']]) - }) - - it('settles on the paired tree once the closer streams in', async () => { - const source = 'text \n\n**a**\n\n' - const parse = createMarkdownParser() - for (let end = 1; end < source.length; end++) { - await expect(parse(source.slice(0, end))).resolves.toBeDefined() - } +it('does not treat a name that only starts like a raw tag as one', async () => { + for (const [name, raw] of [ + ['style-x', 'style'], + ['pre2', 'pre'], + ['textarea-1', 'textarea'], + ]) { + const result = await parseMarkdown(`a <${name}>**x** after`) - expect((await parse(source)).nodes).toEqual((await parseMarkdown(source)).nodes) - expect((await parse('text \n\n**a**')).nodes).toEqual([ - ['p', {}, 'text ', ['x-card', { $: inline }]], - ['p', {}, ['strong', {}, 'a']], - ]) - }) + expect(JSON.stringify(result.nodes)).toContain('"strong"') + } }) -describe('HTML comments', () => { - it('reads as a comment inline', async () => { - const result = await parseMarkdown('a c') - - expect(result.nodes).toEqual([['p', {}, 'a ', [null, {}, '- b -'], ' c']]) - }) - - it('does not wrap an indented comment in a paragraph', async () => { - const result = await parseMarkdown(' \n\n ') +describe('large inputs', () => { + // Correctness at a size where a quadratic scan would time out the suite; scaling is measured by hand. + it('parses many unpaired raw openers in one paragraph', async () => { + const result = await parseMarkdown('a \n\n\n\n' - ) - - expect(result.nodes).toEqual([ - ['textarea', { $: block }, 'a < b'], - ['script', { $: block }, 'a < b'], - ['style', { $: block }, 'a < b'], - ]) - }) - - it('writes the decoded text of
 back as references', async () => {
-    const source = '
a <b>c</b> &amp; <!-- x -->
' - const first = await parseMarkdown(source) - const rendered = await renderMarkdown(first) - - expect(rendered).toBe('
a <b>c</b> &amp; <!-- x -->
') - expect((await parseMarkdown(rendered)).nodes).toEqual(first.nodes) - }) -}) - -describe('literal text round trip', () => { - async function roundTrip(source: string, options?: Parameters[1]) { - const first = await parseMarkdown(source, options) - const markdown = await renderMarkdown(first) - return { first, markdown, again: await parseMarkdown(markdown, options), html: await renderHtmlForTest(first) } - } - it('shows an encoded reference in
, ')
-
-    expect(first.nodes).toEqual([['textarea', { $: block }, 'a & b']])
-    expect(markdown).toBe('')
-    expect(again.nodes).toEqual(first.nodes)
-  })
-
-  it('does not read markdown in a 
 that follows text', async () => {
-    const source = 'before 
**x** < y
after' - const { first, markdown, again, html: output } = await roundTrip(source) - - expect(first.nodes).toEqual([['p', {}, 'before ', ['pre', { $: inline }, '**x** < y'], ' after']]) - expect(markdown).toBe(source) - expect(output).toBe('

before

**x** < y
after

') - expect(again.nodes).toEqual(first.nodes) - }) - it('shows the decoded text of an inline tag as the text it spells', async () => { const first = await parseMarkdown('
<b> &lt;
', literal) expect(await renderHtmlForTest(first)).toBe('
\n <b> &lt;\n
') }) - it('keeps a
 after text unindented in the paragraph', async () => {
-    const { html: output } = await roundTrip('before 
\nline\n  indented
after') - - expect(output).toBe('

before

\nline\n  indented
after

') - }) -}) - -describe('large inputs', () => { - // Correctness at a size where a quadratic scan would time out the suite; scaling is measured by hand. - it('parses many unpaired raw openers in one paragraph', async () => { - const result = await parseMarkdown('a