fix(html): preserve HTML structure and literal text

packages/comark/SPEC/HTML/README.md

@@ -1,6 +1,6 @@
 # HTML parsing contract
 
-Behavioral contract for embedded HTML. Fixtures in this directory are the user-facing cases. `test/html-block.test.ts` covers the option matrix and the cases deferred below.
+Behavioral contract for embedded HTML. Fixtures in this directory are the user-facing cases. `test/html-tags.test.ts` covers tag parsing and rendering; `test/html-block.test.ts` covers the option matrix and the cases deferred below.
 
 HTML tags become element nodes with `$: { html: 1, block: 0 | 1 }`. A node without `$` is markdown-originated (`<strong>` from `**bold**` is `['strong', {}, 'bold']`).
 
@@ -15,6 +15,8 @@ HTML tags become element nodes with `$: { html: 1, block: 0 | 1 }`. A node witho
 
 Void tags (`img`, `br`, …) are always `block: 0`.
 
+A non-void open tag that follows text in its paragraph (`GET /apps/<id>`, `- **a** <b>c`) is prose, not a container. It ends with its paragraph, `block: 0`, unless a later paragraph of the same container holds its close tag. A close tag in a nested container (`::card`, a blockquote) or past the end of the container does not count. A tag that starts its paragraph keeps collecting the blocks after it.
+
 Consecutive same-paragraph tags stay inline even if indented on following lines:
 
 ```md
@@ -41,7 +43,13 @@ Blank-line bodies parse as markdown in both modes. CommonMark ends the HTML bloc
 
 `<style>`, `<pre>`, `<script>`, and `<textarea>` always stay raw blocks, including with attributes. None of those bodies is markdown, in either mode. The match is the tag name plus a boundary, so `<prelude>` is not `<pre>` and `<scripture>` is not `<script>`.
 
-All four keep every character between the tags, including the leading newline and indentation. Inner tags stay text until the matching closer, including inside `<pre>` and `<style>`.
+All four keep every character between the tags, including the leading newline and indentation, whether the element starts its block or follows text in a paragraph (`before <pre>**x**</pre>`) as long as its closer is in that paragraph. What else the body means follows the HTML spec:
+
+| Tag | Inner tags | Character references |
+| --- | --- | --- |
+| `<style>`, `<script>` | text until the matching closer | kept as written |
+| `<textarea>` | text until the matching closer | decoded |
+| `<pre>` | HTML (`<pre><code>…</code></pre>`), unless an open tag has no closer inside the `<pre>` (`Array<string>`), which stays text | decoded |
 
 ```html
 <pre>
@@ -52,6 +60,8 @@ All four keep every character between the tags, including the leading newline an
 
 is the text `\n  const x = 1\n  **not**\n`, not a trimmed line and not a `strong`.
 
+Character references are decoded in text, with or without markdown: `&amp;`, `&lt;`, `&#65;`. As in markdown, a reference needs its `;` (`&copy` stays), and an unknown name stays. Written back, the text keeps its meaning: in HTML and Markdown output a decoded `&lt;` or `<b>` in an element's text is encoded again (`&amp;lt;`), so a reader sees the characters the author wrote. `<script>` and `<style>` text is written as it was read.
+
 ### Whitespace in a literal body
 
 `markdown: false` keeps horizontal spaces. A space between tags (`</a> <a>`) and a space beside text (`Hello <em>`) are content.
@@ -66,11 +76,28 @@ Runs that contain a newline are structural. Leading indent after a newline is st
 
 with `markdown: false` is the text `Hello ` plus `<em>`, not `"\n  Hello "` plus `<em>`.
 
+## Tags
+
+One grammar reads a tag in the inline rule and in the raw-block splitter (`src/internal/parse/html-tag.ts`). It is CommonMark's, plus:
+
+- An attribute value may be a balanced `{…}` expression, up to four levels deep: `<div style={{a: 1}}>` has `style="{{a: 1}}"`. The text is kept as written. Only braces are counted, not JavaScript: a brace inside a string (`{{a: "}"}}`) is not skipped, so the tag is not recognized.
+- A comment may end in `--->`, as in HTML: `<!--- a --->` is a comment with the text `- a -`.
+
+When an attribute repeats, the first value wins, as in HTML.
+
+A comment in a block of its own is a comment node, not a paragraph holding one, whatever its indentation.
+
+A few end tags are implied, as in HTML: a stray `</br>` is a `<br>`, and a `<p>` closes an open `<p>` (not across a `table`, `td`, `th`, `caption`, `button`, `template`, `object`, `marquee`, or `applet`). Nothing else is closed implicitly; for example a `<div>` does not close a `<p>`.
+
+## Inline content in a block
+
+Inside an element that collects blocks, a paragraph's inline content stays in one `p`. Markdown inline nodes (images, line breaks, components, math) always belong to it. An HTML tag does when it is one of `strong`, `em`, `del`, `code`, `a`, `span`, `sub`, `sup`, `kbd`, `img`, `br`, or a custom element (a name with a `-`) that closed in its paragraph. An author-written element that is alone in its paragraph is not wrapped, except for the first group.
+
 ## How a block becomes a tree
 
 1. markdown-exit tokenizes with `html: true`.
 2. When `markdown` is true, the built-in `html_block` rule is narrowed to `<style>`, `<pre>`, `<script>`, and `<textarea>`. Every other closed HTML region falls through to the normal block/inline rules, so `**`, `` ` ``, and nested tags tokenize as markdown. This override is the feature. It reads markdown-exit's private block ruler (`__rules__`); a markdown-exit upgrade that renames or stops consulting `html_block` silently disables markdown-in-HTML.
-3. Regions that remain `html_block` (those four tags, and every closed HTML block when `markdown` is false) are split into `html_inline` + literal `text` and wrapped in a paragraph so the same walk pairs them. The four raw-tag bodies are not split.
+3. Regions that remain `html_block` (those four tags, and every closed HTML block when `markdown` is false) are split into `html_inline` + literal `text` and wrapped in a paragraph so the same walk pairs them. `<pre>` bodies are split into HTML tags and text; the other three bodies stay text.
 4. The tree walk keeps an HTML stack across blocks, the way hast-util-raw feeds parse5. An open tag pushes a frame. Later paragraphs and blocks fill it. A matching close pops it and auto-closes mismatched frames in between. `</div><div>` emits the closed `div` as a sibling of the new open frame, not as its child.
 
 Two scanners must agree on what a tag is: the splitter for raw `html_block` text (`htmlToTokens` in `src/plugins/html.ts`) and the inline opener/closer parser (`parseHtmlInline` in `src/internal/parse/utils.ts`). A tag one accepts and the other rejects becomes a text leaf or an unclosed frame.

packages/comark/src/internal/parse/html-tag.ts

@@ -0,0 +1,27 @@
+// Adapted from markdown-exit's html_re, with two changes: attribute values may be
+// a balanced `{…}` expression (JSX style), and a comment may end in `--->` as in HTML.
+
+const attr_name = '[a-zA-Z_:][a-zA-Z0-9:._-]*'
+
+const braces = (depth: number): string => (depth === 0 ? '[^{}]*' : `(?:[^{}]|\\{${braces(depth - 1)}\\})*`)
+
+/** Source of a `{…}` attribute value. Nesting is limited to four levels. */
+export const braced_value = `\\{${braces(3)}\\}`
+
+const unquoted = '[^"\'=<>`\\x00-\\x20]+'
+const single_quoted = "'[^']*'"
+const double_quoted = '"[^"]*"'
+
+const attr_value = `(?:${braced_value}|${unquoted}|${single_quoted}|${double_quoted})`
+
+const attribute = `(?:\\s+${attr_name}(?:\\s*=\\s*${attr_value})?)`
+
+const open_tag = `<[A-Za-z][A-Za-z0-9\\-]*${attribute}*\\s*\\/?>`
+
+const close_tag = '<\\/[A-Za-z][A-Za-z0-9\\-]*\\s*>'
+const comment = '<!--(?:-?>|[\\s\\S]*?-->)'
+const processing = '<[?][\\s\\S]*?[?]>'
+const declaration = '<![A-Za-z][^>]*>'
+const cdata = '<!\\[CDATA\\[[\\s\\S]*?\\]\\]>'
+
+export const HTML_TAG_RE = new RegExp(`^(?:${open_tag}|${close_tag}|${comment}|${processing}|${declaration}|${cdata})`)

packages/comark/src/internal/parse/tree.ts

@@ -36,6 +36,7 @@ type HtmlOpenFrame = {
   block: boolean
   /** Bare-inline nodes waiting to be flushed as a single <p> on a block frame. */
   pendingInline: Node[]
+  afterText?: boolean
 }
 
 /** Shared mutable context for a whole document walk. */
@@ -53,6 +54,7 @@ interface ProcessState {
    * stack as a whole unit via deliverBlock.
    */
   insideMarkdownContainer: number
+  closers?: CloserIndex
 }
 
 type Processor = (tokens: Token[], start: number, state: ProcessState) => ProcessorResult
@@ -76,10 +78,23 @@ function isInlineish(node: Node): boolean {
   return tag === null || inlineTags.has(tag as string)
 }
 
+function isInlineHtml(node: Node): boolean {
+  if (typeof node === 'string') return false
+  const tag = node[0]
+  if (tag === null || inlineTags.has(tag) || tag === 'img' || tag === 'br') return true
+  return tag.includes('-') && node[1].$?.block === 0
+}
+
 function flushPendingInline(frame: HtmlOpenFrame) {
   if (frame.pendingInline.length === 0) return
   const pending = frame.pendingInline
   frame.pendingInline = []
+  const only = pending.length === 1 ? pending[0] : undefined
+  // A lone author-written element is a block of its own, not a paragraph.
+  if (Array.isArray(only) && only[1].$?.html && !isInlineish(only)) {
+    frame.children.push(only)
+    return
+  }
   frame.children.push(['p', {}, ...pending] as ElementNode)
 }
 
@@ -119,11 +134,11 @@ function deliverBlock(state: ProcessState, node: Node | undefined): Node | undef
 }
 
 /** Nest an inline/finished node into the open HTML frame during processInline. */
-function deliverInline(state: ProcessState, node: Node | undefined): Node | undefined {
+function deliverInline(state: ProcessState, node: Node | undefined, inline?: boolean): Node | undefined {
   if (node === undefined) return undefined
   if (state.htmlStack.length === 0) return node
   const top = state.htmlStack[state.htmlStack.length - 1]
-  if (top.block && isInlineish(node)) {
+  if (top.block && (inline ?? isInlineish(node))) {
     top.pendingInline.push(node)
   } else {
     flushPendingInline(top)
@@ -164,6 +179,91 @@ function tryCommentTag(content: string, state: ProcessState): ProcessorResult |
   return null
 }
 
+function closeFramesTo(state: ProcessState, idx: number): Node | undefined {
+  let closed: Node | undefined
+  while (state.htmlStack.length > idx) {
+    const node = frameToNode(state.htmlStack.pop()!)
+    if (state.htmlStack.length > idx) deliverInline(state, node)
+    else closed = node
+  }
+  return closed
+}
+
+// Elements that stop a `<p>` from being closed by a following `<p>` (HTML "button scope").
+const P_SCOPE_BOUNDARY = new Set(['table', 'td', 'th', 'caption', 'button', 'template', 'object', 'marquee', 'applet'])
+
+function closeImpliedParagraph(token: Token, state: ProcessState): Node | undefined {
+  if (!/^<p[\s/>]/i.test(token.content)) return undefined
+  const parsed = parseHtmlInline(token.content)
+  if (parsed.kind !== 'open' || parsed.tag !== 'p' || parsed.selfClosing) return undefined
+  for (let idx = state.htmlStack.length - 1; idx >= 0; idx--) {
+    const tag = state.htmlStack[idx].tag
+    if (tag === 'p') return closeFramesTo(state, idx)
+    if (tag !== null && P_SCOPE_BOUNDARY.has(tag)) return undefined
+  }
+  return undefined
+}
+
+const firstProse = new WeakMap<Token[], number>()
+
+function followsText(tokens: Token[], start: number): boolean {
+  let first = firstProse.get(tokens)
+  if (first === undefined) {
+    first = tokens.findIndex(
+      ({ type, content }) => !(type === 'softbreak' || type === 'html_inline' || (type === 'text' && !content.trim()))
+    )
+    if (first < 0) first = tokens.length
+    firstProse.set(tokens, first)
+  }
+  return first < start
+}
+
+/** Where each `</tag>` sits, so "is it closed later?" is a lookup and not a scan of the rest of the document. */
+interface CloserIndex {
+  /** Token indexes of the paragraphs holding a close tag, by `tag|level`. */
+  at: Map<string, number[]>
+  /** First later token with a lower level, i.e. where the container of token `i` ends. */
+  end: number[]
+}
+
+function buildCloserIndex(tokens: Token[]): CloserIndex {
+  const at = new Map<string, number[]>()
+  const end: number[] = new Array(tokens.length).fill(tokens.length)
+  const open: number[] = []
+  for (let i = 0; i < tokens.length; i++) {
+    while (open.length > 0 && tokens[open[open.length - 1]].level > tokens[i].level) end[open.pop()!] = i
+    open.push(i)
+    for (const child of tokens[i].type === 'inline' ? (tokens[i].children ?? []) : []) {
+      if (child.type !== 'html_inline' || child.content.charCodeAt(1) !== 0x2f /* / */) continue
+      const parsed = parseHtmlInline(child.content)
+      if (parsed.kind !== 'close') continue
+      const key = `${parsed.tag}|${tokens[i].level}`
+      const list = at.get(key)
+      if (list && list[list.length - 1] !== i) list.push(i)
+      else if (!list) at.set(key, [i])
+    }
+  }
+  return { at, end }
+}
+
+/**
+ * True when a later paragraph of the same container holds `</tag>`. A close tag inside a
+ * nested container, or past the end of this one, cannot pair with an opener out here.
+ */
+function hasCloserAhead(tokens: Token[], state: ProcessState, paragraph: number, from: number, tag: string): boolean {
+  const index = (state.closers ??= buildCloserIndex(tokens))
+  const list = index.at.get(`${tag}|${tokens[paragraph].level + 1}`)
+  if (!list) return false
+  let lo = 0
+  let hi = list.length
+  while (lo < hi) {
+    const mid = (lo + hi) >> 1
+    if (list[mid] < from) lo = mid + 1
+    else hi = mid
+  }
+  return lo < list.length && list[lo] < index.end[paragraph]
+}
+
 function processToken(tokens: Token[], i: number, state: ProcessState): ProcessorResult | undefined {
   return processors[tokens[i].type]?.(tokens, i, state)
 }
@@ -180,12 +280,22 @@ function processInline(inlineTokens: Token[], state: ProcessState, nestHtml = tr
   let i = 0
 
   while (i < inlineTokens.length) {
+    const fromHtml = inlineTokens[i].type === 'html_inline'
+    if (nestHtml && fromHtml) {
+      const closed = closeImpliedParagraph(inlineTokens[i], state)
+      if (closed !== undefined) {
+        const free = deliverInline(state, closed)
+        if (free !== undefined) pushNode(nodes, free)
+      }
+    }
     const result = processToken(inlineTokens, i, state)
     if (!result) {
       i += 1
       continue
     }
-    const node = nestHtml ? deliverInline(state, result.node) : result.node
+    const node = nestHtml
+      ? deliverInline(state, result.node, !fromHtml || (result.node !== undefined && isInlineHtml(result.node)))
+      : result.node
     if (node !== undefined) pushNode(nodes, node)
     i = result.nextIndex
   }
@@ -235,7 +345,7 @@ function processChildren(
     if (result) {
       // Same nesting rule as processInline: free nodes join the open HTML
       // frame when nestHtml is on (e.g. block children under bare HTML).
-      const node = nestHtml ? deliverInline(state, result.node) : result.node
+      const node = nestHtml && !result.siblingOfHtmlStack ? deliverInline(state, result.node) : result.node
       if (node !== undefined) pushNode(children, node)
       i = result.nextIndex
     } else {
@@ -484,6 +594,17 @@ const processors: Record<string, Processor> = {
     const depthBefore = state.htmlStack.length
     const nestHtml = state.insideMarkdownContainer === 0
     const { children, nextIndex } = processChildren(tokens, start, 'paragraph_close', state, nestHtml)
+    // `text <tag>` with no `</tag>` anywhere after is prose, not a container: it ends with its paragraph.
+    const opened = state.htmlStack[depthBefore]
+    if (
+      nestHtml &&
+      opened?.afterText &&
+      opened.tag !== null &&
+      !hasCloserAhead(tokens, state, start, nextIndex, opened.tag)
+    ) {
+      const free = deliverInline(state, closeFramesTo(state, depthBefore), true)
+      if (free !== undefined) pushNode(children, free)
+    }
     const asParagraph = (): ProcessorResult =>
       children.length === 0
         ? { nextIndex, node: undefined }
@@ -571,7 +692,7 @@ const processors: Record<string, Processor> = {
 
     // `<!--` / `-->` are HTML open/close with tag null (may arrive as plain text
     // when blank lines split a multi-line comment).
-    const comment = tryCommentTag(content, state)
+    const comment = tokens[start].meta?.literal ? null : tryCommentTag(content, state)
     if (comment) return { nextIndex: start + 1, node: comment.node }
 
     return { nextIndex: start + 1, node: content }
@@ -597,6 +718,7 @@ const processors: Record<string, Processor> = {
     }
 
     if (parsed.kind === 'close') {
+      if (parsed.tag === 'br') return { nextIndex: start + 1, node: ['br', makeHtmlAttrs({}, false)] }
       // Find the matching open frame (case-insensitive), auto-closing any
       // nested frames above it the way a real HTML parser would.
       const closeTag = parsed.tag.toLowerCase()
@@ -608,17 +730,7 @@ const processors: Record<string, Processor> = {
       }
       if (idx < 0) return { nextIndex: start + 1, node: undefined }
 
-      let closed: Node | undefined
-      while (state.htmlStack.length > idx) {
-        const node = frameToNode(state.htmlStack.pop()!)
-        // Nested auto-closed frames nest into the next outer open frame.
-        if (state.htmlStack.length > idx) {
-          deliverInline(state, node)
-        } else {
-          closed = node
-        }
-      }
-      return { nextIndex: start + 1, node: closed }
+      return { nextIndex: start + 1, node: closeFramesTo(state, idx) }
     }
 
     if (parsed.selfClosing) {
@@ -634,6 +746,7 @@ const processors: Record<string, Processor> = {
       children: [],
       block: false,
       pendingInline: [],
+      afterText: followsText(tokens, start),
     })
     return { nextIndex: start + 1, node: undefined }
   },

packages/comark/src/internal/parse/utils.ts

@@ -1,4 +1,5 @@
 import type { Node, ElementNode, ElementNodeAttributes, MarkdownDocument } from 'comark'
+import { braced_value } from './html-tag.ts'
 import { decodeHTML, isHtmlVoidElement, TREE_WALK_MAX_DEPTH } from 'comark/utils'
 
 export type ParsedHtmlTag =
@@ -62,7 +63,13 @@ function applyAutoUnwrapAt(node: Node, depth: number): Node {
     return node
   }
 
-  if (significantCount === 1 && Array.isArray(significant) && significant[0] === 'p') {
+  // An author-written `<p>` (`$.html`) is content, not a wrapper.
+  if (
+    significantCount === 1 &&
+    Array.isArray(significant) &&
+    significant[0] === 'p' &&
+    !(significant[1] as ElementNodeAttributes | undefined)?.$?.html
+  ) {
     // Lift the paragraph's attrs onto the parent so trailing `{attr}` survives the unwrap.
     // Parent attrs take precedence so explicit component props aren't overridden.
     const paragraphAttrs = significant[1] as Record<string, unknown> | undefined
@@ -181,6 +188,11 @@ export function parseHtmlInline(content: string): ParsedHtmlTag {
   return { kind: 'other', content: trimmed }
 }
 
+const ATTRIBUTE_RE = new RegExp(
+  `([:\\w.-]+)(?:\\s*=\\s*(?:"([^"]*)"|'([^']*)'|(${braced_value})|([^\\s"'=<>\`]+)))?`,
+  'g'
+)
+
 /**
  * Parse HTML attribute string into a key/value map.
  * Supports `key="value"`, `key='value'`, bare `key=value`, and boolean attributes.
@@ -190,11 +202,10 @@ export function parseHtmlAttributes(attrsStr: string): Record<string, unknown> {
   const attrs: Record<string, unknown> = {}
   if (!attrsStr || !attrsStr.trim()) return attrs
 
-  const re = /([:\w.-]+)(?:\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'=<>`]+)))?/g
-  let match: RegExpExecArray | null
-  while ((match = re.exec(attrsStr)) !== null) {
+  for (const match of attrsStr.matchAll(ATTRIBUTE_RE)) {
     const key = match[1].toLowerCase()
-    const value = match[2] ?? match[3] ?? match[4]
+    if (Object.hasOwn(attrs, key)) continue
+    const value = match[2] ?? match[3] ?? match[4] ?? match[5]
     attrs[key] = value === undefined ? true : decodeHTML(value)
   }
 

packages/comark/src/internal/stringify/handlers/comment.ts

@@ -3,7 +3,7 @@ import type { ElementNode } from 'comark'
 
 export function comment(node: ElementNode, _state: State) {
   if (node[0] === null) {
-    return `<!--${node[2]}-->` + _state.context.blockSeparator
+    return `<!--${node[2]}-->` + (_state.context.rawText ? '' : _state.context.blockSeparator)
   }
 
   return ''