feat: entities option for createSaxParser and XmlParseStream

docs/html.md

@@ -7,12 +7,12 @@ Eksml can parse and serialize HTML through the `html: true` option, which is acc
 
 ## What `html: true` changes per API
 
-| API                 | Effect of `html: true`                                                                                                                                                                                                               |
-| ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
-| `parse()`           | `selfClosingTags` defaults to `HTML_VOID_ELEMENTS`, `rawContentTags` defaults to `['script', 'style', 'textarea', 'title']`. With `entities: true`, the full HTML named entity set is decoded instead of only the five XML entities. |
-| `write()`           | Void elements serialize as `<br>` with no closing tag. With `entities: true`, reserved characters are escaped with named entities in both text and attributes.                                                                       |
-| `createSaxParser()` | Same `selfClosingTags` and `rawContentTags` defaults as `parse()`. Void elements fire `openTag` followed immediately by `closeTag`. Raw content tags emit their body as a single `text` event.                                       |
-| `XmlParseStream`    | Same `selfClosingTags` and `rawContentTags` defaults as `parse()`. The stream has no `entities` option, decode after the fact if you need it.                                                                                        |
+| API                 | Effect of `html: true`                                                                                                                                                                                                                                           |
+| ------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `parse()`           | `selfClosingTags` defaults to `HTML_VOID_ELEMENTS`, `rawContentTags` defaults to `['script', 'style', 'textarea', 'title']`. With `entities: true`, the full HTML named entity set is decoded instead of only the five XML entities.                             |
+| `write()`           | Void elements serialize as `<br>` with no closing tag. With `entities: true`, reserved characters are escaped with named entities in both text and attributes.                                                                                                   |
+| `createSaxParser()` | Same `selfClosingTags` and `rawContentTags` defaults as `parse()`. Void elements fire `openTag` followed immediately by `closeTag`. Raw content tags emit their body as a single `text` event. With `entities: true`, the full HTML named entity set is decoded. |
+| `XmlParseStream`    | Same `selfClosingTags` and `rawContentTags` defaults as `parse()`. With `entities: true`, the full HTML named entity set is decoded.                                                                                                                             |
 
 In every API, an explicit `selfClosingTags` or `rawContentTags` array overrides the HTML defaults, so `html: true` is shorthand for sensible defaults rather than a separate parsing algorithm.
 

docs/limitations.md

@@ -76,7 +76,7 @@ There is no `//order/item[@status="paid"]` style query language. If your use cas
 
 ## Entity decoding is opt-in
 
-By default, entity references in text and attribute values are preserved exactly as written. Pass `entities: true` to decode them:
+By default, entity references in text and attribute values are preserved exactly as written. Pass `entities: true` to `parse()`, `createSaxParser()` or `XmlParseStream` to decode them:
 
 ```ts
 import { parse } from '@eksml/xml/parser';

docs/parsing.md

@@ -207,6 +207,13 @@ parse('<a><![CDATA[&amp;]]></a>', { entities: true });
 // => children: ['&amp;']
 ```
 
+Whitespace is judged on the text as written, before decoding. `trimWhitespace` trims the source text and then decodes it, and encoded whitespace such as `&#32;` counts as text, so it is kept. `createSaxParser()` and `XmlParseStream` follow the same order, which means `parse(xml, { entities: true, trimWhitespace: true })` produces the same text as the streaming parsers with `entities: true`:
+
+```ts
+parse('<a> &#32;x </a>', { entities: true, trimWhitespace: true });
+// => children: [' x']
+```
+
 ## Strict mode
 
 `strict: true` makes the parser throw instead of recovering. Errors include line and column position. It catches:

docs/sax-parser.md

@@ -110,8 +110,26 @@ const parser = createSaxParser({ html: true });
     <td><code>[]</code> / <code>['script','style','textarea','title']</code></td>
     <td>Tag names whose content is raw text (emitted as a single <code>text</code> event, not parsed for child elements)</td>
   </tr>
+  <tr>
+    <td><code>entities</code></td>
+    <td><code>boolean</code></td>
+    <td><code>false</code></td>
+    <td>Decode XML/HTML entities in text and attribute values, the same way <code>parse()</code> does</td>
+  </tr>
 </table>
 
+With `entities: true`, `text` events and `openTag` attribute values arrive decoded. Raw content text, CDATA, comments, processing instructions and doctypes are passed through as written. See [entity decoding](./parsing.md#entity-decoding) for which entities each mode recognizes.
+
+```ts
+const parser = createSaxParser({ entities: true });
+parser.on('openTag', (name, attributes) => console.log(attributes));
+parser.on('text', (text) => console.log(text));
+parser.write('<a title="A &amp; B">Tom &amp; Jerry</a>');
+parser.close();
+// { title: 'A & B' }
+// 'Tom & Jerry'
+```
+
 With `html: true`, void elements like `<br>` fire both `openTag` and `closeTag`, and script and style content arrives as one raw `text` event:
 
 ```ts

docs/web-streams.md

@@ -61,6 +61,12 @@ Input can be split anywhere, mid-tag, mid-attribute, or mid-entity; the parser b
     <td><code>false</code></td>
     <td>Include comments in the output (as strings like <code>'&lt;!-- ... --&gt;'</code>, either emitted directly at the top level or attached as children)</td>
   </tr>
+  <tr>
+    <td><code>entities</code></td>
+    <td><code>boolean</code></td>
+    <td><code>false</code></td>
+    <td>Decode XML/HTML entities in text and attribute values, the same way <code>parse()</code> does</td>
+  </tr>
   <tr>
     <td><code>select</code></td>
     <td><code>string | string[]</code></td>

eksml/src/parser.ts

@@ -782,6 +782,10 @@ export function parse(
         ? decodeHTML
         : decodeXML
       : null;
+  // Text is classified and trimmed on the source, then decoded, matching
+  // the SAX engine. decode of a string without '&' is identity.
+  const decodeText = (text: string): string =>
+    decode !== null && text.indexOf('&') !== -1 ? decode(text) : text;
 
   const selfClosingTagList: string[] =
     resolvedOptions.selfClosingTags ?? (htmlMode ? HTML_VOID_ELEMENTS : []);
@@ -1271,7 +1275,7 @@ export function parse(
       } else {
         // ---- Text (inline parseText) ----
         const textStart = pos;
-        if (decode === null && !trimWhitespace) {
+        if (!trimWhitespace) {
           // Fused scan + classification: walk the (typically short)
           // leading whitespace in JS; the first non-ws char both settles
           // the classification and hands the rest of the scan to the
@@ -1321,7 +1325,9 @@ export function parse(
           }
           if ((flags & HAS_TEXT) !== 0) {
             // Mixed content already established — push directly
-            children[children.length] = src.substring(textStart, textEnd);
+            children[children.length] = decodeText(
+              src.substring(textStart, textEnd),
+            );
           } else if (wsOnly) {
             // Defer: record the range, allocate only if the container
             // turns out to keep its whitespace
@@ -1333,7 +1339,9 @@ export function parse(
           } else {
             if (pendingCount > pendingBase)
               materializePending(S, children, pendingBase);
-            children[children.length] = src.substring(textStart, textEnd);
+            children[children.length] = decodeText(
+              src.substring(textStart, textEnd),
+            );
             flags |= HAS_TEXT;
           }
           continue;
@@ -1345,78 +1353,47 @@ export function parse(
         } else {
           pos = textEnd;
         }
-        if (decode === null) {
-          // No decoding, trimWhitespace mode (the non-trim case took the
-          // fused path above).
-          if (strict && depth === 0 && currentTagName === '') {
-            if (!isWsRange(textStart, textEnd)) {
-              throw strictError('Text content outside of a tag', textStart);
-            }
+        // trimWhitespace mode (the non-trim case took the fused path above).
+        if (strict && depth === 0 && currentTagName === '') {
+          if (!isWsRange(textStart, textEnd)) {
+            throw strictError('Text content outside of a tag', textStart);
+          }
+        }
+        // Fused trim: compute bounds on the source, substring once;
+        // ws-only runs allocate nothing. Falls back to substring+trim
+        // when a non-ASCII char sits at a boundary (Unicode whitespace).
+        let trimStart = textStart;
+        let trimLast = textEnd - 1;
+        let exact = true;
+        while (trimStart <= trimLast) {
+          const c = src.charCodeAt(trimStart);
+          if (c === 32 || (c >= 9 && c <= 13)) {
+            trimStart++;
+            continue;
           }
-          // Fused trim: compute bounds on the source, substring once;
-          // ws-only runs allocate nothing. Falls back to substring+trim
-          // when a non-ASCII char sits at a boundary (Unicode whitespace).
-          let trimStart = textStart;
-          let trimLast = textEnd - 1;
-          let exact = true;
-          while (trimStart <= trimLast) {
-            const c = src.charCodeAt(trimStart);
+          if (c >= 128) exact = false;
+          break;
+        }
+        if (exact && trimStart <= trimLast) {
+          while (trimLast > trimStart) {
+            const c = src.charCodeAt(trimLast);
             if (c === 32 || (c >= 9 && c <= 13)) {
-              trimStart++;
+              trimLast--;
               continue;
             }
             if (c >= 128) exact = false;
             break;
           }
-          if (exact && trimStart <= trimLast) {
-            while (trimLast > trimStart) {
-              const c = src.charCodeAt(trimLast);
-              if (c === 32 || (c >= 9 && c <= 13)) {
-                trimLast--;
-                continue;
-              }
-              if (c >= 128) exact = false;
-              break;
-            }
-          }
-          if (!exact) {
-            const trimmed = src.substring(textStart, textEnd).trim();
-            if (trimmed.length > 0) {
-              children.push(trimmed);
-              flags |= HAS_TEXT;
-            }
-          } else if (trimStart <= trimLast) {
-            children.push(src.substring(trimStart, trimLast + 1));
+        }
+        if (!exact) {
+          const trimmed = src.substring(textStart, textEnd).trim();
+          if (trimmed.length > 0) {
+            children.push(decodeText(trimmed));
             flags |= HAS_TEXT;
           }
-        } else {
-          // Entity decoding active: classification must look at the
-          // decoded string, so no deferral on this path. decode of a
-          // string without '&' is identity, so gate on indexOf.
-          let text = src.substring(textStart, textEnd);
-          if (text.indexOf('&') !== -1) text = decode(text);
-          if (
-            strict &&
-            depth === 0 &&
-            currentTagName === '' &&
-            text.trim().length > 0
-          ) {
-            throw strictError('Text content outside of a tag', textStart);
-          }
-          if (trimWhitespace) {
-            const trimmed = text.trim();
-            if (trimmed.length > 0) {
-              children.push(trimmed);
-              flags |= HAS_TEXT;
-            }
-          } else if (text.length > 0) {
-            if ((flags & HAS_TEXT) === 0) {
-              flags |= isWsString(text)
-                ? HAS_WS_TEXT | HAS_WS_IN_ARRAY
-                : HAS_TEXT;
-            }
-            children[children.length] = text;
-          }
+        } else if (trimStart <= trimLast) {
+          children.push(decodeText(src.substring(trimStart, trimLast + 1)));
+          flags |= HAS_TEXT;
         }
       }
     }

eksml/src/sax.ts

@@ -18,6 +18,7 @@
  * ```
  */
 
+import { decodeXML, decodeHTML } from 'entities';
 import type { SaxEngineHandlers, Attributes } from '#src/saxEngine.ts';
 import { saxEngine } from '#src/saxEngine.ts';
 import {
@@ -70,6 +71,21 @@ export interface SaxParserOptions {
   selfClosingTags?: string[];
   /** Tag names whose content is raw text. Defaults to `['script', 'style']` when `html: true`. */
   rawContentTags?: string[];
+  /**
+   * Decode XML/HTML entities in text content and attribute values.
+   * When enabled, named entities (`&`, `<`, etc.), decimal character
+   * references (`ä`), and hex character references (`ä`) are decoded.
+   *
+   * In HTML mode (`html: true`), the full set of HTML named entities is
+   * supported (e.g. ` `, `©`, `—`). In XML mode, only the
+   * five standard XML entities plus numeric references are decoded.
+   *
+   * Text is trimmed before it is decoded, so encoded whitespace such as
+   * ` ` is kept. CDATA sections and raw content text are never decoded.
+   *
+   * Defaults to `false` — entities are preserved as-is in the output.
+   */
+  entities?: boolean;
 }
 
 // ---------------------------------------------------------------------------
@@ -82,7 +98,7 @@ export interface SaxParserOptions {
  * Uses the internal SAX engine, but wraps it with `.on()` / `.off()` methods
  * so handlers can be added and removed dynamically.
  *
- * @param options - Parser options (html mode, selfClosingTags, rawContentTags).
+ * @param options - Parser options (html mode, selfClosingTags, rawContentTags, entities).
  * @returns A SaxParser with `.on()`, `.off()`, `.write()`, `.close()`.
  */
 export function createSaxParser(options?: SaxParserOptions): SaxParser {
@@ -208,6 +224,8 @@ export function createSaxParser(options?: SaxParserOptions): SaxParser {
     ...handlers,
     selfClosingTags,
     rawContentTags,
+    decode:
+      opts.entities === true ? (isHtml ? decodeHTML : decodeXML) : undefined,
   });
 
   return {

eksml/src/saxEngine.ts

@@ -70,6 +70,13 @@ export interface SaxEngineOptions extends SaxEngineHandlers {
    * this limit a `RangeError` is thrown.  Default `undefined` (no limit).
    */
   maxBufferSize?: number;
+  /**
+   * Decodes entities in text runs and attribute values. Applied after
+   * trimming, and only to strings containing `&`. Raw content text, CDATA,
+   * comments, processing instructions and doctypes are never decoded.
+   * Default `undefined` (entities are left as written).
+   */
+  decode?: (text: string) => string;
 }
 
 /** The parser instance returned by `saxEngine()`. */
@@ -148,6 +155,7 @@ export function saxEngine(options: SaxEngineOptions = {}): SaxEngineParser {
     rawContentTags = [],
     maxBufferSize,
   } = options;
+  const decode = options.decode ?? null;
 
   const voidSet: Set<string> | null =
     selfClosingTags.length > 0 ? new Set(selfClosingTags) : null;
@@ -265,11 +273,30 @@ export function saxEngine(options: SaxEngineOptions = {}): SaxEngineParser {
     if (text.length === 0) return;
     if (onText && !textSuppressed) {
       const trimmed = trimWhitespace(text);
-      if (trimmed.length > 0) onText(trimmed);
+      if (trimmed.length > 0)
+        onText(
+          decode !== null && trimmed.indexOf('&') !== -1
+            ? decode(trimmed)
+            : trimmed,
+        );
     }
     text = '';
   }
 
+  /** Decode attribute values in place. */
+  function decodeAttributes(
+    attrs: Attributes,
+    decodeValue: (text: string) => string,
+  ): void {
+    // for-in avoids allocating a key array per tag; attribute records are
+    // plain objects with no inherited enumerable keys.
+    for (const name in attrs) {
+      const value = attrs[name];
+      if (value != null && value.indexOf('&') !== -1)
+        setOwnProperty(attrs, name, decodeValue(value));
+    }
+  }
+
   /**
    * Parse the accumulated DOCTYPE body (everything between `<!` and `>`,
    * excluding internal DTD subsets) and emit an onDoctype event.
@@ -345,7 +372,10 @@ export function saxEngine(options: SaxEngineOptions = {}): SaxEngineParser {
 
   /** After we finish parsing an open tag's `>`, handle void/raw transitions. */
   function finishOpenTag(): void {
-    if (onOpenTag) onOpenTag(tagName, attributes);
+    if (onOpenTag) {
+      if (decode !== null) decodeAttributes(attributes, decode);
+      onOpenTag(tagName, attributes);
+    }
     if (voidSet !== null && voidSet.has(tagName)) {
       if (onCloseTag) onCloseTag(tagName);
     } else if (rawSet !== null && rawSet.has(tagName)) {
@@ -429,7 +459,12 @@ export function saxEngine(options: SaxEngineOptions = {}): SaxEngineParser {
                     if (c !== SPACE && c !== TAB && c !== LF && c !== CR) break;
                     e--;
                   }
-                  handleText(chunk.substring(s, e + 1));
+                  const run = chunk.substring(s, e + 1);
+                  handleText(
+                    decode !== null && run.indexOf('&') !== -1
+                      ? decode(run)
+                      : run,
+                  );
                 }
               }
             } else {
@@ -703,7 +738,11 @@ export function saxEngine(options: SaxEngineOptions = {}): SaxEngineParser {
             }
 
             // --- emit open tag ---
-            if (handleOpenTag !== undefined) handleOpenTag(name, attrs!);
+            if (handleOpenTag !== undefined) {
+              if (decode !== null && attrs !== null)
+                decodeAttributes(attrs, decode);
+              handleOpenTag(name, attrs!);
+            }
             if (selfClosed) {
               if (handleCloseTag !== undefined) handleCloseTag(name);
               continue fastPath;
@@ -1020,7 +1059,10 @@ export function saxEngine(options: SaxEngineOptions = {}): SaxEngineParser {
           if (charCode === GT) {
             state = State.TEXT;
             i++;
-            if (onOpenTag) onOpenTag(tagName, attributes);
+            if (onOpenTag) {
+              if (decode !== null) decodeAttributes(attributes, decode);
+              onOpenTag(tagName, attributes);
+            }
             if (onCloseTag) onCloseTag(tagName);
           } else {
             state = State.OPEN_TAG_BODY;