From b04943e21c107c23ff33cb91f11c084e9fa2c9d2 Mon Sep 17 00:00:00 2001 From: Liam Potter Date: Tue, 20 Oct 2026 15:08:01 +0100 Subject: [PATCH] feat: entities option for createSaxParser and XmlParseStream Both streaming parsers take `entities: true` and decode text and attribute values the same way parse() does, using the HTML entity set in HTML mode. CDATA, raw content, comments, processing instructions and doctypes are left as written. Text is trimmed before it is decoded. parse() now follows the same order, so encoded whitespace such as counts as text and is kept. parse() with entities and trimWhitespace now produces the same text as the streaming parsers. --- docs/html.md | 12 +- docs/limitations.md | 2 +- docs/parsing.md | 7 + docs/sax-parser.md | 18 ++ docs/web-streams.md | 6 + eksml/src/parser.ts | 109 +++++------- eksml/src/sax.ts | 20 ++- eksml/src/saxEngine.ts | 52 +++++- eksml/src/xmlParseStream.ts | 27 ++- eksml/test/entities.test.ts | 325 ++++++++++++++++++++++++++++++++++++ 10 files changed, 497 insertions(+), 81 deletions(-) create mode 100644 eksml/test/entities.test.ts diff --git a/docs/html.md b/docs/html.md index 62509da..4100bf8 100644 --- a/docs/html.md +++ b/docs/html.md @@ -7,12 +7,12 @@ Eksml can parse and serialize HTML through the `html: true` option, which is acc ## What `html: true` changes per API -| API | Effect of `html: true` | -| ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `parse()` | `selfClosingTags` defaults to `HTML_VOID_ELEMENTS`, `rawContentTags` defaults to `['script', 'style', 'textarea', 'title']`. With `entities: true`, the full HTML named entity set is decoded instead of only the five XML entities. | -| `write()` | Void elements serialize as `
` with no closing tag. With `entities: true`, reserved characters are escaped with named entities in both text and attributes. | -| `createSaxParser()` | Same `selfClosingTags` and `rawContentTags` defaults as `parse()`. Void elements fire `openTag` followed immediately by `closeTag`. Raw content tags emit their body as a single `text` event. | -| `XmlParseStream` | Same `selfClosingTags` and `rawContentTags` defaults as `parse()`. The stream has no `entities` option, decode after the fact if you need it. | +| API | Effect of `html: true` | +| ------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `parse()` | `selfClosingTags` defaults to `HTML_VOID_ELEMENTS`, `rawContentTags` defaults to `['script', 'style', 'textarea', 'title']`. With `entities: true`, the full HTML named entity set is decoded instead of only the five XML entities. | +| `write()` | Void elements serialize as `
` with no closing tag. With `entities: true`, reserved characters are escaped with named entities in both text and attributes. | +| `createSaxParser()` | Same `selfClosingTags` and `rawContentTags` defaults as `parse()`. Void elements fire `openTag` followed immediately by `closeTag`. Raw content tags emit their body as a single `text` event. With `entities: true`, the full HTML named entity set is decoded. | +| `XmlParseStream` | Same `selfClosingTags` and `rawContentTags` defaults as `parse()`. With `entities: true`, the full HTML named entity set is decoded. | In every API, an explicit `selfClosingTags` or `rawContentTags` array overrides the HTML defaults, so `html: true` is shorthand for sensible defaults rather than a separate parsing algorithm. diff --git a/docs/limitations.md b/docs/limitations.md index ad28c6a..991a124 100644 --- a/docs/limitations.md +++ b/docs/limitations.md @@ -76,7 +76,7 @@ There is no `//order/item[@status="paid"]` style query language. If your use cas ## Entity decoding is opt-in -By default, entity references in text and attribute values are preserved exactly as written. Pass `entities: true` to decode them: +By default, entity references in text and attribute values are preserved exactly as written. Pass `entities: true` to `parse()`, `createSaxParser()` or `XmlParseStream` to decode them: ```ts import { parse } from '@eksml/xml/parser'; diff --git a/docs/parsing.md b/docs/parsing.md index a6deb48..e582388 100644 --- a/docs/parsing.md +++ b/docs/parsing.md @@ -207,6 +207,13 @@ parse('', { entities: true }); // => children: ['&'] ``` +Whitespace is judged on the text as written, before decoding. `trimWhitespace` trims the source text and then decodes it, and encoded whitespace such as ` ` counts as text, so it is kept. `createSaxParser()` and `XmlParseStream` follow the same order, which means `parse(xml, { entities: true, trimWhitespace: true })` produces the same text as the streaming parsers with `entities: true`: + +```ts +parse(' x ', { entities: true, trimWhitespace: true }); +// => children: [' x'] +``` + ## Strict mode `strict: true` makes the parser throw instead of recovering. Errors include line and column position. It catches: diff --git a/docs/sax-parser.md b/docs/sax-parser.md index 5a7cb48..dbd8506 100644 --- a/docs/sax-parser.md +++ b/docs/sax-parser.md @@ -110,8 +110,26 @@ const parser = createSaxParser({ html: true }); [] / ['script','style','textarea','title'] Tag names whose content is raw text (emitted as a single text event, not parsed for child elements) + + entities + boolean + false + Decode XML/HTML entities in text and attribute values, the same way parse() does + +With `entities: true`, `text` events and `openTag` attribute values arrive decoded. Raw content text, CDATA, comments, processing instructions and doctypes are passed through as written. See [entity decoding](./parsing.md#entity-decoding) for which entities each mode recognizes. + +```ts +const parser = createSaxParser({ entities: true }); +parser.on('openTag', (name, attributes) => console.log(attributes)); +parser.on('text', (text) => console.log(text)); +parser.write('Tom & Jerry'); +parser.close(); +// { title: 'A & B' } +// 'Tom & Jerry' +``` + With `html: true`, void elements like `
` fire both `openTag` and `closeTag`, and script and style content arrives as one raw `text` event: ```ts diff --git a/docs/web-streams.md b/docs/web-streams.md index 40c6735..e4f7a4a 100644 --- a/docs/web-streams.md +++ b/docs/web-streams.md @@ -61,6 +61,12 @@ Input can be split anywhere, mid-tag, mid-attribute, or mid-entity; the parser b false Include comments in the output (as strings like '<!-- ... -->', either emitted directly at the top level or attached as children) + + entities + boolean + false + Decode XML/HTML entities in text and attribute values, the same way parse() does + select string | string[] diff --git a/eksml/src/parser.ts b/eksml/src/parser.ts index 739fb86..19e34e1 100644 --- a/eksml/src/parser.ts +++ b/eksml/src/parser.ts @@ -782,6 +782,10 @@ export function parse( ? decodeHTML : decodeXML : null; + // Text is classified and trimmed on the source, then decoded, matching + // the SAX engine. decode of a string without '&' is identity. + const decodeText = (text: string): string => + decode !== null && text.indexOf('&') !== -1 ? decode(text) : text; const selfClosingTagList: string[] = resolvedOptions.selfClosingTags ?? (htmlMode ? HTML_VOID_ELEMENTS : []); @@ -1271,7 +1275,7 @@ export function parse( } else { // ---- Text (inline parseText) ---- const textStart = pos; - if (decode === null && !trimWhitespace) { + if (!trimWhitespace) { // Fused scan + classification: walk the (typically short) // leading whitespace in JS; the first non-ws char both settles // the classification and hands the rest of the scan to the @@ -1321,7 +1325,9 @@ export function parse( } if ((flags & HAS_TEXT) !== 0) { // Mixed content already established — push directly - children[children.length] = src.substring(textStart, textEnd); + children[children.length] = decodeText( + src.substring(textStart, textEnd), + ); } else if (wsOnly) { // Defer: record the range, allocate only if the container // turns out to keep its whitespace @@ -1333,7 +1339,9 @@ export function parse( } else { if (pendingCount > pendingBase) materializePending(S, children, pendingBase); - children[children.length] = src.substring(textStart, textEnd); + children[children.length] = decodeText( + src.substring(textStart, textEnd), + ); flags |= HAS_TEXT; } continue; @@ -1345,78 +1353,47 @@ export function parse( } else { pos = textEnd; } - if (decode === null) { - // No decoding, trimWhitespace mode (the non-trim case took the - // fused path above). - if (strict && depth === 0 && currentTagName === '') { - if (!isWsRange(textStart, textEnd)) { - throw strictError('Text content outside of a tag', textStart); - } + // trimWhitespace mode (the non-trim case took the fused path above). + if (strict && depth === 0 && currentTagName === '') { + if (!isWsRange(textStart, textEnd)) { + throw strictError('Text content outside of a tag', textStart); + } + } + // Fused trim: compute bounds on the source, substring once; + // ws-only runs allocate nothing. Falls back to substring+trim + // when a non-ASCII char sits at a boundary (Unicode whitespace). + let trimStart = textStart; + let trimLast = textEnd - 1; + let exact = true; + while (trimStart <= trimLast) { + const c = src.charCodeAt(trimStart); + if (c === 32 || (c >= 9 && c <= 13)) { + trimStart++; + continue; } - // Fused trim: compute bounds on the source, substring once; - // ws-only runs allocate nothing. Falls back to substring+trim - // when a non-ASCII char sits at a boundary (Unicode whitespace). - let trimStart = textStart; - let trimLast = textEnd - 1; - let exact = true; - while (trimStart <= trimLast) { - const c = src.charCodeAt(trimStart); + if (c >= 128) exact = false; + break; + } + if (exact && trimStart <= trimLast) { + while (trimLast > trimStart) { + const c = src.charCodeAt(trimLast); if (c === 32 || (c >= 9 && c <= 13)) { - trimStart++; + trimLast--; continue; } if (c >= 128) exact = false; break; } - if (exact && trimStart <= trimLast) { - while (trimLast > trimStart) { - const c = src.charCodeAt(trimLast); - if (c === 32 || (c >= 9 && c <= 13)) { - trimLast--; - continue; - } - if (c >= 128) exact = false; - break; - } - } - if (!exact) { - const trimmed = src.substring(textStart, textEnd).trim(); - if (trimmed.length > 0) { - children.push(trimmed); - flags |= HAS_TEXT; - } - } else if (trimStart <= trimLast) { - children.push(src.substring(trimStart, trimLast + 1)); + } + if (!exact) { + const trimmed = src.substring(textStart, textEnd).trim(); + if (trimmed.length > 0) { + children.push(decodeText(trimmed)); flags |= HAS_TEXT; } - } else { - // Entity decoding active: classification must look at the - // decoded string, so no deferral on this path. decode of a - // string without '&' is identity, so gate on indexOf. - let text = src.substring(textStart, textEnd); - if (text.indexOf('&') !== -1) text = decode(text); - if ( - strict && - depth === 0 && - currentTagName === '' && - text.trim().length > 0 - ) { - throw strictError('Text content outside of a tag', textStart); - } - if (trimWhitespace) { - const trimmed = text.trim(); - if (trimmed.length > 0) { - children.push(trimmed); - flags |= HAS_TEXT; - } - } else if (text.length > 0) { - if ((flags & HAS_TEXT) === 0) { - flags |= isWsString(text) - ? HAS_WS_TEXT | HAS_WS_IN_ARRAY - : HAS_TEXT; - } - children[children.length] = text; - } + } else if (trimStart <= trimLast) { + children.push(decodeText(src.substring(trimStart, trimLast + 1))); + flags |= HAS_TEXT; } } } diff --git a/eksml/src/sax.ts b/eksml/src/sax.ts index 289a84f..5fd7233 100644 --- a/eksml/src/sax.ts +++ b/eksml/src/sax.ts @@ -18,6 +18,7 @@ * ``` */ +import { decodeXML, decodeHTML } from 'entities'; import type { SaxEngineHandlers, Attributes } from '#src/saxEngine.ts'; import { saxEngine } from '#src/saxEngine.ts'; import { @@ -70,6 +71,21 @@ export interface SaxParserOptions { selfClosingTags?: string[]; /** Tag names whose content is raw text. Defaults to `['script', 'style']` when `html: true`. */ rawContentTags?: string[]; + /** + * Decode XML/HTML entities in text content and attribute values. + * When enabled, named entities (`&`, `<`, etc.), decimal character + * references (`ä`), and hex character references (`ä`) are decoded. + * + * In HTML mode (`html: true`), the full set of HTML named entities is + * supported (e.g. ` `, `©`, `—`). In XML mode, only the + * five standard XML entities plus numeric references are decoded. + * + * Text is trimmed before it is decoded, so encoded whitespace such as + * ` ` is kept. CDATA sections and raw content text are never decoded. + * + * Defaults to `false` — entities are preserved as-is in the output. + */ + entities?: boolean; } // --------------------------------------------------------------------------- @@ -82,7 +98,7 @@ export interface SaxParserOptions { * Uses the internal SAX engine, but wraps it with `.on()` / `.off()` methods * so handlers can be added and removed dynamically. * - * @param options - Parser options (html mode, selfClosingTags, rawContentTags). + * @param options - Parser options (html mode, selfClosingTags, rawContentTags, entities). * @returns A SaxParser with `.on()`, `.off()`, `.write()`, `.close()`. */ export function createSaxParser(options?: SaxParserOptions): SaxParser { @@ -208,6 +224,8 @@ export function createSaxParser(options?: SaxParserOptions): SaxParser { ...handlers, selfClosingTags, rawContentTags, + decode: + opts.entities === true ? (isHtml ? decodeHTML : decodeXML) : undefined, }); return { diff --git a/eksml/src/saxEngine.ts b/eksml/src/saxEngine.ts index d301d5d..3c064fc 100644 --- a/eksml/src/saxEngine.ts +++ b/eksml/src/saxEngine.ts @@ -70,6 +70,13 @@ export interface SaxEngineOptions extends SaxEngineHandlers { * this limit a `RangeError` is thrown. Default `undefined` (no limit). */ maxBufferSize?: number; + /** + * Decodes entities in text runs and attribute values. Applied after + * trimming, and only to strings containing `&`. Raw content text, CDATA, + * comments, processing instructions and doctypes are never decoded. + * Default `undefined` (entities are left as written). + */ + decode?: (text: string) => string; } /** The parser instance returned by `saxEngine()`. */ @@ -148,6 +155,7 @@ export function saxEngine(options: SaxEngineOptions = {}): SaxEngineParser { rawContentTags = [], maxBufferSize, } = options; + const decode = options.decode ?? null; const voidSet: Set | null = selfClosingTags.length > 0 ? new Set(selfClosingTags) : null; @@ -265,11 +273,30 @@ export function saxEngine(options: SaxEngineOptions = {}): SaxEngineParser { if (text.length === 0) return; if (onText && !textSuppressed) { const trimmed = trimWhitespace(text); - if (trimmed.length > 0) onText(trimmed); + if (trimmed.length > 0) + onText( + decode !== null && trimmed.indexOf('&') !== -1 + ? decode(trimmed) + : trimmed, + ); } text = ''; } + /** Decode attribute values in place. */ + function decodeAttributes( + attrs: Attributes, + decodeValue: (text: string) => string, + ): void { + // for-in avoids allocating a key array per tag; attribute records are + // plain objects with no inherited enumerable keys. + for (const name in attrs) { + const value = attrs[name]; + if (value != null && value.indexOf('&') !== -1) + setOwnProperty(attrs, name, decodeValue(value)); + } + } + /** * Parse the accumulated DOCTYPE body (everything between ``, * excluding internal DTD subsets) and emit an onDoctype event. @@ -345,7 +372,10 @@ export function saxEngine(options: SaxEngineOptions = {}): SaxEngineParser { /** After we finish parsing an open tag's `>`, handle void/raw transitions. */ function finishOpenTag(): void { - if (onOpenTag) onOpenTag(tagName, attributes); + if (onOpenTag) { + if (decode !== null) decodeAttributes(attributes, decode); + onOpenTag(tagName, attributes); + } if (voidSet !== null && voidSet.has(tagName)) { if (onCloseTag) onCloseTag(tagName); } else if (rawSet !== null && rawSet.has(tagName)) { @@ -429,7 +459,12 @@ export function saxEngine(options: SaxEngineOptions = {}): SaxEngineParser { if (c !== SPACE && c !== TAB && c !== LF && c !== CR) break; e--; } - handleText(chunk.substring(s, e + 1)); + const run = chunk.substring(s, e + 1); + handleText( + decode !== null && run.indexOf('&') !== -1 + ? decode(run) + : run, + ); } } } else { @@ -703,7 +738,11 @@ export function saxEngine(options: SaxEngineOptions = {}): SaxEngineParser { } // --- emit open tag --- - if (handleOpenTag !== undefined) handleOpenTag(name, attrs!); + if (handleOpenTag !== undefined) { + if (decode !== null && attrs !== null) + decodeAttributes(attrs, decode); + handleOpenTag(name, attrs!); + } if (selfClosed) { if (handleCloseTag !== undefined) handleCloseTag(name); continue fastPath; @@ -1020,7 +1059,10 @@ export function saxEngine(options: SaxEngineOptions = {}): SaxEngineParser { if (charCode === GT) { state = State.TEXT; i++; - if (onOpenTag) onOpenTag(tagName, attributes); + if (onOpenTag) { + if (decode !== null) decodeAttributes(attributes, decode); + onOpenTag(tagName, attributes); + } if (onCloseTag) onCloseTag(tagName); } else { state = State.OPEN_TAG_BODY; diff --git a/eksml/src/xmlParseStream.ts b/eksml/src/xmlParseStream.ts index 58989b8..e4097a2 100644 --- a/eksml/src/xmlParseStream.ts +++ b/eksml/src/xmlParseStream.ts @@ -1,3 +1,4 @@ +import { decodeXML, decodeHTML } from 'entities'; import type { TNode } from '#src/parser.ts'; import { saxEngine } from '#src/saxEngine.ts'; import type { Attributes } from '#src/saxEngine.ts'; @@ -19,8 +20,8 @@ import type { LosslessEntry } from '#src/converters/lossless.ts'; * Options for `XmlParseStream`. * * Only the fields that the transform layer actually consumes are exposed here. - * This keeps the API honest — options like `trimWhitespace`, `entities`, and - * `strict` belong to the synchronous `parse()` function, not the streaming + * This keeps the API honest — options like `trimWhitespace` and `strict` + * belong to the synchronous `parse()` function, not the streaming * tree-builder. */ export interface XmlParseStreamOptions { @@ -43,6 +44,21 @@ export interface XmlParseStreamOptions { rawContentTags?: string[]; /** Keep XML comments in the output. Defaults to `false`. */ keepComments?: boolean; + /** + * Decode XML/HTML entities in text content and attribute values. + * When enabled, named entities (`&`, `<`, etc.), decimal character + * references (`ä`), and hex character references (`ä`) are decoded. + * + * In HTML mode (`html: true`), the full set of HTML named entities is + * supported (e.g. ` `, `©`, `—`). In XML mode, only the + * five standard XML entities plus numeric references are decoded. + * + * Text is trimmed before it is decoded, so encoded whitespace such as + * ` ` is kept. CDATA sections and raw content text are never decoded. + * + * Defaults to `false` — entities are preserved as-is in the output. + */ + entities?: boolean; /** * Emit only elements matching these tag names instead of waiting for the * entire top-level tree to close. @@ -328,6 +344,12 @@ export class XmlParseStream extends TransformStream< const rawContentTags = resolvedOptions.rawContentTags ?? (isHtml ? HTML_RAW_CONTENT_TAGS : []); const keepComments = resolvedOptions.keepComments === true; + const decode = + resolvedOptions.entities === true + ? isHtml + ? decodeHTML + : decodeXML + : undefined; // Resolve select into a Set for O(1) lookup (null = emit at depth 0) const selectSet: Set | null = @@ -585,6 +607,7 @@ export class XmlParseStream extends TransformStream< const parser = saxEngine({ selfClosingTags, rawContentTags, + decode, onOpenTag: selectSet ? selectOnOpenTag : defaultOnOpenTag, onCloseTag: selectSet ? selectOnCloseTag : defaultOnCloseTag, onText: selectSet ? selectOnText : defaultOnText, diff --git a/eksml/test/entities.test.ts b/eksml/test/entities.test.ts new file mode 100644 index 0000000..90f8e46 --- /dev/null +++ b/eksml/test/entities.test.ts @@ -0,0 +1,325 @@ +import { readFileSync, readdirSync } from 'node:fs'; +import { resolve, dirname } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { describe, it, expect } from 'vitest'; +import { parse, type TNode } from '#src/parser.ts'; +import { createSaxParser, type SaxParserOptions } from '#src/sax.ts'; +import { + XmlParseStream, + type XmlParseStreamOptions, +} from '#src/xmlParseStream.ts'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); + +/** Feed chunks through createSaxParser and record every event in order. */ +function saxEvents(chunks: string[], options?: SaxParserOptions): unknown[] { + const events: unknown[] = []; + const sax = createSaxParser(options); + sax.on('openTag', (name, attributes) => + events.push(['openTag', name, { ...attributes }]), + ); + sax.on('closeTag', (name) => events.push(['closeTag', name])); + sax.on('text', (text) => events.push(['text', text])); + sax.on('cdata', (data) => events.push(['cdata', data])); + for (const chunk of chunks) sax.write(chunk); + sax.close(); + return events; +} + +/** Build a TNode tree from SAX events, the same shape parse() produces. */ +function saxTree(chunks: string[], options?: SaxParserOptions): TNode[] { + const roots: TNode[] = []; + const stack: TNode[] = []; + const sax = createSaxParser(options); + sax.on('openTag', (tagName, attributes) => { + const node: TNode = { + tagName, + attributes: Object.keys(attributes).length ? attributes : null, + children: [], + }; + if (stack.length) stack[stack.length - 1]!.children.push(node); + else roots.push(node); + stack.push(node); + }); + sax.on('closeTag', () => stack.pop()); + const onText = (text: string) => { + if (stack.length) stack[stack.length - 1]!.children.push(text); + }; + sax.on('text', onText); + sax.on('cdata', onText); + for (const chunk of chunks) sax.write(chunk); + sax.close(); + return roots; +} + +async function streamNodes( + chunks: string[], + options?: XmlParseStreamOptions, +): Promise<(TNode | string)[]> { + const stream = new XmlParseStream(options ?? {}); + const writer = stream.writable.getWriter(); + const reader = stream.readable.getReader(); + const results: (TNode | string)[] = []; + const readAll = (async () => { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + results.push(value); + } + })(); + for (const chunk of chunks) await writer.write(chunk); + await writer.close(); + await readAll; + return results; +} + +/** Split a string into chunks of a fixed size. */ +function chunked(input: string, size: number): string[] { + const chunks: string[] = []; + for (let i = 0; i < input.length; i += size) { + chunks.push(input.slice(i, i + size)); + } + return chunks; +} + +describe('entities option', () => { + describe('createSaxParser()', () => { + it('leaves entities as written by default', () => { + expect(saxEvents(['Tom & Jerry'])).toEqual([ + ['openTag', 'a', { t: 'x & y' }], + ['text', 'Tom & Jerry'], + ['closeTag', 'a'], + ]); + }); + + it('decodes text and attribute values', () => { + expect( + saxEvents( + ['Tom & Jerry ä ä'], + { + entities: true, + }, + ), + ).toEqual([ + ['openTag', 'a', { t: ' "q"' }], + ['text', 'Tom & Jerry ä ä'], + ['closeTag', 'a'], + ]); + }); + + it('decodes unquoted and single-quoted attribute values', () => { + expect( + saxEvents([""], { entities: true }), + ).toEqual([ + ['openTag', 'a', { s: 'x & y', u: 'a&b' }], + ['closeTag', 'a'], + ]); + }); + + it('decodes an entity split across chunks', () => { + expect( + saxEvents(['Tom &a', 'mp; Jerry'], { + entities: true, + }), + ).toEqual([ + ['openTag', 'a', { t: 'x & y' }], + ['text', 'Tom & Jerry'], + ['closeTag', 'a'], + ]); + }); + + it('never decodes CDATA', () => { + expect( + saxEvents([''], { entities: true }), + ).toEqual([ + ['openTag', 'a', {}], + ['cdata', 'x & y'], + ['closeTag', 'a'], + ]); + }); + + it('never decodes raw content', () => { + expect( + saxEvents([''], { + html: true, + entities: true, + }), + ).toEqual([ + ['openTag', 'script', {}], + ['text', 'a && b'], + ['closeTag', 'script'], + ]); + }); + + it('decodes HTML named entities only in HTML mode', () => { + expect(saxEvents(['

© &

'], { entities: true })).toEqual([ + ['openTag', 'p', {}], + ['text', '© &'], + ['closeTag', 'p'], + ]); + expect( + saxEvents(['

© &

'], { entities: true, html: true }), + ).toEqual([ + ['openTag', 'p', {}], + ['text', '© &'], + ['closeTag', 'p'], + ]); + }); + + it('trims text before decoding it', () => { + expect( + saxEvents([' x '], { entities: true }), + ).toEqual([ + ['openTag', 'a', {}], + ['text', ' x '], + ['closeTag', 'a'], + ['openTag', 'b', {}], + ['text', ' '], + ['closeTag', 'b'], + ]); + }); + }); + + describe('XmlParseStream', () => { + it('leaves entities as written by default', async () => { + expect( + await streamNodes(['Tom & Jerry']), + ).toEqual([ + { + tagName: 'a', + attributes: { t: 'x & y' }, + children: ['Tom & Jerry'], + }, + ]); + }); + + it('decodes text and attribute values', async () => { + expect( + await streamNodes(['Tom & Jerry'], { + entities: true, + }), + ).toEqual([ + { tagName: 'a', attributes: { t: 'x & y' }, children: ['Tom & Jerry'] }, + ]); + }); + + it('decodes inside selected subtrees split across chunks', async () => { + expect( + await streamNodes( + [ + 'Tom &am', + 'p; Jerry', + ], + { entities: true, select: 'programme' }, + ), + ).toEqual([ + { + tagName: 'programme', + attributes: { channel: 'a&b' }, + children: [ + { tagName: 'title', attributes: null, children: ['Tom & Jerry'] }, + ], + }, + ]); + }); + + it('never decodes CDATA or raw content', async () => { + expect( + await streamNodes( + ['
'], + { entities: true, html: true }, + ), + ).toEqual([ + { + tagName: 'div', + attributes: null, + children: [ + 'x & y', + { + tagName: 'script', + attributes: null, + children: ['a && b'], + }, + ], + }, + ]); + }); + + it('decodes before converting to lossy output', async () => { + expect( + await streamNodes(['Tom & Jerry'], { + entities: true, + output: 'lossy', + }), + ).toEqual([{ a: 'Tom & Jerry' }]); + }); + }); + + describe('parse()', () => { + it('trims text before decoding it', () => { + expect( + parse(' x ', { + entities: true, + trimWhitespace: true, + }), + ).toEqual([ + { + tagName: 'r', + attributes: null, + children: [ + { tagName: 'a', attributes: null, children: [' x '] }, + { tagName: 'b', attributes: null, children: [' '] }, + ], + }, + ]); + }); + + it('treats encoded whitespace as text, not as whitespace to drop', () => { + expect(parse('\n \n', { entities: true })).toEqual([ + { + tagName: 'r', + attributes: null, + children: [{ tagName: 'a', attributes: null, children: [' '] }], + }, + ]); + }); + + it('decodes mixed content without trimming it', () => { + expect( + parse('

Tom & Jerry ä

', { entities: true }), + ).toEqual([ + { + tagName: 'p', + attributes: null, + children: [ + 'Tom & ', + { tagName: 'b', attributes: null, children: ['Jerry'] }, + ' ä ', + ], + }, + ]); + }); + }); + + describe('createSaxParser() matches parse()', () => { + const fixtures = readdirSync(resolve(__dirname, 'fixtures')).filter( + (name) => name.endsWith('.xml'), + ); + + it.each(fixtures)('%s', (name) => { + const xml = readFileSync(resolve(__dirname, 'fixtures', name), 'utf-8'); + const elementsOnly = (nodes: (TNode | string)[]) => + nodes.filter((node): node is TNode => typeof node === 'object'); + const expected = elementsOnly( + parse(xml, { entities: true, trimWhitespace: true }), + ).filter( + (node) => + !node.tagName.startsWith('?') && !node.tagName.startsWith('!'), + ); + + expect(saxTree([xml], { entities: true })).toEqual(expected); + expect(saxTree(chunked(xml, 7), { entities: true })).toEqual(expected); + }); + }); +});