diff --git a/CHANGELOG.md b/CHANGELOG.md index 040ff138..e68f8af2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +* šŸ› Preserve inline and image child edits when converting a `SyntaxTreeNode` back to tokens. * ✨ Add `--enable-tables` to the CLI for file, standard input and interactive parsing in [#422](https://github.com/executablebooks/markdown-it-py/pull/422) * šŸ› Fix CLI interactive mode joining input lines with an extra newline, which split every line into its own paragraph and broke hard line breaks, in [#172](https://github.com/executablebooks/markdown-it-py/issues/172) * šŸ› Fix trimming and splitting with the Python whitespace set instead of the CommonMark one, which dropped U+001C–U+001F and U+0085 from paragraphs, headings, table cells and fence info strings and let distinct reference labels resolve each other, in [#418](https://github.com/executablebooks/markdown-it-py/pull/418), thanks to [@Nexory](https://github.com/Nexory) diff --git a/docs/using.md b/docs/using.md index aae3e858..7cccab53 100644 --- a/docs/using.md +++ b/docs/using.md @@ -286,6 +286,10 @@ print(node[0]) node[0].next_sibling ``` +Use `node.to_tokens()` to recover the linear token stream after modifying the +tree's `children`, including those of inline and image nodes. This reuses the +original token objects and updates their child lists to match the tree. + ## Renderers +++ diff --git a/markdown_it/tree.py b/markdown_it/tree.py index ebe00d23..d436bbe6 100644 --- a/markdown_it/tree.py +++ b/markdown_it/tree.py @@ -91,13 +91,22 @@ def __getitem__(self, item: int | slice) -> Self | list[Self]: return self.children[item] def to_tokens(self) -> list[Token]: - """Recover the linear token stream.""" + """Recover the linear token stream, including changes to the tree. + + The original tokens are reused, with their child lists updated to match + the tree. + """ def recursive_collect_tokens(node: Self, token_list: list[Token]) -> None: if node.type == "root": for child in node.children: recursive_collect_tokens(child, token_list) elif node.token: + if node.token.children is not None: + children: list[Token] = [] + for child in node.children: + recursive_collect_tokens(child, children) + node.token.children = children token_list.append(node.token) else: assert node.nester_tokens diff --git a/tests/test_tree.py b/tests/test_tree.py index 36bd0b67..7e5d5108 100644 --- a/tests/test_tree.py +++ b/tests/test_tree.py @@ -8,10 +8,71 @@ """ -def test_tree_to_tokens_conversion(): +def test_tree_to_tokens_conversion() -> None: tokens = MarkdownIt().parse(EXAMPLE_MARKDOWN) + serialized_tokens = [token.as_dict() for token in tokens] tokens_after_roundtrip = SyntaxTreeNode(tokens).to_tokens() assert tokens == tokens_after_roundtrip + assert [token.as_dict() for token in tokens_after_roundtrip] == serialized_tokens + assert all( + before is after + for before, after in zip(tokens, tokens_after_roundtrip, strict=True) + ) + + +def test_tree_to_tokens_reordered_children() -> None: + cases: list[tuple[str, tuple[int, ...], str]] = [ + ("one\n\ntwo", (), "

two

\n

one

\n"), + ( + "before *nested* after", + (0, 0), + "

afternestedbefore

\n", + ), + ( + "![before *nested* after](image.png)", + (0, 0, 0), + '

 afternestedbefore

\n', + ), + ] + for source, path, expected in cases: + md = MarkdownIt() + tree = SyntaxTreeNode(md.parse(source)) + node = tree + for index in path: + node = node[index] + node.children.reverse() + + assert md.renderer.render(tree.to_tokens(), md.options, {}) == expected + + +def test_tree_to_tokens_cleared_children() -> None: + cases: list[tuple[str, tuple[int, ...], str]] = [ + ("before *nested* after", (0, 0), "

\n"), + ("before *nested* after", (0, 0, 1), "

before after

\n"), + ( + "![old alt](image.png)", + (0, 0, 0), + '

\n', + ), + ] + for source, path, expected in cases: + md = MarkdownIt() + tree = SyntaxTreeNode(md.parse(source)) + node = tree + for index in path: + node = node[index] + node.children.clear() + + assert md.renderer.render(tree.to_tokens(), md.options, {}) == expected + + +def test_tree_to_tokens_replaced_empty_inline_children() -> None: + md = MarkdownIt() + tree = SyntaxTreeNode(md.parseInline("")) + replacement = SyntaxTreeNode(md.parseInline("*new*")) + tree[0].children = replacement[0].children + + assert md.renderer.render(tree.to_tokens(), md.options, {}) == "new" def test_property_passthrough():