🐛 Preserve inline child edits when converting syntax trees to tokens

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## Unreleased
 
+* 🐛 Preserve inline and image child edits when converting a `SyntaxTreeNode` back to tokens.
 * ✨ Add `--enable-tables` to the CLI for file, standard input and interactive parsing in [#422](https://github.com/executablebooks/markdown-it-py/pull/422)
 * 🐛 Fix CLI interactive mode joining input lines with an extra newline, which split every line into its own paragraph and broke hard line breaks, in [#172](https://github.com/executablebooks/markdown-it-py/issues/172)
 * 🐛 Fix trimming and splitting with the Python whitespace set instead of the CommonMark one, which dropped U+001C–U+001F and U+0085 from paragraphs, headings, table cells and fence info strings and let distinct reference labels resolve each other, in [#418](https://github.com/executablebooks/markdown-it-py/pull/418), thanks to [@Nexory](https://github.com/Nexory)

docs/using.md

@@ -286,6 +286,10 @@ print(node[0])
 node[0].next_sibling
 ```
 
+Use `node.to_tokens()` to recover the linear token stream after modifying the
+tree's `children`, including those of inline and image nodes. This reuses the
+original token objects and updates their child lists to match the tree.
+
 ## Renderers
 
 +++

markdown_it/tree.py

@@ -91,13 +91,22 @@ def __getitem__(self, item: int | slice) -> Self | list[Self]:
         return self.children[item]
 
     def to_tokens(self) -> list[Token]:
-        """Recover the linear token stream."""
+        """Recover the linear token stream, including changes to the tree.
+
+        The original tokens are reused, with their child lists updated to match
+        the tree.
+        """
 
         def recursive_collect_tokens(node: Self, token_list: list[Token]) -> None:
             if node.type == "root":
                 for child in node.children:
                     recursive_collect_tokens(child, token_list)
             elif node.token:
+                if node.token.children is not None:
+                    children: list[Token] = []
+                    for child in node.children:
+                        recursive_collect_tokens(child, children)
+                    node.token.children = children
                 token_list.append(node.token)
             else:
                 assert node.nester_tokens

tests/test_tree.py

@@ -8,10 +8,71 @@
 """
 
 
-def test_tree_to_tokens_conversion():
+def test_tree_to_tokens_conversion() -> None:
     tokens = MarkdownIt().parse(EXAMPLE_MARKDOWN)
+    serialized_tokens = [token.as_dict() for token in tokens]
     tokens_after_roundtrip = SyntaxTreeNode(tokens).to_tokens()
     assert tokens == tokens_after_roundtrip
+    assert [token.as_dict() for token in tokens_after_roundtrip] == serialized_tokens
+    assert all(
+        before is after
+        for before, after in zip(tokens, tokens_after_roundtrip, strict=True)
+    )
+
+
+def test_tree_to_tokens_reordered_children() -> None:
+    cases: list[tuple[str, tuple[int, ...], str]] = [
+        ("one\n\ntwo", (), "<p>two</p>\n<p>one</p>\n"),
+        (
+            "before *nested* after",
+            (0, 0),
+            "<p> after<em>nested</em>before </p>\n",
+        ),
+        (
+            "![before *nested* after](image.png)",
+            (0, 0, 0),
+            '<p><img src="image.png" alt=" afternestedbefore " /></p>\n',
+        ),
+    ]
+    for source, path, expected in cases:
+        md = MarkdownIt()
+        tree = SyntaxTreeNode(md.parse(source))
+        node = tree
+        for index in path:
+            node = node[index]
+        node.children.reverse()
+
+        assert md.renderer.render(tree.to_tokens(), md.options, {}) == expected
+
+
+def test_tree_to_tokens_cleared_children() -> None:
+    cases: list[tuple[str, tuple[int, ...], str]] = [
+        ("before *nested* after", (0, 0), "<p></p>\n"),
+        ("before *nested* after", (0, 0, 1), "<p>before <em></em> after</p>\n"),
+        (
+            "![old alt](image.png)",
+            (0, 0, 0),
+            '<p><img src="image.png" alt="" /></p>\n',
+        ),
+    ]
+    for source, path, expected in cases:
+        md = MarkdownIt()
+        tree = SyntaxTreeNode(md.parse(source))
+        node = tree
+        for index in path:
+            node = node[index]
+        node.children.clear()
+
+        assert md.renderer.render(tree.to_tokens(), md.options, {}) == expected
+
+
+def test_tree_to_tokens_replaced_empty_inline_children() -> None:
+    md = MarkdownIt()
+    tree = SyntaxTreeNode(md.parseInline(""))
+    replacement = SyntaxTreeNode(md.parseInline("*new*"))
+    tree[0].children = replacement[0].children
+
+    assert md.renderer.render(tree.to_tokens(), md.options, {}) == "<em>new</em>"
 
 
 def test_property_passthrough():