Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions test_unstructured/partition/html/test_parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -1534,14 +1534,14 @@ class DescribeMath:
"""

@pytest.mark.parametrize("wrapper", ["{}", "<font size='2'>{}</font>", "<my-card>{}</my-card>"])
def it_skips_mathml_that_has_no_text_representation_but_not_its_tail(self, wrapper: str):
def it_preserves_presentation_mathml_and_its_tail(self, wrapper: str):
html = "<div>Before {} after.</div>".format(
wrapper.format("<math><mi>x</mi><mo>=</mo><mn>2</mn></math>")
)
div = etree.fromstring(html, html_parser).xpath(".//div")[0]

assert isinstance(div.xpath(".//math")[0], Math)
assert [e.text for e in div.iter_elements()] == ["Before after."]
assert [e.text for e in div.iter_elements()] == ["Before x=2 after."]


class DescribeRemovedPhrasing:
Expand Down
4 changes: 2 additions & 2 deletions test_unstructured/partition/html/test_partition.py
Original file line number Diff line number Diff line change
Expand Up @@ -894,7 +894,7 @@ def test_partition_html_keeps_katex_html_of_unusable_mathml_inside_an_unrecogniz
wrapper.format(
"Before"
' <span class="katex">'
'<span class="katex-mathml"><math><semantics><mi>x</mi></semantics></math></span>'
'<span class="katex-mathml"><math><maction><mi>x</mi></maction></math></span>'
'<span class="katex-html" aria-hidden="true">'
'<span class="base"><span>x</span><span>2</span></span>'
"</span></span>"
Expand Down Expand Up @@ -2223,7 +2223,7 @@ def test_partition_html_keeps_katex_html_when_mathml_is_unusable():
<span class="katex">
<span class="katex-mathml">
<math>
<semantics><mi>x</mi></semantics>
<maction><mi>x</mi></maction>
</math>
</span>
<span class="katex-html" aria-hidden="true">
Expand Down
74 changes: 74 additions & 0 deletions test_unstructured/partition/html/test_presentation_mathml.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
import pytest

from unstructured.partition.html import partition_html


@pytest.mark.parametrize(
("expression", "expected"),
[
("<mi>x</mi><mo>+</mo><mn>2</mn>", "x+2"),
("<mfrac><mi>a</mi><mi>b</mi></mfrac>", r"\frac{a}{b}"),
(
"<mfrac><mrow><mn>1</mn><mo>+</mo><msqrt><mn>5</mn></msqrt></mrow><mn>2</mn></mfrac>",
r"\frac{1+\sqrt{5}}{2}",
),
("<mroot><mi>x</mi><mn>3</mn></mroot>", r"\sqrt[3]{x}"),
("<msup><mi>x</mi><mn>2</mn></msup>", r"{x}^{2}"),
("<msub><mi>x</mi><mi>i</mi></msub>", r"{x}_{i}"),
("<msubsup><mi>x</mi><mi>i</mi><mn>2</mn></msubsup>", r"{x}_{i}^{2}"),
("<mover><mi>x</mi><mo>¯</mo></mover>", r"\overset{¯}{x}"),
("<munder><mo>lim</mo><mi>x</mi></munder>", r"\underset{x}{lim}"),
(
"<munderover><mo>∑</mo><mi>i</mi><mi>n</mi></munderover>",
r"\overset{n}{\underset{i}{∑}}",
),
('<mfenced open="[" close="]" separators=";"><mi>x</mi><mi>y</mi></mfenced>', "[x;y]"),
(
"<mtable><mtr><mtd><mn>1</mn></mtd><mtd><mn>2</mn></mtd></mtr>"
"<mtr><mtd><mn>3</mn></mtd><mtd><mn>4</mn></mtd></mtr></mtable>",
r"\begin{matrix}1 & 2 \\ 3 & 4\end{matrix}",
),
("<mi>x</mi><mo>=</mo><mtext>50% &amp; rising</mtext>", r"x=\text{50\% \& rising}"),
(
"<semantics><msup><mi>x</mi><mn>2</mn></msup>"
'<annotation encoding="application/other">do not duplicate</annotation></semantics>',
r"{x}^{2}",
),
],
)
def test_presentation_mathml_is_retained_without_a_tex_annotation(expression, expected):
elements = partition_html(
text=f"<p>Value <math>{expression}</math> remains.</p>", languages=["eng"]
)

assert [element.text for element in elements] == [f"Value {expected} remains."]


def test_presentation_mathml_preserves_display_boundary_and_page_number():
html = (
'<div data-page-number="3">Before<math display="block">'
"<mfrac><mi>x</mi><mn>2</mn></mfrac></math>After</div>"
)

elements = partition_html(text=html, languages=["eng"])

assert [element.text for element in elements] == ["Before", r"\frac{x}{2}", "After"]
assert [element.metadata.page_number for element in elements] == [3, 3, 3]


@pytest.mark.parametrize(
"expression",
[
"<mfrac><mi>x</mi></mfrac>",
"<mrow><mi>x</mi><maction><mi>y</mi></maction></mrow>",
'<mfrac linethickness="0"><mi>x</mi><mi>y</mi></mfrac>',
],
)
def test_unsupported_or_malformed_math_keeps_existing_visual_fallback(expression):
html = (
'<p><span class="katex"><span class="katex-mathml">'
f"<math>{expression}</math></span>"
'<span class="katex-html">Visual fallback</span></span></p>'
)

assert partition_html(text=html, languages=["eng"])[0].text == "Visual fallback"
100 changes: 100 additions & 0 deletions unstructured/partition/html/mathml.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
"""A structural LaTeX fallback for common presentation MathML without a TeX annotation."""

from lxml import etree


def _escape(text: str) -> str:
escapes = {"\\": r"\backslash{}", "^": r"\^{}", "~": r"\~{}"}
escapes.update({char: "\\" + char for char in "{}_%&#$"})
return "".join(escapes.get(char, char) for char in text)


def presentation_to_latex(node: etree._Element) -> str | None:
"""Render supported presentation constructs, or return None for an unsupported expression.

This is a fallback for extraction, not a MathML layout engine. Unknown nodes, malformed
arity and unhandled fraction-line semantics reject the whole expression so a paired visual
fallback remains available. Existing TeX annotations and ``alttext`` take precedence.
"""
tag = node.tag
if tag in {"mi", "mn", "mo", "mtext"}:
if len(node):
return None
text = _escape(node.text or "")
return rf"\text{{{text}}}" if tag == "mtext" else text.strip()
if tag == "mspace":
return " "
if tag == "semantics":
children = [child for child in node if child.tag not in {"annotation", "annotation-xml"}]
if len(children) != 1 or (node.text or "").strip():
return None
return presentation_to_latex(children[0])

supported = {
"math",
"mrow",
"mstyle",
"mpadded",
"mtd",
"mfrac",
"msqrt",
"mroot",
"msup",
"msub",
"msubsup",
"mover",
"munder",
"munderover",
"mfenced",
"mtable",
"mtr",
}
if tag not in supported:
return None
if (node.text or "").strip() or any((child.tail or "").strip() for child in node):
return None
parts = []
for child in node:
part = presentation_to_latex(child)
if part is None:
return None
parts.append(part)
if tag in {"math", "mrow", "mstyle", "mpadded", "mtd"}:
return "".join(parts)
if tag == "msqrt":
return rf"\sqrt{{{''.join(parts)}}}"
if tag == "mfenced":
separators = "".join(node.get("separators", ",").split())
inside = "".join(
(_escape(separators[min(i - 1, len(separators) - 1)]) if i and separators else "")
+ part
for i, part in enumerate(parts)
)
return _escape(node.get("open", "(")) + inside + _escape(node.get("close", ")"))
if tag == "mtr":
return " & ".join(parts) if all(child.tag == "mtd" for child in node) else None
if tag == "mtable":
if any(child.tag != "mtr" for child in node):
return None
return r"\begin{matrix}" + r" \\ ".join(parts) + r"\end{matrix}"
expected_arity = 3 if tag in {"msubsup", "munderover"} else 2
if len(parts) != expected_arity:
return None
base, second = parts[:2]
if tag == "mfrac":
if node.get("linethickness") is not None:
return None
return rf"\frac{{{base}}}{{{second}}}"
if tag == "mroot":
return rf"\sqrt[{second}]{{{base}}}"
if tag == "msup":
return f"{{{base}}}^{{{second}}}"
if tag == "msub":
return f"{{{base}}}_{{{second}}}"
if tag == "msubsup":
return f"{{{base}}}_{{{second}}}^{{{parts[2]}}}"
if tag == "mover":
return rf"\overset{{{second}}}{{{base}}}"
if tag == "munder":
return rf"\underset{{{second}}}{{{base}}}"
return rf"\overset{{{parts[2]}}}{{\underset{{{second}}}{{{base}}}}}"
3 changes: 2 additions & 1 deletion unstructured/partition/html/parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,7 @@
Title,
)
from unstructured.partition.common.metadata import category_depth_from_html_tag
from unstructured.partition.html.mathml import presentation_to_latex
from unstructured.partition.text_type import (
is_bulleted_text,
is_email_address,
Expand Down Expand Up @@ -798,7 +799,7 @@ def _latex(self) -> str | None:
if alt_text is not None and alt_text.strip():
return alt_text

return None
return presentation_to_latex(self)

@cached_property
def _page_number(self) -> int | None:
Expand Down