diff --git a/test_unstructured/partition/test_docx_ruby.py b/test_unstructured/partition/test_docx_ruby.py new file mode 100644 index 0000000000..65c85fa3ae --- /dev/null +++ b/test_unstructured/partition/test_docx_ruby.py @@ -0,0 +1,77 @@ +import io + +import docx +import pytest +from docx.oxml import OxmlElement + +from unstructured.partition.docx import partition_docx + + +def _ruby(base_texts=("漢", "字")): + ruby = OxmlElement("w:ruby") + pronunciation = OxmlElement("w:rt") + ruby.append(pronunciation) + run = OxmlElement("w:r") + pronunciation.append(run) + text = OxmlElement("w:t") + text.text = "かんじ" + run.append(text) + base = OxmlElement("w:rubyBase") + ruby.append(base) + for value in base_texts: + run = OxmlElement("w:r") + base.append(run) + text = OxmlElement("w:t") + text.text = value + run.append(text) + return ruby + + +@pytest.mark.parametrize("location", ["body", "table", "header", "footer", "hyperlink"]) +def test_ruby_base_is_preserved_in_document_order_without_phonetic_annotation(location): + document = docx.Document() + if location == "table": + paragraph = document.add_table(rows=1, cols=1).cell(0, 0).paragraphs[0] + elif location in ("header", "footer"): + paragraph = getattr(document.sections[0], location).paragraphs[0] + else: + paragraph = document.add_paragraph() + run = paragraph.add_run("Before ") + run._r.append(_ruby()) + run.add_text(" after") + if location == "hyperlink": + link = OxmlElement("w:hyperlink") + paragraph._p.append(link) + link.append(run._r) + file = io.BytesIO() + document.save(file) + original = file.getvalue() + + elements = partition_docx(file=file, languages=["eng"]) + + assert [element.text for element in elements] == ["Before 漢字 after"] + assert file.getvalue() == original + if location == "table": + assert "漢字" in elements[0].metadata.text_as_html + assert "かんじ" not in elements[0].metadata.text_as_html + + +def test_paragraph_containing_only_ruby_is_not_dropped(): + document = docx.Document() + document.add_paragraph().add_run()._r.append(_ruby()) + file = io.BytesIO() + document.save(file) + + assert [element.text for element in partition_docx(file=file, languages=["eng"])] == ["漢字"] + + +def test_ruby_base_preserves_embedded_tabs(): + document = docx.Document() + run = document.add_paragraph().add_run() + ruby = _ruby(("漢", "字")) + ruby.find(docx.oxml.ns.qn("w:rubyBase"))[0].append(OxmlElement("w:tab")) + run._r.append(ruby) + file = io.BytesIO() + document.save(file) + + assert partition_docx(file=file, languages=["eng"])[0].text == "漢\t字" diff --git a/unstructured/partition/docx.py b/unstructured/partition/docx.py index c55651824e..33760b441a 100644 --- a/unstructured/partition/docx.py +++ b/unstructured/partition/docx.py @@ -256,6 +256,7 @@ def document(self) -> Document: for part in document.part.package.iter_parts(): if isinstance(part, (DocumentPart, HeaderPart, FooterPart)): _unwrap_nested_content(part.element) + _unwrap_ruby_base(part.element) return document @cached_property @@ -1338,6 +1339,26 @@ def _unwrap_nested_content(root: BaseOxmlElement) -> None: parent[index : index + 1] = children +def _unwrap_ruby_base(root: BaseOxmlElement) -> None: + """Keep the visible base of East Asian ruby text where python-docx can read it. + + A run's ``w:ruby`` contains pronunciation in ``w:rt`` and the corresponding document + text in ``w:rubyBase``. python-docx ignores both. Lift the base runs' content into the + enclosing run, retaining inline text, tabs and breaks without duplicating pronunciation. + The enclosing run's formatting applies; separate base-run formatting is not resolved. + """ + for ruby in reversed(list(root.iter(_W + "ruby"))): + parent = ruby.getparent() + base = ruby.find(_W + "rubyBase") + if parent is None or base is None: + continue + children = [ + child for run in base.findall(_W + "r") for child in run if child.tag != _W + "rPr" + ] + index = parent.index(ruby) + parent[index : index + 1] = children + + def _content_control_children(sdt: BaseOxmlElement) -> list[BaseOxmlElement] | None: """The elements to put in place of content control `sdt`, or None to leave it as it is.