Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
77 changes: 77 additions & 0 deletions test_unstructured/partition/test_docx_ruby.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
import io

import docx
import pytest
from docx.oxml import OxmlElement

from unstructured.partition.docx import partition_docx


def _ruby(base_texts=("漢", "字")):
ruby = OxmlElement("w:ruby")
pronunciation = OxmlElement("w:rt")
ruby.append(pronunciation)
run = OxmlElement("w:r")
pronunciation.append(run)
text = OxmlElement("w:t")
text.text = "かんじ"
run.append(text)
base = OxmlElement("w:rubyBase")
ruby.append(base)
for value in base_texts:
run = OxmlElement("w:r")
base.append(run)
text = OxmlElement("w:t")
text.text = value
run.append(text)
return ruby


@pytest.mark.parametrize("location", ["body", "table", "header", "footer", "hyperlink"])
def test_ruby_base_is_preserved_in_document_order_without_phonetic_annotation(location):
document = docx.Document()
if location == "table":
paragraph = document.add_table(rows=1, cols=1).cell(0, 0).paragraphs[0]
elif location in ("header", "footer"):
paragraph = getattr(document.sections[0], location).paragraphs[0]
else:
paragraph = document.add_paragraph()
run = paragraph.add_run("Before ")
run._r.append(_ruby())
run.add_text(" after")
if location == "hyperlink":
link = OxmlElement("w:hyperlink")
paragraph._p.append(link)
link.append(run._r)
file = io.BytesIO()
document.save(file)
original = file.getvalue()

elements = partition_docx(file=file, languages=["eng"])

assert [element.text for element in elements] == ["Before 漢字 after"]
assert file.getvalue() == original
if location == "table":
assert "漢字" in elements[0].metadata.text_as_html
assert "かんじ" not in elements[0].metadata.text_as_html


def test_paragraph_containing_only_ruby_is_not_dropped():
document = docx.Document()
document.add_paragraph().add_run()._r.append(_ruby())
file = io.BytesIO()
document.save(file)

assert [element.text for element in partition_docx(file=file, languages=["eng"])] == ["漢字"]


def test_ruby_base_preserves_embedded_tabs():
document = docx.Document()
run = document.add_paragraph().add_run()
ruby = _ruby(("漢", "字"))
ruby.find(docx.oxml.ns.qn("w:rubyBase"))[0].append(OxmlElement("w:tab"))
run._r.append(ruby)
file = io.BytesIO()
document.save(file)

assert partition_docx(file=file, languages=["eng"])[0].text == "漢\t字"
21 changes: 21 additions & 0 deletions unstructured/partition/docx.py
Original file line number Diff line number Diff line change
Expand Up @@ -256,6 +256,7 @@ def document(self) -> Document:
for part in document.part.package.iter_parts():
if isinstance(part, (DocumentPart, HeaderPart, FooterPart)):
_unwrap_nested_content(part.element)
_unwrap_ruby_base(part.element)
return document

@cached_property
Expand Down Expand Up @@ -1338,6 +1339,26 @@ def _unwrap_nested_content(root: BaseOxmlElement) -> None:
parent[index : index + 1] = children


def _unwrap_ruby_base(root: BaseOxmlElement) -> None:
"""Keep the visible base of East Asian ruby text where python-docx can read it.

A run's ``w:ruby`` contains pronunciation in ``w:rt`` and the corresponding document
text in ``w:rubyBase``. python-docx ignores both. Lift the base runs' content into the
enclosing run, retaining inline text, tabs and breaks without duplicating pronunciation.
The enclosing run's formatting applies; separate base-run formatting is not resolved.
"""
for ruby in reversed(list(root.iter(_W + "ruby"))):
parent = ruby.getparent()
base = ruby.find(_W + "rubyBase")
if parent is None or base is None:
continue
children = [
child for run in base.findall(_W + "r") for child in run if child.tag != _W + "rPr"
]
index = parent.index(ruby)
parent[index : index + 1] = children


def _content_control_children(sdt: BaseOxmlElement) -> list[BaseOxmlElement] | None:
"""The elements to put in place of content control `sdt`, or None to leave it as it is.

Expand Down