Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
64 changes: 64 additions & 0 deletions test_unstructured/partition/test_docx_strikethrough.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
import io

import docx
import pytest
from docx.oxml import OxmlElement

from unstructured.partition.docx import partition_docx


@pytest.mark.parametrize("location", ["body", "table", "header", "footer", "hyperlink"])
@pytest.mark.parametrize("double", [False, True])
def test_opt_in_omits_explicitly_struck_text(location, double):
document = docx.Document()
if location == "table":
paragraph = document.add_table(rows=1, cols=1).cell(0, 0).paragraphs[0]
elif location in ("header", "footer"):
paragraph = getattr(document.sections[0], location).paragraphs[0]
else:
paragraph = document.add_paragraph()
paragraph.add_run("Keep ")
run = paragraph.add_run("obsolete ")
run.bold = True
if double:
run.font.double_strike = True
else:
run.font.strike = True
if location == "hyperlink":
hyperlink = OxmlElement("w:hyperlink")
paragraph._p.append(hyperlink)
hyperlink.append(run._r)
paragraph.add_run("current")
file = io.BytesIO()
document.save(file)
original = file.getvalue()

elements = partition_docx(file=file, skip_strikethrough=True, languages=["eng"])

assert [e.text for e in elements] == ["Keep current"]
assert all("obsolete" not in (e.metadata.text_as_html or "") for e in elements)
assert all(not e.metadata.emphasized_text_contents for e in elements)
assert file.getvalue() == original


@pytest.mark.parametrize("skip", [False, True])
def test_explicitly_disabled_strike_is_kept(skip):
document = docx.Document()
run = document.add_paragraph().add_run("Current content")
run.font.strike = False
run.font.double_strike = False
file = io.BytesIO()
document.save(file)

assert partition_docx(file=file, skip_strikethrough=skip, languages=["eng"])[0].text == (
"Current content"
)


def test_default_preserves_struck_content():
document = docx.Document()
document.add_paragraph().add_run("Still present").font.strike = True
file = io.BytesIO()
document.save(file)

assert partition_docx(file=file, languages=["eng"])[0].text == "Still present"
24 changes: 24 additions & 0 deletions unstructured/partition/docx.py
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,7 @@ def partition_docx(
infer_table_structure: bool = True,
starting_page_number: int = 1,
strategy: str | None = None,
skip_strikethrough: bool = False,
**kwargs: Any,
) -> list[Element]:
"""Partitions Microsoft Word Documents in .docx format into its document elements.
Expand Down Expand Up @@ -189,10 +190,14 @@ def partition_docx(
starting_page_number
Assign this number to the first page of this document and increment the page number from
there.
skip_strikethrough
Omit text with explicit single/double strikethrough run formatting. Defaults to False.
Style-inherited strikethrough is not resolved. Drawings and page breaks are preserved.
"""
opts = DocxPartitionerOptions.load(
file=file,
file_path=filename,
skip_strikethrough=skip_strikethrough,
include_page_breaks=include_page_breaks,
infer_table_structure=infer_table_structure,
starting_page_number=starting_page_number,
Expand Down Expand Up @@ -226,9 +231,11 @@ def __init__(
infer_table_structure: bool,
starting_page_number: int = 1,
strategy: str | None = None,
skip_strikethrough: bool = False,
):
self._file = file
self._file_path = file_path
self._skip_strikethrough = skip_strikethrough
self._include_page_breaks = include_page_breaks
self._infer_table_structure = infer_table_structure
self._strategy = strategy
Expand Down Expand Up @@ -256,6 +263,8 @@ def document(self) -> Document:
for part in document.part.package.iter_parts():
if isinstance(part, (DocumentPart, HeaderPart, FooterPart)):
_unwrap_nested_content(part.element)
if self._skip_strikethrough:
_remove_struck_text(part.element)
return document

@cached_property
Expand Down Expand Up @@ -1361,6 +1370,21 @@ def _content_control_children(sdt: BaseOxmlElement) -> list[BaseOxmlElement] | N
# ================================================================================================


def _remove_struck_text(root: Any) -> None:
"""Remove explicitly struck run text while retaining structural and drawing content."""
text_tags = {qn(tag) for tag in ("w:t", "w:tab", "w:cr", "w:noBreakHyphen", "w:softHyphen")}
for run in root.xpath(".//w:r[w:rPr/w:strike or w:rPr/w:dstrike]"):
marks = run.xpath("./w:rPr/w:strike | ./w:rPr/w:dstrike")
if not any(mark.get(qn("w:val"), "true") in ("true", "1", "on") for mark in marks):
continue
for child in list(run):
line_break = child.tag == qn("w:br") and child.get(qn("w:type"), "textWrapping") == (
"textWrapping"
)
if child.tag in text_tags or line_break:
run.remove(child)


def _paragraph_text(paragraph: Paragraph) -> str:
"""The text of `paragraph` that becomes element text: its runs, hyperlinks and inline runs."""
return "".join(
Expand Down