Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 37 additions & 1 deletion test_unstructured/partition/html/test_partition.py
Original file line number Diff line number Diff line change
Expand Up @@ -260,8 +260,10 @@ def test_partition_html_on_ideas_page():

assert len(elements) == 1
e = elements[0]
# -- the essay title is an image, `<img alt="How to Get New Ideas">`, so it is text too --
assert e == Table(
"January 2023 ( Someone fed my essays into GPT to make something that could answer"
"How to Get New Ideas January 2023 ( Someone fed my essays into GPT to make something"
" that could answer"
"\nquestions based on them, then asked it where good ideas come from. The"
"\nanswer was ok, but not what I would have said. This is what I would have said.)"
" The way to get new ideas is to notice anomalies: what seems strange,"
Expand Down Expand Up @@ -475,6 +477,40 @@ def test_it_keeps_the_table_caption_in_the_text_and_html():
)


def test_it_keeps_the_alt_text_of_an_image_in_a_table_cell():
# -- a feature table often marks a cell with an image only, such as a check mark --
html_text = (
"<table>\n"
" <tr><th>Feature</th><th>Supported</th></tr>\n"
' <tr><td>Search</td><td><img src="check.png" alt="Yes"></td></tr>\n'
' <tr><td>Export</td><td>Partly <img src="half.png" alt="(beta)"> now</td></tr>\n'
' <tr><td>Sync</td><td><img src="spacer.gif"></td></tr>\n'
"</table>"
)

(element,) = partition_html(text=html_text)

assert isinstance(element, Table)
assert element.text == "Feature Supported Search Yes Export Partly (beta) now Sync"
assert element.metadata.text_as_html == (
"<table>"
"<tr><th>Feature</th><th>Supported</th></tr>"
"<tr><td>Search</td><td>Yes</td></tr>"
"<tr><td>Export</td><td>Partly (beta) now</td></tr>"
"<tr><td>Sync</td><td/></tr>"
"</table>"
)


def test_it_emits_a_Table_element_for_a_table_whose_cells_are_only_images():
html_text = '<table><tr><td><img src="a.png" alt="Accepted"></td></tr></table>'

(element,) = partition_html(text=html_text)

assert isinstance(element, Table)
assert element.text == "Accepted"


def test_it_does_not_emit_a_Table_element_for_a_table_with_only_a_caption():
html_text = "<table><caption>Empty results</caption><tr><td> </td></tr></table>"

Expand Down
5 changes: 4 additions & 1 deletion test_unstructured/partition/test_email.py
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,10 @@ def test_partition_email_partitions_an_html_part_with_quoted_printable_encoded_I

assert len(elements) == 1
assert isinstance(elements[0], Table)
assert elements[0].text.startswith("Slava Gxyzxyz Hi Slava, The password for your Google")
# "Google Accounts" is the alt text of the logo in the first table cell
assert elements[0].text.startswith(
"Google Accounts Slava Gxyzxyz Hi Slava, The password for your Google"
)


# -- edge-cases ----------------------------------------------------------------------------------
Expand Down
23 changes: 21 additions & 2 deletions unstructured/partition/html/parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -576,12 +576,31 @@ def iter_elements(self) -> Iterator[Table]:
caption_texts: list[str] = []
row_texts: list[str] = []

def text_nodes(node: etree._Element) -> Iterator[str]:
"""Text of `node` in document order, like `.itertext()`, plus each image's `alt`.

An image in a cell, such as a check mark with `alt="Yes"`, is often all the cell
holds, so its alt text is the cell's text.
"""
if node.text:
yield node.text
for child in node:
if child.tag == "img":
if alt := child.get("alt"):
yield alt
elif isinstance(child.tag, str):
# -- comments and processing instructions contribute no text, as with
# -- `.itertext()`, but their tail still does
yield from text_nodes(child)
if child.tail:
yield child.tail

def sanitized_text_and_html(node: etree._Element) -> tuple[str, str]:
"""Text of a cell or caption, and the bare tag wrapping that escaped text."""
# -- a cell can contain other elements like spans etc. so we can't count on the
# -- text being directly below the cell. `.itertext()` gets all of it recursively.
# -- text being directly below the cell. `text_nodes()` gets all of it recursively.
# -- Filter out whitespace text nodes resulting from HTML formatting.
stripped_text_nodes = (t.strip() for t in node.itertext())
stripped_text_nodes = (t.strip() for t in text_nodes(node))
text = " ".join(t for t in stripped_text_nodes if t)

# -- Reconstruct only the recognized tag and escaped text. Source attributes and
Expand Down