diff --git a/test_unstructured/partition/html/test_partition.py b/test_unstructured/partition/html/test_partition.py index 26379f14ec..8b414c0bf6 100644 --- a/test_unstructured/partition/html/test_partition.py +++ b/test_unstructured/partition/html/test_partition.py @@ -260,8 +260,10 @@ def test_partition_html_on_ideas_page(): assert len(elements) == 1 e = elements[0] + # -- the essay title is an image, `How to Get New Ideas`, so it is text too -- assert e == Table( - "January 2023 ( Someone fed my essays into GPT to make something that could answer" + "How to Get New Ideas January 2023 ( Someone fed my essays into GPT to make something" + " that could answer" "\nquestions based on them, then asked it where good ideas come from. The" "\nanswer was ok, but not what I would have said. This is what I would have said.)" " The way to get new ideas is to notice anomalies: what seems strange," @@ -475,6 +477,40 @@ def test_it_keeps_the_table_caption_in_the_text_and_html(): ) +def test_it_keeps_the_alt_text_of_an_image_in_a_table_cell(): + # -- a feature table often marks a cell with an image only, such as a check mark -- + html_text = ( + "\n" + " \n" + ' \n' + ' \n' + ' \n' + "
FeatureSupported
SearchYes
ExportPartly (beta) now
Sync
" + ) + + (element,) = partition_html(text=html_text) + + assert isinstance(element, Table) + assert element.text == "Feature Supported Search Yes Export Partly (beta) now Sync" + assert element.metadata.text_as_html == ( + "" + "" + "" + "" + "" + "
FeatureSupported
SearchYes
ExportPartly (beta) now
Sync
" + ) + + +def test_it_emits_a_Table_element_for_a_table_whose_cells_are_only_images(): + html_text = '
Accepted
' + + (element,) = partition_html(text=html_text) + + assert isinstance(element, Table) + assert element.text == "Accepted" + + def test_it_does_not_emit_a_Table_element_for_a_table_with_only_a_caption(): html_text = "
Empty results
" diff --git a/test_unstructured/partition/test_email.py b/test_unstructured/partition/test_email.py index f4bc018ee1..b0997accb7 100644 --- a/test_unstructured/partition/test_email.py +++ b/test_unstructured/partition/test_email.py @@ -132,7 +132,10 @@ def test_partition_email_partitions_an_html_part_with_quoted_printable_encoded_I assert len(elements) == 1 assert isinstance(elements[0], Table) - assert elements[0].text.startswith("Slava Gxyzxyz Hi Slava, The password for your Google") + # "Google Accounts" is the alt text of the logo in the first table cell + assert elements[0].text.startswith( + "Google Accounts Slava Gxyzxyz Hi Slava, The password for your Google" + ) # -- edge-cases ---------------------------------------------------------------------------------- diff --git a/unstructured/partition/html/parser.py b/unstructured/partition/html/parser.py index 30c45e836e..838a885ce9 100644 --- a/unstructured/partition/html/parser.py +++ b/unstructured/partition/html/parser.py @@ -576,12 +576,31 @@ def iter_elements(self) -> Iterator[Table]: caption_texts: list[str] = [] row_texts: list[str] = [] + def text_nodes(node: etree._Element) -> Iterator[str]: + """Text of `node` in document order, like `.itertext()`, plus each image's `alt`. + + An image in a cell, such as a check mark with `alt="Yes"`, is often all the cell + holds, so its alt text is the cell's text. + """ + if node.text: + yield node.text + for child in node: + if child.tag == "img": + if alt := child.get("alt"): + yield alt + elif isinstance(child.tag, str): + # -- comments and processing instructions contribute no text, as with + # -- `.itertext()`, but their tail still does + yield from text_nodes(child) + if child.tail: + yield child.tail + def sanitized_text_and_html(node: etree._Element) -> tuple[str, str]: """Text of a cell or caption, and the bare tag wrapping that escaped text.""" # -- a cell can contain other elements like spans etc. so we can't count on the - # -- text being directly below the cell. `.itertext()` gets all of it recursively. + # -- text being directly below the cell. `text_nodes()` gets all of it recursively. # -- Filter out whitespace text nodes resulting from HTML formatting. - stripped_text_nodes = (t.strip() for t in node.itertext()) + stripped_text_nodes = (t.strip() for t in text_nodes(node)) text = " ".join(t for t in stripped_text_nodes if t) # -- Reconstruct only the recognized tag and escaped text. Source attributes and