Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
70 changes: 70 additions & 0 deletions test_unstructured/partition/html/test_table_rendered_order.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
import pytest

from unstructured.common.html_table import HtmlTable
from unstructured.partition.html import partition_html


@pytest.mark.parametrize(
("body", "expected"),
[
(
"<tfoot><tr><td>Footer</td></tr></tfoot>"
"<tbody><tr><td>Body</td></tr></tbody>"
"<thead><tr><th>Header</th></tr></thead>",
"Header Body Footer",
),
(
"<tfoot><tr><td>Footer</td></tr></tfoot>"
"<tr><td>First</td></tr><tbody><tr><td>Second</td></tr></tbody>",
"First Second Footer",
),
(
"<tfoot><tr><td>Footer</td></tr></tfoot>"
"<tfoot><tr><td>Extra footer</td></tr></tfoot>"
"<tbody><tr><td>Body</td></tr></tbody>"
"<thead><tr><th>Header</th></tr></thead>"
"<thead><tr><th>Extra header</th></tr></thead>",
"Header Extra footer Body Extra header Footer",
),
("<tbody><tr><td>First</td></tr><tr><td>Second</td></tr></tbody>", "First Second"),
],
)
def test_partition_and_chunking_table_reader_follow_rendered_row_group_order(body, expected):
html = f"<table>{body}</table>"

element = partition_html(text=html, languages=["eng"])[0]
compact = HtmlTable.from_html_text(html)

assert element.text == expected
assert compact.text == expected
assert " ".join(" ".join(row.iter_cell_texts()) for row in compact.iter_rows()) == expected


def test_source_rows_and_header_flags_remain_aligned_after_ordering():
table = HtmlTable.from_html_text(
'<table><tfoot><tr><td data-id="f">Foot</td></tr></tfoot>'
'<tbody><tr><td rowspan="0">Body</td></tr></tbody>'
'<thead><tr><th scope="col">Head</th></tr></thead></table>'
)
rows = list(table.iter_rows())

assert [list(row.iter_cell_texts()) for row in rows] == [["Head"], ["Body"], ["Foot"]]
assert [row.is_header for row in rows] == [True, False, False]
assert [row.row_group_key.tag for row in rows] == ["thead", "tbody", "tfoot"]
assert 'scope="col"' in rows[0].source_html
assert 'rowspan="0"' in rows[1].html
assert 'data-id="f"' in rows[2].source_html


def test_nested_table_order_is_normalized_without_moving_rows_between_tables():
html = (
"<table><tbody><tr><td>Outer <table>"
"<tfoot><tr><td>Inner footer</td></tr></tfoot>"
"<tbody><tr><td>Inner body</td></tr></tbody>"
"</table></td></tr></tbody></table>"
)
table = HtmlTable.from_html_text(html)

assert table.text == "Outer Inner body Inner footer"
assert len(list(table.iter_rows())) == 1
assert partition_html(text=html, languages=["eng"])[0].text == table.text
22 changes: 22 additions & 0 deletions unstructured/common/html_table.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,26 @@
SpannedCell: TypeAlias = "tuple[str, int, int]"


def iter_table_children_in_render_order(table: etree._Element) -> Iterator[etree._Element]:
"""Yield direct children with the first header first and first footer last among rows.

Under the default HTML table display rules, the first ``thead``/``tfoot`` is laid out
before/after other row groups regardless of source order. Additional headers/footers
behave as ordinary row groups. Non-row children retain their positions.
"""
row_tags = {"thead", "tbody", "tfoot", "tr"}
rows = [child for child in table if child.tag in row_tags]
header = next((child for child in rows if child.tag == "thead"), None)
footer = next((child for child in rows if child.tag == "tfoot"), None)
ordered = iter(
([header] if header is not None else [])
+ [child for child in rows if child is not header and child is not footer]
+ ([footer] if footer is not None else [])
)
for child in table:
yield next(ordered) if child.tag in row_tags else child


def normalize_html_cell_text(cell: HtmlElement) -> str:
"""Return normalized cell text, treating a ``<br>`` as a word boundary.

Expand Down Expand Up @@ -178,6 +198,8 @@ def from_html_text(cls, html_text: str) -> HtmlTable:
if not tables:
raise ValueError("`html_text` contains no `<table>` element")
table = tables[0]
for nested_table in table.iter("table"):
nested_table[:] = list(iter_table_children_in_render_order(nested_table))

# -- capture header semantics, source row HTML, and row-group identity before
# -- compactification strips those details --
Expand Down
4 changes: 4 additions & 0 deletions unstructured/partition/html/parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,7 @@
from typing_extensions import TypeAlias

from unstructured.cleaners.core import clean_bullets
from unstructured.common.html_table import iter_table_children_in_render_order
from unstructured.documents.elements import (
Address,
CodeSnippet,
Expand Down Expand Up @@ -568,6 +569,9 @@ class TableBlock(Flow):
def iter_elements(self) -> Iterator[Table]:
"""Generate paragraph string for each block item within."""

for table in self.iter("table"):
table[:] = list(iter_table_children_in_render_order(table))

# -- NOTE this algorithm handles a nested-table by parsing all of its text into the text
# -- for the _cell_ containing the table (and this is recursive, so a table nested within
# -- a cell within a table within a cell too.)
Expand Down