Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
66 changes: 66 additions & 0 deletions test_unstructured/partition/test_xlsx_hidden_sheets.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
import io
from unittest.mock import Mock

import openpyxl
import pytest
import xlrd

from unstructured.partition import xlsx


@pytest.mark.parametrize("find_subtable", [False, True])
def test_hidden_worksheets_can_be_skipped_without_renumbering(monkeypatch, find_subtable):
workbook = openpyxl.Workbook()
workbook.active.title = "Visible first"
workbook.create_sheet("Hidden").sheet_state = "hidden"
workbook.create_sheet("Very hidden").sheet_state = "veryHidden"
workbook.create_sheet("Visible last")
for sheet in workbook:
sheet.append(["Name", "Value"])
sheet.append([sheet.title, 1])
file = io.BytesIO()
workbook.save(file)
parsed = []
original = xlsx.ExcelWorkbookReader.parse

def parse(reader, name, *, header=None):
parsed.append(name)
return original(reader, name, header=header)

monkeypatch.setattr(xlsx.ExcelWorkbookReader, "parse", parse)

elements = xlsx.partition_xlsx(
file=file,
find_subtable=find_subtable,
include_hidden_sheets=False,
starting_page_number=5,
languages=["eng"],
)

assert parsed == ["Visible first", "Visible last"]
assert [e.metadata.page_name for e in elements] == parsed
assert [e.metadata.page_number for e in elements] == [5, 8]


def test_hidden_worksheets_remain_included_by_default():
workbook = openpyxl.Workbook()
workbook.active.append(["Public", "value"])
hidden = workbook.create_sheet("Hidden")
hidden.sheet_state = "hidden"
hidden.append(["Hidden", "value"])
file = io.BytesIO()
workbook.save(file)

elements = xlsx.partition_xlsx(file=file, find_subtable=False, languages=["eng"])

assert [e.metadata.page_name for e in elements] == ["Sheet", "Hidden"]


@pytest.mark.parametrize(("visibility", "expected"), [(0, False), (1, True), (2, True)])
def test_xls_hidden_and_very_hidden_states_are_recognized(visibility, expected):
book = Mock(spec=xlrd.book.Book)
book.sheet_by_name.return_value.visibility = visibility
reader = xlsx.ExcelWorkbookReader(Mock(book=book))

assert reader.is_sheet_hidden("Legacy sheet") is expected
book.sheet_by_name.assert_called_once_with("Legacy sheet")
27 changes: 24 additions & 3 deletions unstructured/partition/xlsx.py
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ def partition_xlsx(
file: Optional[IO[bytes]] = None,
find_subtable: bool = True,
include_header: bool = False,
include_hidden_sheets: bool = True,
infer_table_structure: bool = True,
starting_page_number: int = 1,
**kwargs: Any,
Expand All @@ -84,6 +85,9 @@ def partition_xlsx(
and is the text content of the table (no structure).
include_header
Determines whether or not header info is included in text and medatada.text_as_html
include_hidden_sheets
Include hidden and very-hidden worksheets by default. When False, skip them without
reading their cell data. Page numbers still refer to the original workbook sheet order.

Notes
-----
Expand All @@ -97,13 +101,16 @@ def partition_xlsx(
file=file,
find_subtable=find_subtable,
include_header=include_header,
include_hidden_sheets=include_hidden_sheets,
infer_table_structure=infer_table_structure,
)

elements: list[Element] = []
for page_number, (sheet_name, sheet) in enumerate(
opts.sheets.items(), start=starting_page_number
):
if sheet is None:
continue
if not opts.find_subtable:
text, html = _render_table(sheet, opts)

Expand Down Expand Up @@ -200,6 +207,13 @@ def parse(self, sheet_name: str, *, header: int | None = None) -> pd.DataFrame:
sheet_name=sheet_name, header=header, keep_default_na=False, na_values=[""]
)

def is_sheet_hidden(self, sheet_name: str) -> bool:
"""Whether XLSX/XLS marks the worksheet hidden or very hidden."""
book = self._reader.book
if isinstance(book, xlrd.book.Book):
return book.sheet_by_name(sheet_name).visibility != 0
return book[sheet_name].sheet_state != "visible"


@contextmanager
def open_excel_workbook(
Expand Down Expand Up @@ -263,12 +277,14 @@ def __init__(
find_subtable: bool,
include_header: bool,
infer_table_structure: bool,
include_hidden_sheets: bool = True,
):
self._file_path = file_path
self._file = file
self._find_subtable = find_subtable
self._include_header = include_header
self._infer_table_structure = infer_table_structure
self._include_hidden_sheets = include_hidden_sheets

@cached_property
def find_subtable(self) -> bool:
Expand Down Expand Up @@ -304,11 +320,16 @@ def metadata_file_path(self) -> str | None:
return self._file_path

@cached_property
def sheets(self) -> dict[str, pd.DataFrame]:
"""The spreadsheet worksheets, each as a data-frame mapped by sheet-name."""
def sheets(self) -> dict[str, pd.DataFrame | None]:
"""Worksheets in workbook order; skipped hidden sheets retain a None placeholder."""
with open_excel_workbook(self._file_bytes, compact_empty_rows=self.find_subtable) as reader:
return {
name: reader.parse(name, header=self.header_row_idx) for name in reader.sheet_names
name: (
reader.parse(name, header=self.header_row_idx)
if self._include_hidden_sheets or not reader.is_sheet_hidden(name)
else None
)
for name in reader.sheet_names
}

@cached_property
Expand Down