From 7bd7997bb98ae3fdf0ff94e59b703859b934e067 Mon Sep 17 00:00:00 2001 From: Afloat Date: Sat, 21 Nov 2026 11:25:21 +0800 Subject: [PATCH] feat(html): optionally extract authored form values --- .../partition/html/test_form_values.py | 120 ++++++++++++++++++ unstructured/partition/html/forms.py | 80 ++++++++++++ unstructured/partition/html/partition.py | 14 ++ 3 files changed, 214 insertions(+) create mode 100644 test_unstructured/partition/html/test_form_values.py create mode 100644 unstructured/partition/html/forms.py diff --git a/test_unstructured/partition/html/test_form_values.py b/test_unstructured/partition/html/test_form_values.py new file mode 100644 index 0000000000..9e6163f9e3 --- /dev/null +++ b/test_unstructured/partition/html/test_form_values.py @@ -0,0 +1,120 @@ +import pytest + +from unstructured.partition.html import partition_html + + +@pytest.mark.parametrize("version", ["v1", "v2"]) +def test_opt_in_extracts_authored_form_labels_and_values(version): + html = ( + '
' + '

' + '

Country:

" + '

Notes:

' + "
" + ) + + elements = partition_html( + text=html, include_form_values=True, html_parser_version=version, languages=["eng"] + ) + + assert [" ".join(e.text.split()) for e in elements if e.text] == [ + "Name: Ada", + "Country: Japan", + "Notes: Use clear words.", + ] + + +def test_forms_remain_excluded_by_default(): + elements = partition_html( + text='
Before
After
', + languages=["eng"], + ) + + assert all("Ada" not in e.text and "Name" not in e.text for e in elements) + + +@pytest.mark.parametrize( + ("kind", "value"), + [ + ("text", "Ada"), + ("email", "ada@example.com"), + ("search", "query"), + ("tel", "+1 555 0100"), + ("url", "https://example.com"), + ("number", "42.5"), + ("date", "2026-11-21"), + ], +) +def test_textual_input_values_are_retained(kind, value): + elements = partition_html( + text=f'

Field:

', + include_form_values=True, + languages=["eng"], + ) + + assert elements[0].text == f"Field: {value}" + + +def test_values_containing_markup_stay_text(): + elements = partition_html( + text='

Example:

', + include_form_values=True, + languages=["eng"], + ) + + assert elements[0].text == "Example: " + + +@pytest.mark.parametrize("kind", ["password", "hidden", "file", "submit", "reset", "button"]) +def test_nontext_controls_do_not_expose_value_attributes(kind): + elements = partition_html( + text=f'

Field:

', + include_form_values=True, + languages=["eng"], + ) + + assert [e.text for e in elements] == ["Field:"] + + +@pytest.mark.parametrize( + ("select", "expected"), + [ + ( + "", + "First usable", + ), + ("", "Last"), + ( + "", + "A; C", + ), + ( + "', + "Label", + ), + ('', "First"), + ], +) +def test_select_extracts_the_selected_label_instead_of_the_entire_option_list(select, expected): + elements = partition_html( + text=f"

Choice: {select}

", include_form_values=True, languages=["eng"] + ) + + assert elements[0].text == f"Choice: {expected}" + + +def test_hidden_forms_and_controls_are_not_materialized(): + html = ( + '
Before' + '

Visible

After
' + ) + + elements = partition_html(text=html, include_form_values=True, languages=["eng"]) + + text = " ".join(e.text for e in elements) + assert "Visible" in text + assert "Secret" not in text + assert "hidden" not in text diff --git a/unstructured/partition/html/forms.py b/unstructured/partition/html/forms.py new file mode 100644 index 0000000000..a9191600ef --- /dev/null +++ b/unstructured/partition/html/forms.py @@ -0,0 +1,80 @@ +"""Opt-in materialization of authored form values for document extraction.""" + +from lxml import etree + +_TEXT_INPUT_TYPES = frozenset( + ( + "text", + "search", + "email", + "tel", + "url", + "number", + "date", + "time", + "datetime-local", + "month", + "week", + ) +) + + +def _selected_labels(select: etree._Element) -> str: + options = list(select.iter("option")) + selected = [option for option in options if "selected" in option.attrib] + try: + size = int(select.get("size", "1")) + except ValueError: + size = 1 + if "multiple" not in select.attrib: + if selected: + selected = selected[-1:] + elif size <= 1: + selected = [ + option + for option in options + if "disabled" not in option.attrib + and not any( + "disabled" in group.attrib for group in option.iterancestors("optgroup") + ) + ][:1] + return "; ".join( + " ".join((option.get("label") or "".join(option.itertext())).split()) for option in selected + ) + + +def materialize_form_values(html: str) -> str: + """Replace form controls with static text while preserving their place in the document. + + Only authored values are available; script state, placeholder prompts, passwords and + non-text controls are not extracted. The source HTML is unchanged. Reparse the serialized + result with the chosen partitioner so replacement tags receive its normal element classes. + """ + parser = etree.HTMLParser(remove_comments=True) + try: + root = etree.fromstring(html, parser) + except ValueError: + root = etree.fromstring(html.encode("utf-8"), parser) + if root is None: + return html + for node in list(root.iter("form", "label", "input", "textarea", "select")): + if any("hidden" in ancestor.attrib for ancestor in (node, *node.iterancestors())): + text = "" + elif node.tag in {"form", "label"}: + node.tag = "div" if node.tag == "form" else "span" + continue + elif node.tag == "input": + text = ( + node.get("value", "") + if node.get("type", "text").lower() in _TEXT_INPUT_TYPES + else "" + ) + elif node.tag == "select": + text = _selected_labels(node) + else: + text = "".join(node.itertext()) + node.tag = "span" + node.attrib.clear() + node[:] = [] + node.text = text + return etree.tostring(root, encoding="unicode", method="html") diff --git a/unstructured/partition/html/partition.py b/unstructured/partition/html/partition.py index 847cc836b8..66d6e78bc0 100644 --- a/unstructured/partition/html/partition.py +++ b/unstructured/partition/html/partition.py @@ -14,6 +14,7 @@ from unstructured.file_utils.encoding import read_txt_file from unstructured.file_utils.model import FileType from unstructured.partition.common.metadata import apply_metadata, get_last_modified_date +from unstructured.partition.html.forms import materialize_form_values from unstructured.partition.html.parser import Flow, Math, html_parser from unstructured.partition.html.transformations import ( ontology_to_unstructured_elements, @@ -37,6 +38,7 @@ def partition_html( headers: dict[str, str] = {}, ssl_verify: bool = True, skip_headers_and_footers: bool = False, + include_form_values: bool = False, detection_origin: Optional[str] = None, html_parser_version: Literal["v1", "v2"] = "v1", image_alt_mode: Optional[Literal["to_text"]] = "to_text", @@ -70,6 +72,9 @@ def partition_html( The encoding method used to decode the text input. If None, utf-8 will be used. skip_headers_and_footers If True, ignores any content that is within
or