Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
120 changes: 120 additions & 0 deletions test_unstructured/partition/html/test_form_values.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
import pytest

from unstructured.partition.html import partition_html


@pytest.mark.parametrize("version", ["v1", "v2"])
def test_opt_in_extracts_authored_form_labels_and_values(version):
html = (
'<body class="Document"><div class="Page"><form>'
'<p class="NarrativeText"><label>Name: <input value="Ada"></label></p>'
'<p class="NarrativeText">Country: <select><option>France</option>'
"<option selected>Japan</option></select></p>"
'<p class="NarrativeText">Notes: <textarea>Use clear words.</textarea></p>'
"</form></div></body>"
)

elements = partition_html(
text=html, include_form_values=True, html_parser_version=version, languages=["eng"]
)

assert [" ".join(e.text.split()) for e in elements if e.text] == [
"Name: Ada",
"Country: Japan",
"Notes: Use clear words.",
]


def test_forms_remain_excluded_by_default():
elements = partition_html(
text='<div>Before<form><label>Name<input value="Ada"></label></form>After</div>',
languages=["eng"],
)

assert all("Ada" not in e.text and "Name" not in e.text for e in elements)


@pytest.mark.parametrize(
("kind", "value"),
[
("text", "Ada"),
("email", "ada@example.com"),
("search", "query"),
("tel", "+1 555 0100"),
("url", "https://example.com"),
("number", "42.5"),
("date", "2026-11-21"),
],
)
def test_textual_input_values_are_retained(kind, value):
elements = partition_html(
text=f'<p>Field: <input type="{kind}" value="{value}"></p>',
include_form_values=True,
languages=["eng"],
)

assert elements[0].text == f"Field: {value}"


def test_values_containing_markup_stay_text():
elements = partition_html(
text='<p>Example: <input value="&lt;safe&gt;"></p>',
include_form_values=True,
languages=["eng"],
)

assert elements[0].text == "Example: <safe>"


@pytest.mark.parametrize("kind", ["password", "hidden", "file", "submit", "reset", "button"])
def test_nontext_controls_do_not_expose_value_attributes(kind):
elements = partition_html(
text=f'<p>Field: <input type="{kind}" value="Never expose"></p>',
include_form_values=True,
languages=["eng"],
)

assert [e.text for e in elements] == ["Field:"]


@pytest.mark.parametrize(
("select", "expected"),
[
(
"<select><option disabled>Unavailable</option><option>First usable</option></select>",
"First usable",
),
("<select><option selected>First</option><option selected>Last</option></select>", "Last"),
(
"<select multiple><option selected>A</option><option>B</option>"
"<option selected>C</option></select>",
"A; C",
),
(
"<select><optgroup disabled><option>A</option></optgroup>"
'<option label="Label">Value</option></select>',
"Label",
),
('<select size="invalid"><option>First</option><option>Second</option></select>', "First"),
],
)
def test_select_extracts_the_selected_label_instead_of_the_entire_option_list(select, expected):
elements = partition_html(
text=f"<p>Choice: {select}</p>", include_form_values=True, languages=["eng"]
)

assert elements[0].text == f"Choice: {expected}"


def test_hidden_forms_and_controls_are_not_materialized():
html = (
'<div>Before<form hidden><label>Secret<input value="hidden form"></label></form>'
'<p>Visible <input hidden value="hidden input"></p>After</div>'
)

elements = partition_html(text=html, include_form_values=True, languages=["eng"])

text = " ".join(e.text for e in elements)
assert "Visible" in text
assert "Secret" not in text
assert "hidden" not in text
80 changes: 80 additions & 0 deletions unstructured/partition/html/forms.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
"""Opt-in materialization of authored form values for document extraction."""

from lxml import etree

_TEXT_INPUT_TYPES = frozenset(
(
"text",
"search",
"email",
"tel",
"url",
"number",
"date",
"time",
"datetime-local",
"month",
"week",
)
)


def _selected_labels(select: etree._Element) -> str:
options = list(select.iter("option"))
selected = [option for option in options if "selected" in option.attrib]
try:
size = int(select.get("size", "1"))
except ValueError:
size = 1
if "multiple" not in select.attrib:
if selected:
selected = selected[-1:]
elif size <= 1:
selected = [
option
for option in options
if "disabled" not in option.attrib
and not any(
"disabled" in group.attrib for group in option.iterancestors("optgroup")
)
][:1]
return "; ".join(
" ".join((option.get("label") or "".join(option.itertext())).split()) for option in selected
)


def materialize_form_values(html: str) -> str:
"""Replace form controls with static text while preserving their place in the document.

Only authored values are available; script state, placeholder prompts, passwords and
non-text controls are not extracted. The source HTML is unchanged. Reparse the serialized
result with the chosen partitioner so replacement tags receive its normal element classes.
"""
parser = etree.HTMLParser(remove_comments=True)
try:
root = etree.fromstring(html, parser)
except ValueError:
root = etree.fromstring(html.encode("utf-8"), parser)
if root is None:
return html
for node in list(root.iter("form", "label", "input", "textarea", "select")):
if any("hidden" in ancestor.attrib for ancestor in (node, *node.iterancestors())):
text = ""
elif node.tag in {"form", "label"}:
node.tag = "div" if node.tag == "form" else "span"
continue
elif node.tag == "input":
text = (
node.get("value", "")
if node.get("type", "text").lower() in _TEXT_INPUT_TYPES
else ""
)
elif node.tag == "select":
text = _selected_labels(node)
else:
text = "".join(node.itertext())
node.tag = "span"
node.attrib.clear()
node[:] = []
node.text = text
return etree.tostring(root, encoding="unicode", method="html")
14 changes: 14 additions & 0 deletions unstructured/partition/html/partition.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
from unstructured.file_utils.encoding import read_txt_file
from unstructured.file_utils.model import FileType
from unstructured.partition.common.metadata import apply_metadata, get_last_modified_date
from unstructured.partition.html.forms import materialize_form_values
from unstructured.partition.html.parser import Flow, Math, html_parser
from unstructured.partition.html.transformations import (
ontology_to_unstructured_elements,
Expand All @@ -37,6 +38,7 @@ def partition_html(
headers: dict[str, str] = {},
ssl_verify: bool = True,
skip_headers_and_footers: bool = False,
include_form_values: bool = False,
detection_origin: Optional[str] = None,
html_parser_version: Literal["v1", "v2"] = "v1",
image_alt_mode: Optional[Literal["to_text"]] = "to_text",
Expand Down Expand Up @@ -70,6 +72,9 @@ def partition_html(
The encoding method used to decode the text input. If None, utf-8 will be used.
skip_headers_and_footers
If True, ignores any content that is within <header> or <footer> tags
include_form_values
If True, include authored textual input values, textarea text, selected option labels and
form labels. Passwords, hidden controls, placeholders and script-updated state are omitted.

html_parser_version (Literal['v1', 'v2']):
The version of the HTML parser to use. The default is 'v1'. For 'v2' the parser will
Expand Down Expand Up @@ -99,6 +104,7 @@ def partition_html(
headers=headers,
ssl_verify=ssl_verify,
skip_headers_and_footers=skip_headers_and_footers,
include_form_values=include_form_values,
detection_origin=detection_origin,
html_parser_version=html_parser_version,
image_alt_mode=image_alt_mode,
Expand Down Expand Up @@ -128,6 +134,7 @@ def __init__(
image_alt_mode: Optional[Literal["to_text"]] = "to_text",
extract_image_block_types: Optional[list[str]] = None,
extract_image_block_to_payload: bool = False,
include_form_values: bool = False,
):
self._file_path = file_path
self._file = file
Expand All @@ -137,6 +144,7 @@ def __init__(
self._headers = headers
self._ssl_verify = ssl_verify
self._skip_headers_and_footers = skip_headers_and_footers
self._include_form_values = include_form_values
self._detection_origin = detection_origin
self._html_parser_version = html_parser_version
self._image_alt_mode = image_alt_mode
Expand All @@ -150,6 +158,12 @@ def detection_origin(self) -> str | None:

@cached_property
def html_text(self) -> str:
"""Source HTML, optionally with authored form values made visible to either parser."""
text = self._source_html_text
return materialize_form_values(text) if self._include_form_values else text

@cached_property
def _source_html_text(self) -> str:
"""The HTML document as a string, loaded from wherever the caller specified."""
if self._file_path:
return read_txt_file(filename=self._file_path, encoding=self._encoding)[1]
Expand Down