diff --git a/test_unstructured/partition/html/test_relative_urls.py b/test_unstructured/partition/html/test_relative_urls.py new file mode 100644 index 0000000000..fa477b93cd --- /dev/null +++ b/test_unstructured/partition/html/test_relative_urls.py @@ -0,0 +1,75 @@ +from types import SimpleNamespace + +import pytest + +from unstructured.partition.html import partition_html + + +@pytest.mark.parametrize("parser_version", ["v1", "v2"]) +def test_opt_in_resolves_links_using_the_document_base(parser_version): + elements = partition_html( + text='' + '

' + 'Guide

', + resolve_relative_urls=True, + html_parser_version=parser_version, + languages=["eng"], + ) + + element = next(e for e in elements if e.text == "Guide") + if parser_version == "v1": + assert element.metadata.link_urls == ["https://example.com/guide?q=1#section"] + else: + assert 'href="https://example.com/guide?q=1#section"' in element.metadata.text_as_html + + +def test_url_input_resolves_against_the_final_response_url(monkeypatch): + from unstructured.partition.html import partition as module + + monkeypatch.setattr( + module, + "safe_get", + lambda *args, **kwargs: SimpleNamespace( + ok=True, + headers={"Content-Type": "text/html"}, + url="https://example.com/final/article/", + text='

Source

', + ), + ) + + element = partition_html( + url="https://example.com/old", resolve_relative_urls=True, languages=["eng"] + )[0] + + assert element.metadata.link_urls == ["https://example.com/final/source"] + + +def test_resolution_uses_only_the_first_base_and_keeps_absolute_schemes(): + elements = partition_html( + text='' + '

Page Mail

' + 'Image', + resolve_relative_urls=True, + languages=["eng"], + ) + + assert elements[0].metadata.link_urls == ["https://example.com/a/page", "mailto:a@example.com"] + assert elements[1].metadata.image_url == "https://example.com/image.png" + + +@pytest.mark.parametrize("resolve", [False, True]) +def test_without_a_base_url_relative_links_remain_relative(resolve): + element = partition_html( + text='

Source

', + resolve_relative_urls=resolve, + languages=["eng"], + )[0] + assert element.metadata.link_urls == ["source"] + + +def test_existing_default_keeps_relative_link_values(): + element = partition_html( + text='

Source

', + languages=["eng"], + )[0] + assert element.metadata.link_urls == ["source"] diff --git a/unstructured/partition/html/partition.py b/unstructured/partition/html/partition.py index 847cc836b8..a4eee0c1f0 100644 --- a/unstructured/partition/html/partition.py +++ b/unstructured/partition/html/partition.py @@ -6,6 +6,7 @@ from functools import cached_property from typing import IO, Any, Callable, Iterator, List, Literal, Optional, cast +from urllib.parse import urljoin, urlsplit from lxml import etree @@ -45,6 +46,7 @@ def partition_html( languages: Optional[list[str]] = None, detect_language_per_element: bool = False, language_fallback: Optional[Callable[[str], Optional[list[str]]]] = None, + resolve_relative_urls: bool = False, **kwargs: Any, ) -> list[Element]: """Partitions an HTML document into its constituent elements. @@ -85,6 +87,9 @@ def partition_html( language_fallback Optional callable for short text; called with the text, return ISO 639-3 codes or None. Called only when ``languages`` contains "auto"; explicit languages are kept for short text. + resolve_relative_urls + Resolve link and image URLs against the first HTML base URL or the final response URL. + Defaults to False, preserving relative values. Resolution does not fetch linked resources. """ # -- parser rejects an empty str, nip that edge-case in the bud here -- if text is not None and text.strip() == "" and not file and not filename and not url: @@ -104,6 +109,7 @@ def partition_html( image_alt_mode=image_alt_mode, extract_image_block_types=extract_image_block_types, extract_image_block_to_payload=extract_image_block_to_payload, + resolve_relative_urls=resolve_relative_urls, ) return list(_HtmlPartitioner.iter_elements(opts)) @@ -128,6 +134,7 @@ def __init__( image_alt_mode: Optional[Literal["to_text"]] = "to_text", extract_image_block_types: Optional[list[str]] = None, extract_image_block_to_payload: bool = False, + resolve_relative_urls: bool = False, ): self._file_path = file_path self._file = file @@ -142,6 +149,8 @@ def __init__( self._image_alt_mode = image_alt_mode self._extract_image_block_types = extract_image_block_types self._extract_image_block_to_payload = extract_image_block_to_payload + self._resolve_relative_urls = resolve_relative_urls + self._document_url = url @cached_property def detection_origin(self) -> str | None: @@ -150,6 +159,30 @@ def detection_origin(self) -> str | None: @cached_property def html_text(self) -> str: + """Loaded HTML, optionally with link and image references made absolute.""" + text = self._raw_html_text + if not self._resolve_relative_urls or not text.strip(): + return text + root = etree.HTML(text) + if root is None: + return text + bases = root.xpath("//base[@href]") + base_url = self._document_url or "" + if bases: + base_url = urljoin(base_url, bases[0].get("href", "")) + if urlsplit(base_url).scheme.lower() not in ("http", "https"): + return text + for element in root.iter(): + if element.tag == "base": + continue + for attribute in ("href", "src", "data-src"): + value = element.get(attribute) + if value: + element.set(attribute, urljoin(base_url, value)) + return etree.tostring(root, encoding="unicode", method="html") + + @cached_property + def _raw_html_text(self) -> str: """The HTML document as a string, loaded from wherever the caller specified.""" if self._file_path: return read_txt_file(filename=self._file_path, encoding=self._encoding)[1] @@ -170,6 +203,7 @@ def html_text(self) -> str: if not content_type.startswith("text/html"): raise ValueError(f"Expected content type text/html. Got {content_type}.") + self._document_url = getattr(response, "url", None) or self._url return response.text raise ValueError("Exactly one of filename, file, text, or url must be specified.")