Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
75 changes: 75 additions & 0 deletions test_unstructured/partition/html/test_relative_urls.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
from types import SimpleNamespace

import pytest

from unstructured.partition.html import partition_html


@pytest.mark.parametrize("parser_version", ["v1", "v2"])
def test_opt_in_resolves_links_using_the_document_base(parser_version):
elements = partition_html(
text='<head><base href="https://example.com/docs/"></head>'
'<body class="Document"><div class="Page"><p class="NarrativeText">'
'<a href="../guide?q=1#section">Guide</a></p></div></body>',
resolve_relative_urls=True,
html_parser_version=parser_version,
languages=["eng"],
)

element = next(e for e in elements if e.text == "Guide")
if parser_version == "v1":
assert element.metadata.link_urls == ["https://example.com/guide?q=1#section"]
else:
assert 'href="https://example.com/guide?q=1#section"' in element.metadata.text_as_html


def test_url_input_resolves_against_the_final_response_url(monkeypatch):
from unstructured.partition.html import partition as module

monkeypatch.setattr(
module,
"safe_get",
lambda *args, **kwargs: SimpleNamespace(
ok=True,
headers={"Content-Type": "text/html"},
url="https://example.com/final/article/",
text='<p><a href="../source">Source</a></p>',
),
)

element = partition_html(
url="https://example.com/old", resolve_relative_urls=True, languages=["eng"]
)[0]

assert element.metadata.link_urls == ["https://example.com/final/source"]


def test_resolution_uses_only_the_first_base_and_keeps_absolute_schemes():
elements = partition_html(
text='<head><base href="https://example.com/a/"><base href="https://other.test/"></head>'
'<body><p><a href="page">Page</a> <a href="mailto:a@example.com">Mail</a></p>'
'<img src="../image.png" alt="Image"></body>',
resolve_relative_urls=True,
languages=["eng"],
)

assert elements[0].metadata.link_urls == ["https://example.com/a/page", "mailto:a@example.com"]
assert elements[1].metadata.image_url == "https://example.com/image.png"


@pytest.mark.parametrize("resolve", [False, True])
def test_without_a_base_url_relative_links_remain_relative(resolve):
element = partition_html(
text='<p><a href="source">Source</a></p>',
resolve_relative_urls=resolve,
languages=["eng"],
)[0]
assert element.metadata.link_urls == ["source"]


def test_existing_default_keeps_relative_link_values():
element = partition_html(
text='<base href="https://example.com/"><p><a href="source">Source</a></p>',
languages=["eng"],
)[0]
assert element.metadata.link_urls == ["source"]
34 changes: 34 additions & 0 deletions unstructured/partition/html/partition.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@

from functools import cached_property
from typing import IO, Any, Callable, Iterator, List, Literal, Optional, cast
from urllib.parse import urljoin, urlsplit

from lxml import etree

Expand Down Expand Up @@ -45,6 +46,7 @@ def partition_html(
languages: Optional[list[str]] = None,
detect_language_per_element: bool = False,
language_fallback: Optional[Callable[[str], Optional[list[str]]]] = None,
resolve_relative_urls: bool = False,
**kwargs: Any,
) -> list[Element]:
"""Partitions an HTML document into its constituent elements.
Expand Down Expand Up @@ -85,6 +87,9 @@ def partition_html(
language_fallback
Optional callable for short text; called with the text, return ISO 639-3 codes or None.
Called only when ``languages`` contains "auto"; explicit languages are kept for short text.
resolve_relative_urls
Resolve link and image URLs against the first HTML base URL or the final response URL.
Defaults to False, preserving relative values. Resolution does not fetch linked resources.
"""
# -- parser rejects an empty str, nip that edge-case in the bud here --
if text is not None and text.strip() == "" and not file and not filename and not url:
Expand All @@ -104,6 +109,7 @@ def partition_html(
image_alt_mode=image_alt_mode,
extract_image_block_types=extract_image_block_types,
extract_image_block_to_payload=extract_image_block_to_payload,
resolve_relative_urls=resolve_relative_urls,
)

return list(_HtmlPartitioner.iter_elements(opts))
Expand All @@ -128,6 +134,7 @@ def __init__(
image_alt_mode: Optional[Literal["to_text"]] = "to_text",
extract_image_block_types: Optional[list[str]] = None,
extract_image_block_to_payload: bool = False,
resolve_relative_urls: bool = False,
):
self._file_path = file_path
self._file = file
Expand All @@ -142,6 +149,8 @@ def __init__(
self._image_alt_mode = image_alt_mode
self._extract_image_block_types = extract_image_block_types
self._extract_image_block_to_payload = extract_image_block_to_payload
self._resolve_relative_urls = resolve_relative_urls
self._document_url = url

@cached_property
def detection_origin(self) -> str | None:
Expand All @@ -150,6 +159,30 @@ def detection_origin(self) -> str | None:

@cached_property
def html_text(self) -> str:
"""Loaded HTML, optionally with link and image references made absolute."""
text = self._raw_html_text
if not self._resolve_relative_urls or not text.strip():
return text
root = etree.HTML(text)
if root is None:
return text
bases = root.xpath("//base[@href]")
base_url = self._document_url or ""
if bases:
base_url = urljoin(base_url, bases[0].get("href", ""))
if urlsplit(base_url).scheme.lower() not in ("http", "https"):
return text
for element in root.iter():
if element.tag == "base":
continue
for attribute in ("href", "src", "data-src"):
value = element.get(attribute)
if value:
element.set(attribute, urljoin(base_url, value))
return etree.tostring(root, encoding="unicode", method="html")

@cached_property
def _raw_html_text(self) -> str:
"""The HTML document as a string, loaded from wherever the caller specified."""
if self._file_path:
return read_txt_file(filename=self._file_path, encoding=self._encoding)[1]
Expand All @@ -170,6 +203,7 @@ def html_text(self) -> str:
if not content_type.startswith("text/html"):
raise ValueError(f"Expected content type text/html. Got {content_type}.")

self._document_url = getattr(response, "url", None) or self._url
return response.text

raise ValueError("Exactly one of filename, file, text, or url must be specified.")
Expand Down