diff --git a/test_unstructured/partition/html/test_relative_urls.py b/test_unstructured/partition/html/test_relative_urls.py
new file mode 100644
index 0000000000..fa477b93cd
--- /dev/null
+++ b/test_unstructured/partition/html/test_relative_urls.py
@@ -0,0 +1,75 @@
+from types import SimpleNamespace
+
+import pytest
+
+from unstructured.partition.html import partition_html
+
+
+@pytest.mark.parametrize("parser_version", ["v1", "v2"])
+def test_opt_in_resolves_links_using_the_document_base(parser_version):
+ elements = partition_html(
+ text='
'
+ '',
+ resolve_relative_urls=True,
+ html_parser_version=parser_version,
+ languages=["eng"],
+ )
+
+ element = next(e for e in elements if e.text == "Guide")
+ if parser_version == "v1":
+ assert element.metadata.link_urls == ["https://example.com/guide?q=1#section"]
+ else:
+ assert 'href="https://example.com/guide?q=1#section"' in element.metadata.text_as_html
+
+
+def test_url_input_resolves_against_the_final_response_url(monkeypatch):
+ from unstructured.partition.html import partition as module
+
+ monkeypatch.setattr(
+ module,
+ "safe_get",
+ lambda *args, **kwargs: SimpleNamespace(
+ ok=True,
+ headers={"Content-Type": "text/html"},
+ url="https://example.com/final/article/",
+ text='Source
',
+ ),
+ )
+
+ element = partition_html(
+ url="https://example.com/old", resolve_relative_urls=True, languages=["eng"]
+ )[0]
+
+ assert element.metadata.link_urls == ["https://example.com/final/source"]
+
+
+def test_resolution_uses_only_the_first_base_and_keeps_absolute_schemes():
+ elements = partition_html(
+ text=''
+ 'Page Mail
'
+ '
',
+ resolve_relative_urls=True,
+ languages=["eng"],
+ )
+
+ assert elements[0].metadata.link_urls == ["https://example.com/a/page", "mailto:a@example.com"]
+ assert elements[1].metadata.image_url == "https://example.com/image.png"
+
+
+@pytest.mark.parametrize("resolve", [False, True])
+def test_without_a_base_url_relative_links_remain_relative(resolve):
+ element = partition_html(
+ text='Source
',
+ resolve_relative_urls=resolve,
+ languages=["eng"],
+ )[0]
+ assert element.metadata.link_urls == ["source"]
+
+
+def test_existing_default_keeps_relative_link_values():
+ element = partition_html(
+ text='Source
',
+ languages=["eng"],
+ )[0]
+ assert element.metadata.link_urls == ["source"]
diff --git a/unstructured/partition/html/partition.py b/unstructured/partition/html/partition.py
index 847cc836b8..a4eee0c1f0 100644
--- a/unstructured/partition/html/partition.py
+++ b/unstructured/partition/html/partition.py
@@ -6,6 +6,7 @@
from functools import cached_property
from typing import IO, Any, Callable, Iterator, List, Literal, Optional, cast
+from urllib.parse import urljoin, urlsplit
from lxml import etree
@@ -45,6 +46,7 @@ def partition_html(
languages: Optional[list[str]] = None,
detect_language_per_element: bool = False,
language_fallback: Optional[Callable[[str], Optional[list[str]]]] = None,
+ resolve_relative_urls: bool = False,
**kwargs: Any,
) -> list[Element]:
"""Partitions an HTML document into its constituent elements.
@@ -85,6 +87,9 @@ def partition_html(
language_fallback
Optional callable for short text; called with the text, return ISO 639-3 codes or None.
Called only when ``languages`` contains "auto"; explicit languages are kept for short text.
+ resolve_relative_urls
+ Resolve link and image URLs against the first HTML base URL or the final response URL.
+ Defaults to False, preserving relative values. Resolution does not fetch linked resources.
"""
# -- parser rejects an empty str, nip that edge-case in the bud here --
if text is not None and text.strip() == "" and not file and not filename and not url:
@@ -104,6 +109,7 @@ def partition_html(
image_alt_mode=image_alt_mode,
extract_image_block_types=extract_image_block_types,
extract_image_block_to_payload=extract_image_block_to_payload,
+ resolve_relative_urls=resolve_relative_urls,
)
return list(_HtmlPartitioner.iter_elements(opts))
@@ -128,6 +134,7 @@ def __init__(
image_alt_mode: Optional[Literal["to_text"]] = "to_text",
extract_image_block_types: Optional[list[str]] = None,
extract_image_block_to_payload: bool = False,
+ resolve_relative_urls: bool = False,
):
self._file_path = file_path
self._file = file
@@ -142,6 +149,8 @@ def __init__(
self._image_alt_mode = image_alt_mode
self._extract_image_block_types = extract_image_block_types
self._extract_image_block_to_payload = extract_image_block_to_payload
+ self._resolve_relative_urls = resolve_relative_urls
+ self._document_url = url
@cached_property
def detection_origin(self) -> str | None:
@@ -150,6 +159,30 @@ def detection_origin(self) -> str | None:
@cached_property
def html_text(self) -> str:
+ """Loaded HTML, optionally with link and image references made absolute."""
+ text = self._raw_html_text
+ if not self._resolve_relative_urls or not text.strip():
+ return text
+ root = etree.HTML(text)
+ if root is None:
+ return text
+ bases = root.xpath("//base[@href]")
+ base_url = self._document_url or ""
+ if bases:
+ base_url = urljoin(base_url, bases[0].get("href", ""))
+ if urlsplit(base_url).scheme.lower() not in ("http", "https"):
+ return text
+ for element in root.iter():
+ if element.tag == "base":
+ continue
+ for attribute in ("href", "src", "data-src"):
+ value = element.get(attribute)
+ if value:
+ element.set(attribute, urljoin(base_url, value))
+ return etree.tostring(root, encoding="unicode", method="html")
+
+ @cached_property
+ def _raw_html_text(self) -> str:
"""The HTML document as a string, loaded from wherever the caller specified."""
if self._file_path:
return read_txt_file(filename=self._file_path, encoding=self._encoding)[1]
@@ -170,6 +203,7 @@ def html_text(self) -> str:
if not content_type.startswith("text/html"):
raise ValueError(f"Expected content type text/html. Got {content_type}.")
+ self._document_url = getattr(response, "url", None) or self._url
return response.text
raise ValueError("Exactly one of filename, file, text, or url must be specified.")