From 33a4bcce67dca09da7254370c40d9989ad2112b1 Mon Sep 17 00:00:00 2001 From: Gagan baghel <78648366+gagan-baghel@users.noreply.github.com> Date: Sat, 21 Nov 2026 12:30:49 +0530 Subject: [PATCH] fix(html): leave ruby text out of the paragraph text MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit and had no element class, so they were transparent and their text was glued to the base text: "東京とうきょう" became "東京とうきょう". Skip them like other content that is not shown as body text. --- test_unstructured/partition/html/test_parser.py | 9 +++++++++ unstructured/partition/html/parser.py | 2 ++ 2 files changed, 11 insertions(+) diff --git a/test_unstructured/partition/html/test_parser.py b/test_unstructured/partition/html/test_parser.py index 37014f18f3..fe18e38936 100644 --- a/test_unstructured/partition/html/test_parser.py +++ b/test_unstructured/partition/html/test_parser.py @@ -1595,6 +1595,15 @@ def it_is_used_for_elements_whose_content_is_not_rendered_as_text(self, html_tex assert isinstance(div[0], RemovedPhrasing) assert [e.text for e in div.iter_elements()] == ["Before after."] + def and_it_is_used_for_ruby_text_so_it_is_not_glued_to_the_base_text(self): + html = ( + "

首都は東京(とうきょう)です。" + "漢かん字じ

" + ) + p = etree.fromstring(html, html_parser).xpath(".//p")[0] + + assert [e.text for e in p.iter_elements()] == ["首都は東京です。漢字"] + def and_it_is_used_for_head_which_is_only_reached_when_there_is_no_body(self): root = etree.fromstring("Metadata", html_parser) diff --git a/unstructured/partition/html/parser.py b/unstructured/partition/html/parser.py index 30c45e836e..9a693ab9bd 100644 --- a/unstructured/partition/html/parser.py +++ b/unstructured/partition/html/parser.py @@ -1218,6 +1218,8 @@ def derive_element_type_from_text(text: str) -> type[Text] | None: "noembed": RemovedPhrasing, # -- obsolete; fallback content for `` -- "noframes": RemovedPhrasing, # -- obsolete; fallback content for frames -- "object": RemovedPhrasing, # -- content is fallback for when the object can't be shown -- + "rp": RemovedPhrasing, # -- parentheses shown around `` only without ruby support -- + "rt": RemovedPhrasing, # -- ruby text, a reading aid like furigana for the base text -- "select": RemovedPhrasing, # -- form control; its option list is not document text -- "svg": RemovedPhrasing, # -- graphics; `` and `<desc>` inside it are metadata -- "textarea": RemovedPhrasing, # -- form control; its content is the editable value --