From 6544f77d03fae7fe89d777a6639f73face4b5dae Mon Sep 17 00:00:00 2001 From: Shlok Bam Date: Wed, 11 Nov 2026 15:08:06 +0530 Subject: [PATCH 1/3] fix: classify Hindi/Bengali paragraphs as narrative text (count vowel signs as letters, split sentences on danda) --- test_unstructured/nlp/test_tokenize.py | 11 +++++ test_unstructured/partition/test_text.py | 25 ++++++++++ test_unstructured/partition/test_text_type.py | 46 +++++++++++++++++++ unstructured/nlp/tokenize.py | 10 +++- unstructured/partition/text_type.py | 18 +++++++- 5 files changed, 107 insertions(+), 3 deletions(-) diff --git a/test_unstructured/nlp/test_tokenize.py b/test_unstructured/nlp/test_tokenize.py index c58fbe5c77..da4746a60a 100644 --- a/test_unstructured/nlp/test_tokenize.py +++ b/test_unstructured/nlp/test_tokenize.py @@ -153,3 +153,14 @@ def __call__(self, text, *args, **kwargs): monkeypatch.setattr(tokenize, "_get_nlp", lambda: CountingNlp()) return runs + + +def test_sent_tokenize_splits_on_danda_and_double_danda(): + assert tokenize.sent_tokenize("हिंदी सबसे अधिक बोली जाने वाली भाषा है। गंगा पवित्र नदी है।") == [ + "हिंदी सबसे अधिक बोली जाने वाली भाषा है।", + "गंगा पवित्र नदी है।", + ] + assert tokenize.sent_tokenize("राम वन को गए॥ सीता साथ गईं॥") == [ + "राम वन को गए॥", + "सीता साथ गईं॥", + ] diff --git a/test_unstructured/partition/test_text.py b/test_unstructured/partition/test_text.py index 91ebaf85a9..66ce722457 100644 --- a/test_unstructured/partition/test_text.py +++ b/test_unstructured/partition/test_text.py @@ -722,3 +722,28 @@ def test_partition_text_detects_more_than_3_languages(): ) langs = [e.metadata.languages[0] for e in elements if e.metadata.languages] assert len(langs) > 10 + + +@pytest.mark.parametrize( + "paragraphs", + [ + [ + "भारत एक विशाल देश है। इसकी राजधानी नई दिल्ली है।", + "हिंदी सबसे अधिक बोली जाने वाली भाषा है। गंगा पवित्र नदी है।", + "हिमालय उत्तर में स्थित है। वह बहुत ऊँचा है।", + "मुंबई देश की आर्थिक राजधानी मानी जाती है। यहाँ फिल्में बनती हैं।", + "दिल्ली में सरकार की बैठकें होती हैं। संसद भी यहीं है।", + "किसान खेतों में मेहनत करते हैं। वे अनाज उगाते हैं।", + ], + [ + "বাংলা একটি সুন্দর ভাষা। এটি অনেক মানুষ বলে।", + "কলকাতা পশ্চিমবঙ্গের রাজধানী। এখানে অনেক মানুষ থাকে।", + "রবীন্দ্রনাথ ঠাকুর নোবেল পুরস্কার পেয়েছিলেন। তিনি কবি ছিলেন।", + ], + ], + ids=["hindi", "bengali"], +) +def test_partition_text_classifies_hindi_and_bengali_paragraphs_as_narrative_text(paragraphs): + elements = partition_text(text="\n\n".join(paragraphs)) + + assert elements == [NarrativeText(text=paragraph) for paragraph in paragraphs] diff --git a/test_unstructured/partition/test_text_type.py b/test_unstructured/partition/test_text_type.py index 98443df080..b3d622c20d 100644 --- a/test_unstructured/partition/test_text_type.py +++ b/test_unstructured/partition/test_text_type.py @@ -336,3 +336,49 @@ def test_is_email_address(text, expected): def test_under_non_alpha_ratio_zero_divide(): # Threw an error before changes text_type.under_non_alpha_ratio(" ") + + +# -- Hindi and Bengali: vowel signs are Unicode combining marks, not letters, and the danda (।) +# -- ends a sentence. -- +HINDI_PARAGRAPHS = [ + "भारत एक विशाल देश है। इसकी राजधानी नई दिल्ली है।", + "हिंदी सबसे अधिक बोली जाने वाली भाषा है। गंगा पवित्र नदी है।", + "हिमालय उत्तर में स्थित है। वह बहुत ऊँचा है।", + "मुंबई देश की आर्थिक राजधानी मानी जाती है। यहाँ फिल्में बनती हैं।", + "दिल्ली में सरकार की बैठकें होती हैं। संसद भी यहीं है।", + "किसान खेतों में मेहनत करते हैं। वे अनाज उगाते हैं।", +] +BENGALI_PARAGRAPHS = [ + "বাংলা একটি সুন্দর ভাষা। এটি অনেক মানুষ বলে।", + "কলকাতা পশ্চিমবঙ্গের রাজধানী। এখানে অনেক মানুষ থাকে।", + "রবীন্দ্রনাথ ঠাকুর নোবেল পুরস্কার পেয়েছিলেন। তিনি কবি ছিলেন।", +] + + +def test_under_non_alpha_ratio_counts_vowel_signs_as_letters(): + # -- "हिंदी" is 2 base letters and 3 combining marks -- + assert text_type.under_non_alpha_ratio("हिंदी") is False + + +@pytest.mark.parametrize("text", HINDI_PARAGRAPHS + BENGALI_PARAGRAPHS) +def test_under_non_alpha_ratio_is_false_for_hindi_and_bengali_paragraphs(text): + assert text_type.under_non_alpha_ratio(text) is False + + +def test_exceeds_cap_ratio_is_false_for_a_hindi_sentence_whose_words_all_have_vowel_signs(): + assert text_type.exceeds_cap_ratio("हिंदी सबसे अधिक बोली जाने वाली भाषा है।") is False + + +def test_sentence_count_splits_hindi_sentences_on_danda(): + text = "हिंदी सबसे अधिक बोली जाने वाली भाषा है। गंगा पवित्र नदी है।" + assert text_type.sentence_count(text) == 2 + + +@pytest.mark.parametrize("text", HINDI_PARAGRAPHS + BENGALI_PARAGRAPHS) +def test_is_possible_narrative_text_for_hindi_and_bengali_paragraphs(text): + assert text_type.is_possible_narrative_text(text) is True + + +@pytest.mark.parametrize("text", ["हिंदी", "परिचय"]) +def test_is_possible_title_for_short_hindi_headings(text): + assert text_type.is_possible_title(text) is True diff --git a/unstructured/nlp/tokenize.py b/unstructured/nlp/tokenize.py index 5bcf60a5d7..496ed3ef1a 100644 --- a/unstructured/nlp/tokenize.py +++ b/unstructured/nlp/tokenize.py @@ -4,6 +4,7 @@ import importlib import logging import os +import re import shutil import sys import sysconfig @@ -23,6 +24,11 @@ # -- than UTF-8 width. Element text is a paragraph or a table cell, so 8,192 covers it. -- MAX_CACHEABLE_CHARS: Final[int] = 8192 +# NOTE(shlokbam): The English spaCy model does not reliably treat the danda (।) and double +# danda (॥), the sentence terminators of Hindi, Bengali and other Indic scripts, as sentence +# ends, so split on them after spaCy's own sentence segmentation. +_DANDA_SPLIT_RE: Final[re.Pattern[str]] = re.compile(r"(?<=[।॥])\s+") + _SPACY_MODEL_NAME: Final[str] = "en_core_web_sm" _SPACY_MODEL_VERSION: Final[str] = "3.8.0" _SPACY_MODEL_URL: Final[str] = ( @@ -214,4 +220,6 @@ def pos_tag(text: str) -> List[Tuple[str, str]]: @lru_cache(maxsize=CACHE_MAX_SIZE) def _tokenize_for_cache(text: str) -> Tuple[str, ...]: """A wrapper around the spaCy sentence tokenizer with LRU caching enabled.""" - return tuple(sent.text for sent in _process(text).sents) + return tuple( + part for sent in _process(text).sents for part in _DANDA_SPLIT_RE.split(sent.text) if part + ) diff --git a/unstructured/partition/text_type.py b/unstructured/partition/text_type.py index 79f45019d6..ff612d0cff 100644 --- a/unstructured/partition/text_type.py +++ b/unstructured/partition/text_type.py @@ -4,6 +4,7 @@ import os import re +import unicodedata from typing import Final, List, Optional from unstructured.cleaners.core import remove_punctuation @@ -24,6 +25,19 @@ NON_LOWERCASE_ALPHA_RE = re.compile(r"[^a-z]") +# NOTE(shlokbam): Vowel signs, viramas and anusvaras in Indic scripts such as Devanagari and +# Bengali are combining marks, not letters, so `"हिंदी".isalpha()` is False. Treat them as +# part of the word they belong to. +def _is_letter_or_mark(char: str) -> bool: + """True if `char` is a letter or a combining mark (Unicode category M*).""" + return char.isalpha() or unicodedata.category(char).startswith("M") + + +def _is_word(token: str) -> bool: + """True if `token` has at least one letter and consists only of letters and combining marks.""" + return any(char.isalpha() for char in token) and all(_is_letter_or_mark(char) for char in token) + + def is_possible_narrative_text( text: str, cap_threshold: float = 0.5, @@ -259,7 +273,7 @@ def under_non_alpha_ratio(text: str, threshold: float = 0.5): for char in text: if not char.isspace(): total_count += 1 - if char.isalpha(): + if char.isalpha() or unicodedata.category(char)[0] == "M": alpha_count += 1 return ((alpha_count / total_count) < threshold) if total_count > 0 else False @@ -291,7 +305,7 @@ def exceeds_cap_ratio(text: str, threshold: float = 0.5) -> bool: # ex. world_tokenize("ITEM 1. Financial Statements (Unaudited)") # = ['ITEM', '1', '.', 'Financial', 'Statements', '(', 'Unaudited', ')'], # however, "ITEM 1. Financial Statements (Unaudited)" is Title, not NarrativeText - tokens = [tk for tk in word_tokenize(text) if tk.isalpha()] + tokens = [tk for tk in word_tokenize(text) if _is_word(tk)] # NOTE(jay-ylee) - If word_tokenize(text) is empty, return must be True to # avoid being misclassified as Narrative Text. From ffae7b8aacde0d8aaad750fc50247842763b7091 Mon Sep 17 00:00:00 2001 From: Shlok Bam Date: Mon, 16 Nov 2026 17:12:09 +0530 Subject: [PATCH 2/3] fix: count a combining mark as a letter only when it is attached to a letter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Counting every Unicode mark (M*) as a letter also counted the variation selector U+FE0F and the keycap mark U+20E3, so emoji-only lines like "⭐️⭐️⭐️⭐️⭐️" or "1️⃣ 2️⃣ 3️⃣" passed under_non_alpha_ratio and were classified as Title. Add one helper, _letter_flags, that counts a mark only when the previous non-space character is a letter or a counted mark, and use it in both _is_word and under_non_alpha_ratio. Indic vowel signs and viramas still count; emoji marks do not. --- test_unstructured/partition/test_text.py | 6 +++ test_unstructured/partition/test_text_type.py | 15 +++++++ unstructured/partition/text_type.py | 40 ++++++++++++------- 3 files changed, 46 insertions(+), 15 deletions(-) diff --git a/test_unstructured/partition/test_text.py b/test_unstructured/partition/test_text.py index 66ce722457..0fa2c742f4 100644 --- a/test_unstructured/partition/test_text.py +++ b/test_unstructured/partition/test_text.py @@ -19,6 +19,7 @@ EmailAddress, ListItem, NarrativeText, + Text, Title, ) from unstructured.file_utils.model import FileType @@ -747,3 +748,8 @@ def test_partition_text_classifies_hindi_and_bengali_paragraphs_as_narrative_tex elements = partition_text(text="\n\n".join(paragraphs)) assert elements == [NarrativeText(text=paragraph) for paragraph in paragraphs] + + +@pytest.mark.parametrize("text", ["⭐️⭐️⭐️⭐️⭐️", "1️⃣ 2️⃣ 3️⃣"]) +def test_partition_text_keeps_emoji_only_lines_as_text(text): + assert partition_text(text=text) == [Text(text=text)] diff --git a/test_unstructured/partition/test_text_type.py b/test_unstructured/partition/test_text_type.py index b3d622c20d..e20c73cbd7 100644 --- a/test_unstructured/partition/test_text_type.py +++ b/test_unstructured/partition/test_text_type.py @@ -382,3 +382,18 @@ def test_is_possible_narrative_text_for_hindi_and_bengali_paragraphs(text): @pytest.mark.parametrize("text", ["हिंदी", "परिचय"]) def test_is_possible_title_for_short_hindi_headings(text): assert text_type.is_possible_title(text) is True + + +# -- Emoji carry combining marks too: U+FE0F (variation selector, Mn) and U+20E3 (keycap, Me). +# -- They are not attached to a letter, so they do not count as letters. -- +EMOJI_LINES = ["⭐️⭐️⭐️⭐️⭐️", "1️⃣ 2️⃣ 3️⃣", "✅️ ✅️ ✅️", "⚠️"] + + +@pytest.mark.parametrize("text", EMOJI_LINES) +def test_under_non_alpha_ratio_does_not_count_emoji_marks_as_letters(text): + assert text_type.under_non_alpha_ratio(text) is True + + +@pytest.mark.parametrize("text", EMOJI_LINES) +def test_is_possible_title_is_false_for_emoji_lines(text): + assert text_type.is_possible_title(text) is False diff --git a/unstructured/partition/text_type.py b/unstructured/partition/text_type.py index ff612d0cff..990f9b64eb 100644 --- a/unstructured/partition/text_type.py +++ b/unstructured/partition/text_type.py @@ -27,15 +27,32 @@ # NOTE(shlokbam): Vowel signs, viramas and anusvaras in Indic scripts such as Devanagari and # Bengali are combining marks, not letters, so `"हिंदी".isalpha()` is False. Treat them as -# part of the word they belong to. -def _is_letter_or_mark(char: str) -> bool: - """True if `char` is a letter or a combining mark (Unicode category M*).""" - return char.isalpha() or unicodedata.category(char).startswith("M") +# part of the word they belong to. A mark counts only when it is attached to a letter, so the +# variation selector (U+FE0F) and keycap (U+20E3) marks in emoji like "⭐️" and "1️⃣" do not. +def _letter_flags(text: str) -> List[bool]: + """For each non-space character of `text`, whether it counts as a letter. + + A letter counts. A combining mark (Unicode category M*) counts only when the previous + non-space character is a letter or a mark that counts. Anything else does not count. + """ + flags: List[bool] = [] + for char in text: + if char.isspace(): + continue + if char.isalpha(): + flags.append(True) + elif unicodedata.category(char).startswith("M"): + flags.append(bool(flags) and flags[-1]) + else: + flags.append(False) + return flags def _is_word(token: str) -> bool: - """True if `token` has at least one letter and consists only of letters and combining marks.""" - return any(char.isalpha() for char in token) and all(_is_letter_or_mark(char) for char in token) + """True if `token` is letters and the combining marks attached to them, starting with a + letter.""" + flags = _letter_flags(token) + return bool(flags) and all(flags) def is_possible_narrative_text( @@ -268,15 +285,8 @@ def under_non_alpha_ratio(text: str, threshold: float = 0.5): if not text: return False - alpha_count = 0 - total_count = 0 - for char in text: - if not char.isspace(): - total_count += 1 - if char.isalpha() or unicodedata.category(char)[0] == "M": - alpha_count += 1 - - return ((alpha_count / total_count) < threshold) if total_count > 0 else False + flags = _letter_flags(text) + return (sum(flags) / len(flags) < threshold) if flags else False def exceeds_cap_ratio(text: str, threshold: float = 0.5) -> bool: From 72cbaf34c2e8c911aa1a73f9dd4e076f9872ceb5 Mon Sep 17 00:00:00 2001 From: Shlok Bam Date: Mon, 16 Nov 2026 21:31:30 +0530 Subject: [PATCH 3/3] fix: keep the letter checks' memory use bounded on long text _letter_flags built a list with an entry for every non-space character, and under_non_alpha_ratio built the whole list before counting it. Large extracted text blocks can reach this check, so the allocation grew with the input. Replace it with _iter_letter_flags, a generator that keeps only whether the previous non-space character counted. under_non_alpha_ratio uses two running counters again and _is_word stops at the first character that does not count. Results are unchanged. --- test_unstructured/partition/test_text_type.py | 20 +++++++++++ unstructured/partition/text_type.py | 35 +++++++++++-------- 2 files changed, 40 insertions(+), 15 deletions(-) diff --git a/test_unstructured/partition/test_text_type.py b/test_unstructured/partition/test_text_type.py index e20c73cbd7..a84de0cdb1 100644 --- a/test_unstructured/partition/test_text_type.py +++ b/test_unstructured/partition/test_text_type.py @@ -1,3 +1,4 @@ +import tracemalloc from unittest.mock import patch import pytest @@ -397,3 +398,22 @@ def test_under_non_alpha_ratio_does_not_count_emoji_marks_as_letters(text): @pytest.mark.parametrize("text", EMOJI_LINES) def test_is_possible_title_is_false_for_emoji_lines(text): assert text_type.is_possible_title(text) is False + + +def test_letter_checks_use_bounded_memory_on_long_input(): + # -- about 5 MB of mixed Hindi, Latin with combining accents, emoji and punctuation -- + unit = "हिंदी सबसे अधिक बोली जाने वाली भाषा है। Café e\u0301 ⭐️ 1️⃣ 42, — " + text = unit * (5_000_000 // len(unit.encode())) + word = "हिंदी" * 200_000 + + tracemalloc.start() + try: + ratio_result = text_type.under_non_alpha_ratio(text) + word_result = text_type._is_word(word) + _, peak = tracemalloc.get_traced_memory() + finally: + tracemalloc.stop() + + assert ratio_result is False + assert word_result is True + assert peak < 100_000 diff --git a/unstructured/partition/text_type.py b/unstructured/partition/text_type.py index 990f9b64eb..b3fef5327f 100644 --- a/unstructured/partition/text_type.py +++ b/unstructured/partition/text_type.py @@ -5,7 +5,7 @@ import os import re import unicodedata -from typing import Final, List, Optional +from typing import Final, Iterator, List, Optional from unstructured.cleaners.core import remove_punctuation from unstructured.logger import trace_logger @@ -29,30 +29,30 @@ # Bengali are combining marks, not letters, so `"हिंदी".isalpha()` is False. Treat them as # part of the word they belong to. A mark counts only when it is attached to a letter, so the # variation selector (U+FE0F) and keycap (U+20E3) marks in emoji like "⭐️" and "1️⃣" do not. -def _letter_flags(text: str) -> List[bool]: - """For each non-space character of `text`, whether it counts as a letter. +def _iter_letter_flags(text: str) -> Iterator[bool]: + """Yield, for each non-space character of `text`, whether it counts as a letter. A letter counts. A combining mark (Unicode category M*) counts only when the previous - non-space character is a letter or a mark that counts. Anything else does not count. + non-space character counted. Anything else does not count. Only that one flag is kept, so + memory use does not grow with the length of `text`. """ - flags: List[bool] = [] + category = unicodedata.category + counts = False for char in text: if char.isspace(): continue if char.isalpha(): - flags.append(True) - elif unicodedata.category(char).startswith("M"): - flags.append(bool(flags) and flags[-1]) - else: - flags.append(False) - return flags + counts = True + elif category(char)[0] != "M": + counts = False + yield counts def _is_word(token: str) -> bool: """True if `token` is letters and the combining marks attached to them, starting with a letter.""" - flags = _letter_flags(token) - return bool(flags) and all(flags) + flags = _iter_letter_flags(token) + return next(flags, False) and all(flags) def is_possible_narrative_text( @@ -285,8 +285,13 @@ def under_non_alpha_ratio(text: str, threshold: float = 0.5): if not text: return False - flags = _letter_flags(text) - return (sum(flags) / len(flags) < threshold) if flags else False + alpha_count = 0 + total_count = 0 + for counts in _iter_letter_flags(text): + total_count += 1 + alpha_count += counts + + return ((alpha_count / total_count) < threshold) if total_count > 0 else False def exceeds_cap_ratio(text: str, threshold: float = 0.5) -> bool: