diff --git a/test_unstructured/nlp/test_tokenize.py b/test_unstructured/nlp/test_tokenize.py index c58fbe5c77..da4746a60a 100644 --- a/test_unstructured/nlp/test_tokenize.py +++ b/test_unstructured/nlp/test_tokenize.py @@ -153,3 +153,14 @@ def __call__(self, text, *args, **kwargs): monkeypatch.setattr(tokenize, "_get_nlp", lambda: CountingNlp()) return runs + + +def test_sent_tokenize_splits_on_danda_and_double_danda(): + assert tokenize.sent_tokenize("हिंदी सबसे अधिक बोली जाने वाली भाषा है। गंगा पवित्र नदी है।") == [ + "हिंदी सबसे अधिक बोली जाने वाली भाषा है।", + "गंगा पवित्र नदी है।", + ] + assert tokenize.sent_tokenize("राम वन को गए॥ सीता साथ गईं॥") == [ + "राम वन को गए॥", + "सीता साथ गईं॥", + ] diff --git a/test_unstructured/partition/test_text.py b/test_unstructured/partition/test_text.py index 91ebaf85a9..0fa2c742f4 100644 --- a/test_unstructured/partition/test_text.py +++ b/test_unstructured/partition/test_text.py @@ -19,6 +19,7 @@ EmailAddress, ListItem, NarrativeText, + Text, Title, ) from unstructured.file_utils.model import FileType @@ -722,3 +723,33 @@ def test_partition_text_detects_more_than_3_languages(): ) langs = [e.metadata.languages[0] for e in elements if e.metadata.languages] assert len(langs) > 10 + + +@pytest.mark.parametrize( + "paragraphs", + [ + [ + "भारत एक विशाल देश है। इसकी राजधानी नई दिल्ली है।", + "हिंदी सबसे अधिक बोली जाने वाली भाषा है। गंगा पवित्र नदी है।", + "हिमालय उत्तर में स्थित है। वह बहुत ऊँचा है।", + "मुंबई देश की आर्थिक राजधानी मानी जाती है। यहाँ फिल्में बनती हैं।", + "दिल्ली में सरकार की बैठकें होती हैं। संसद भी यहीं है।", + "किसान खेतों में मेहनत करते हैं। वे अनाज उगाते हैं।", + ], + [ + "বাংলা একটি সুন্দর ভাষা। এটি অনেক মানুষ বলে।", + "কলকাতা পশ্চিমবঙ্গের রাজধানী। এখানে অনেক মানুষ থাকে।", + "রবীন্দ্রনাথ ঠাকুর নোবেল পুরস্কার পেয়েছিলেন। তিনি কবি ছিলেন।", + ], + ], + ids=["hindi", "bengali"], +) +def test_partition_text_classifies_hindi_and_bengali_paragraphs_as_narrative_text(paragraphs): + elements = partition_text(text="\n\n".join(paragraphs)) + + assert elements == [NarrativeText(text=paragraph) for paragraph in paragraphs] + + +@pytest.mark.parametrize("text", ["⭐️⭐️⭐️⭐️⭐️", "1️⃣ 2️⃣ 3️⃣"]) +def test_partition_text_keeps_emoji_only_lines_as_text(text): + assert partition_text(text=text) == [Text(text=text)] diff --git a/test_unstructured/partition/test_text_type.py b/test_unstructured/partition/test_text_type.py index 98443df080..a84de0cdb1 100644 --- a/test_unstructured/partition/test_text_type.py +++ b/test_unstructured/partition/test_text_type.py @@ -1,3 +1,4 @@ +import tracemalloc from unittest.mock import patch import pytest @@ -336,3 +337,83 @@ def test_is_email_address(text, expected): def test_under_non_alpha_ratio_zero_divide(): # Threw an error before changes text_type.under_non_alpha_ratio(" ") + + +# -- Hindi and Bengali: vowel signs are Unicode combining marks, not letters, and the danda (।) +# -- ends a sentence. -- +HINDI_PARAGRAPHS = [ + "भारत एक विशाल देश है। इसकी राजधानी नई दिल्ली है।", + "हिंदी सबसे अधिक बोली जाने वाली भाषा है। गंगा पवित्र नदी है।", + "हिमालय उत्तर में स्थित है। वह बहुत ऊँचा है।", + "मुंबई देश की आर्थिक राजधानी मानी जाती है। यहाँ फिल्में बनती हैं।", + "दिल्ली में सरकार की बैठकें होती हैं। संसद भी यहीं है।", + "किसान खेतों में मेहनत करते हैं। वे अनाज उगाते हैं।", +] +BENGALI_PARAGRAPHS = [ + "বাংলা একটি সুন্দর ভাষা। এটি অনেক মানুষ বলে।", + "কলকাতা পশ্চিমবঙ্গের রাজধানী। এখানে অনেক মানুষ থাকে।", + "রবীন্দ্রনাথ ঠাকুর নোবেল পুরস্কার পেয়েছিলেন। তিনি কবি ছিলেন।", +] + + +def test_under_non_alpha_ratio_counts_vowel_signs_as_letters(): + # -- "हिंदी" is 2 base letters and 3 combining marks -- + assert text_type.under_non_alpha_ratio("हिंदी") is False + + +@pytest.mark.parametrize("text", HINDI_PARAGRAPHS + BENGALI_PARAGRAPHS) +def test_under_non_alpha_ratio_is_false_for_hindi_and_bengali_paragraphs(text): + assert text_type.under_non_alpha_ratio(text) is False + + +def test_exceeds_cap_ratio_is_false_for_a_hindi_sentence_whose_words_all_have_vowel_signs(): + assert text_type.exceeds_cap_ratio("हिंदी सबसे अधिक बोली जाने वाली भाषा है।") is False + + +def test_sentence_count_splits_hindi_sentences_on_danda(): + text = "हिंदी सबसे अधिक बोली जाने वाली भाषा है। गंगा पवित्र नदी है।" + assert text_type.sentence_count(text) == 2 + + +@pytest.mark.parametrize("text", HINDI_PARAGRAPHS + BENGALI_PARAGRAPHS) +def test_is_possible_narrative_text_for_hindi_and_bengali_paragraphs(text): + assert text_type.is_possible_narrative_text(text) is True + + +@pytest.mark.parametrize("text", ["हिंदी", "परिचय"]) +def test_is_possible_title_for_short_hindi_headings(text): + assert text_type.is_possible_title(text) is True + + +# -- Emoji carry combining marks too: U+FE0F (variation selector, Mn) and U+20E3 (keycap, Me). +# -- They are not attached to a letter, so they do not count as letters. -- +EMOJI_LINES = ["⭐️⭐️⭐️⭐️⭐️", "1️⃣ 2️⃣ 3️⃣", "✅️ ✅️ ✅️", "⚠️"] + + +@pytest.mark.parametrize("text", EMOJI_LINES) +def test_under_non_alpha_ratio_does_not_count_emoji_marks_as_letters(text): + assert text_type.under_non_alpha_ratio(text) is True + + +@pytest.mark.parametrize("text", EMOJI_LINES) +def test_is_possible_title_is_false_for_emoji_lines(text): + assert text_type.is_possible_title(text) is False + + +def test_letter_checks_use_bounded_memory_on_long_input(): + # -- about 5 MB of mixed Hindi, Latin with combining accents, emoji and punctuation -- + unit = "हिंदी सबसे अधिक बोली जाने वाली भाषा है। Café e\u0301 ⭐️ 1️⃣ 42, — " + text = unit * (5_000_000 // len(unit.encode())) + word = "हिंदी" * 200_000 + + tracemalloc.start() + try: + ratio_result = text_type.under_non_alpha_ratio(text) + word_result = text_type._is_word(word) + _, peak = tracemalloc.get_traced_memory() + finally: + tracemalloc.stop() + + assert ratio_result is False + assert word_result is True + assert peak < 100_000 diff --git a/unstructured/nlp/tokenize.py b/unstructured/nlp/tokenize.py index 5bcf60a5d7..496ed3ef1a 100644 --- a/unstructured/nlp/tokenize.py +++ b/unstructured/nlp/tokenize.py @@ -4,6 +4,7 @@ import importlib import logging import os +import re import shutil import sys import sysconfig @@ -23,6 +24,11 @@ # -- than UTF-8 width. Element text is a paragraph or a table cell, so 8,192 covers it. -- MAX_CACHEABLE_CHARS: Final[int] = 8192 +# NOTE(shlokbam): The English spaCy model does not reliably treat the danda (।) and double +# danda (॥), the sentence terminators of Hindi, Bengali and other Indic scripts, as sentence +# ends, so split on them after spaCy's own sentence segmentation. +_DANDA_SPLIT_RE: Final[re.Pattern[str]] = re.compile(r"(?<=[।॥])\s+") + _SPACY_MODEL_NAME: Final[str] = "en_core_web_sm" _SPACY_MODEL_VERSION: Final[str] = "3.8.0" _SPACY_MODEL_URL: Final[str] = ( @@ -214,4 +220,6 @@ def pos_tag(text: str) -> List[Tuple[str, str]]: @lru_cache(maxsize=CACHE_MAX_SIZE) def _tokenize_for_cache(text: str) -> Tuple[str, ...]: """A wrapper around the spaCy sentence tokenizer with LRU caching enabled.""" - return tuple(sent.text for sent in _process(text).sents) + return tuple( + part for sent in _process(text).sents for part in _DANDA_SPLIT_RE.split(sent.text) if part + ) diff --git a/unstructured/partition/text_type.py b/unstructured/partition/text_type.py index 79f45019d6..b3fef5327f 100644 --- a/unstructured/partition/text_type.py +++ b/unstructured/partition/text_type.py @@ -4,7 +4,8 @@ import os import re -from typing import Final, List, Optional +import unicodedata +from typing import Final, Iterator, List, Optional from unstructured.cleaners.core import remove_punctuation from unstructured.logger import trace_logger @@ -24,6 +25,36 @@ NON_LOWERCASE_ALPHA_RE = re.compile(r"[^a-z]") +# NOTE(shlokbam): Vowel signs, viramas and anusvaras in Indic scripts such as Devanagari and +# Bengali are combining marks, not letters, so `"हिंदी".isalpha()` is False. Treat them as +# part of the word they belong to. A mark counts only when it is attached to a letter, so the +# variation selector (U+FE0F) and keycap (U+20E3) marks in emoji like "⭐️" and "1️⃣" do not. +def _iter_letter_flags(text: str) -> Iterator[bool]: + """Yield, for each non-space character of `text`, whether it counts as a letter. + + A letter counts. A combining mark (Unicode category M*) counts only when the previous + non-space character counted. Anything else does not count. Only that one flag is kept, so + memory use does not grow with the length of `text`. + """ + category = unicodedata.category + counts = False + for char in text: + if char.isspace(): + continue + if char.isalpha(): + counts = True + elif category(char)[0] != "M": + counts = False + yield counts + + +def _is_word(token: str) -> bool: + """True if `token` is letters and the combining marks attached to them, starting with a + letter.""" + flags = _iter_letter_flags(token) + return next(flags, False) and all(flags) + + def is_possible_narrative_text( text: str, cap_threshold: float = 0.5, @@ -256,11 +287,9 @@ def under_non_alpha_ratio(text: str, threshold: float = 0.5): alpha_count = 0 total_count = 0 - for char in text: - if not char.isspace(): - total_count += 1 - if char.isalpha(): - alpha_count += 1 + for counts in _iter_letter_flags(text): + total_count += 1 + alpha_count += counts return ((alpha_count / total_count) < threshold) if total_count > 0 else False @@ -291,7 +320,7 @@ def exceeds_cap_ratio(text: str, threshold: float = 0.5) -> bool: # ex. world_tokenize("ITEM 1. Financial Statements (Unaudited)") # = ['ITEM', '1', '.', 'Financial', 'Statements', '(', 'Unaudited', ')'], # however, "ITEM 1. Financial Statements (Unaudited)" is Title, not NarrativeText - tokens = [tk for tk in word_tokenize(text) if tk.isalpha()] + tokens = [tk for tk in word_tokenize(text) if _is_word(tk)] # NOTE(jay-ylee) - If word_tokenize(text) is empty, return must be True to # avoid being misclassified as Narrative Text.