Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions test_unstructured/nlp/test_tokenize.py
Original file line number Diff line number Diff line change
Expand Up @@ -153,3 +153,14 @@ def __call__(self, text, *args, **kwargs):

monkeypatch.setattr(tokenize, "_get_nlp", lambda: CountingNlp())
return runs


def test_sent_tokenize_splits_on_danda_and_double_danda():
assert tokenize.sent_tokenize("हिंदी सबसे अधिक बोली जाने वाली भाषा है। गंगा पवित्र नदी है।") == [
"हिंदी सबसे अधिक बोली जाने वाली भाषा है।",
"गंगा पवित्र नदी है।",
]
assert tokenize.sent_tokenize("राम वन को गए॥ सीता साथ गईं॥") == [
"राम वन को गए॥",
"सीता साथ गईं॥",
]
31 changes: 31 additions & 0 deletions test_unstructured/partition/test_text.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
EmailAddress,
ListItem,
NarrativeText,
Text,
Title,
)
from unstructured.file_utils.model import FileType
Expand Down Expand Up @@ -722,3 +723,33 @@ def test_partition_text_detects_more_than_3_languages():
)
langs = [e.metadata.languages[0] for e in elements if e.metadata.languages]
assert len(langs) > 10


@pytest.mark.parametrize(
"paragraphs",
[
[
"भारत एक विशाल देश है। इसकी राजधानी नई दिल्ली है।",
"हिंदी सबसे अधिक बोली जाने वाली भाषा है। गंगा पवित्र नदी है।",
"हिमालय उत्तर में स्थित है। वह बहुत ऊँचा है।",
"मुंबई देश की आर्थिक राजधानी मानी जाती है। यहाँ फिल्में बनती हैं।",
"दिल्ली में सरकार की बैठकें होती हैं। संसद भी यहीं है।",
"किसान खेतों में मेहनत करते हैं। वे अनाज उगाते हैं।",
],
[
"বাংলা একটি সুন্দর ভাষা। এটি অনেক মানুষ বলে।",
"কলকাতা পশ্চিমবঙ্গের রাজধানী। এখানে অনেক মানুষ থাকে।",
"রবীন্দ্রনাথ ঠাকুর নোবেল পুরস্কার পেয়েছিলেন। তিনি কবি ছিলেন।",
],
],
ids=["hindi", "bengali"],
)
def test_partition_text_classifies_hindi_and_bengali_paragraphs_as_narrative_text(paragraphs):
elements = partition_text(text="\n\n".join(paragraphs))

assert elements == [NarrativeText(text=paragraph) for paragraph in paragraphs]


@pytest.mark.parametrize("text", ["⭐️⭐️⭐️⭐️⭐️", "1️⃣ 2️⃣ 3️⃣"])
def test_partition_text_keeps_emoji_only_lines_as_text(text):
assert partition_text(text=text) == [Text(text=text)]
81 changes: 81 additions & 0 deletions test_unstructured/partition/test_text_type.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import tracemalloc
from unittest.mock import patch

import pytest
Expand Down Expand Up @@ -336,3 +337,83 @@ def test_is_email_address(text, expected):
def test_under_non_alpha_ratio_zero_divide():
# Threw an error before changes
text_type.under_non_alpha_ratio(" ")


# -- Hindi and Bengali: vowel signs are Unicode combining marks, not letters, and the danda (।)
# -- ends a sentence. --
HINDI_PARAGRAPHS = [
"भारत एक विशाल देश है। इसकी राजधानी नई दिल्ली है।",
"हिंदी सबसे अधिक बोली जाने वाली भाषा है। गंगा पवित्र नदी है।",
"हिमालय उत्तर में स्थित है। वह बहुत ऊँचा है।",
"मुंबई देश की आर्थिक राजधानी मानी जाती है। यहाँ फिल्में बनती हैं।",
"दिल्ली में सरकार की बैठकें होती हैं। संसद भी यहीं है।",
"किसान खेतों में मेहनत करते हैं। वे अनाज उगाते हैं।",
]
BENGALI_PARAGRAPHS = [
"বাংলা একটি সুন্দর ভাষা। এটি অনেক মানুষ বলে।",
"কলকাতা পশ্চিমবঙ্গের রাজধানী। এখানে অনেক মানুষ থাকে।",
"রবীন্দ্রনাথ ঠাকুর নোবেল পুরস্কার পেয়েছিলেন। তিনি কবি ছিলেন।",
]


def test_under_non_alpha_ratio_counts_vowel_signs_as_letters():
# -- "हिंदी" is 2 base letters and 3 combining marks --
assert text_type.under_non_alpha_ratio("हिंदी") is False


@pytest.mark.parametrize("text", HINDI_PARAGRAPHS + BENGALI_PARAGRAPHS)
def test_under_non_alpha_ratio_is_false_for_hindi_and_bengali_paragraphs(text):
assert text_type.under_non_alpha_ratio(text) is False


def test_exceeds_cap_ratio_is_false_for_a_hindi_sentence_whose_words_all_have_vowel_signs():
assert text_type.exceeds_cap_ratio("हिंदी सबसे अधिक बोली जाने वाली भाषा है।") is False


def test_sentence_count_splits_hindi_sentences_on_danda():
text = "हिंदी सबसे अधिक बोली जाने वाली भाषा है। गंगा पवित्र नदी है।"
assert text_type.sentence_count(text) == 2


@pytest.mark.parametrize("text", HINDI_PARAGRAPHS + BENGALI_PARAGRAPHS)
def test_is_possible_narrative_text_for_hindi_and_bengali_paragraphs(text):
assert text_type.is_possible_narrative_text(text) is True


@pytest.mark.parametrize("text", ["हिंदी", "परिचय"])
def test_is_possible_title_for_short_hindi_headings(text):
assert text_type.is_possible_title(text) is True


# -- Emoji carry combining marks too: U+FE0F (variation selector, Mn) and U+20E3 (keycap, Me).
# -- They are not attached to a letter, so they do not count as letters. --
EMOJI_LINES = ["⭐️⭐️⭐️⭐️⭐️", "1️⃣ 2️⃣ 3️⃣", "✅️ ✅️ ✅️", "⚠️"]


@pytest.mark.parametrize("text", EMOJI_LINES)
def test_under_non_alpha_ratio_does_not_count_emoji_marks_as_letters(text):
assert text_type.under_non_alpha_ratio(text) is True


@pytest.mark.parametrize("text", EMOJI_LINES)
def test_is_possible_title_is_false_for_emoji_lines(text):
assert text_type.is_possible_title(text) is False


def test_letter_checks_use_bounded_memory_on_long_input():
# -- about 5 MB of mixed Hindi, Latin with combining accents, emoji and punctuation --
unit = "हिंदी सबसे अधिक बोली जाने वाली भाषा है। Café e\u0301 ⭐️ 1️⃣ 42, — "
text = unit * (5_000_000 // len(unit.encode()))
word = "हिंदी" * 200_000

tracemalloc.start()
try:
ratio_result = text_type.under_non_alpha_ratio(text)
word_result = text_type._is_word(word)
_, peak = tracemalloc.get_traced_memory()
finally:
tracemalloc.stop()

assert ratio_result is False
assert word_result is True
assert peak < 100_000
10 changes: 9 additions & 1 deletion unstructured/nlp/tokenize.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import importlib
import logging
import os
import re
import shutil
import sys
import sysconfig
Expand All @@ -23,6 +24,11 @@
# -- than UTF-8 width. Element text is a paragraph or a table cell, so 8,192 covers it. --
MAX_CACHEABLE_CHARS: Final[int] = 8192

# NOTE(shlokbam): The English spaCy model does not reliably treat the danda (।) and double
# danda (॥), the sentence terminators of Hindi, Bengali and other Indic scripts, as sentence
# ends, so split on them after spaCy's own sentence segmentation.
_DANDA_SPLIT_RE: Final[re.Pattern[str]] = re.compile(r"(?<=[।॥])\s+")

_SPACY_MODEL_NAME: Final[str] = "en_core_web_sm"
_SPACY_MODEL_VERSION: Final[str] = "3.8.0"
_SPACY_MODEL_URL: Final[str] = (
Expand Down Expand Up @@ -214,4 +220,6 @@ def pos_tag(text: str) -> List[Tuple[str, str]]:
@lru_cache(maxsize=CACHE_MAX_SIZE)
def _tokenize_for_cache(text: str) -> Tuple[str, ...]:
"""A wrapper around the spaCy sentence tokenizer with LRU caching enabled."""
return tuple(sent.text for sent in _process(text).sents)
return tuple(
part for sent in _process(text).sents for part in _DANDA_SPLIT_RE.split(sent.text) if part
)
43 changes: 36 additions & 7 deletions unstructured/partition/text_type.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,8 @@

import os
import re
from typing import Final, List, Optional
import unicodedata
from typing import Final, Iterator, List, Optional

from unstructured.cleaners.core import remove_punctuation
from unstructured.logger import trace_logger
Expand All @@ -24,6 +25,36 @@
NON_LOWERCASE_ALPHA_RE = re.compile(r"[^a-z]")


# NOTE(shlokbam): Vowel signs, viramas and anusvaras in Indic scripts such as Devanagari and
# Bengali are combining marks, not letters, so `"हिंदी".isalpha()` is False. Treat them as
# part of the word they belong to. A mark counts only when it is attached to a letter, so the
# variation selector (U+FE0F) and keycap (U+20E3) marks in emoji like "⭐️" and "1️⃣" do not.
def _iter_letter_flags(text: str) -> Iterator[bool]:
"""Yield, for each non-space character of `text`, whether it counts as a letter.

A letter counts. A combining mark (Unicode category M*) counts only when the previous
non-space character counted. Anything else does not count. Only that one flag is kept, so
memory use does not grow with the length of `text`.
"""
category = unicodedata.category
counts = False
for char in text:
if char.isspace():
continue
if char.isalpha():
counts = True
elif category(char)[0] != "M":
counts = False
yield counts


def _is_word(token: str) -> bool:
"""True if `token` is letters and the combining marks attached to them, starting with a
letter."""
flags = _iter_letter_flags(token)
return next(flags, False) and all(flags)


def is_possible_narrative_text(
text: str,
cap_threshold: float = 0.5,
Expand Down Expand Up @@ -256,11 +287,9 @@ def under_non_alpha_ratio(text: str, threshold: float = 0.5):

alpha_count = 0
total_count = 0
for char in text:
if not char.isspace():
total_count += 1
if char.isalpha():
alpha_count += 1
for counts in _iter_letter_flags(text):
total_count += 1
alpha_count += counts

return ((alpha_count / total_count) < threshold) if total_count > 0 else False

Expand Down Expand Up @@ -291,7 +320,7 @@ def exceeds_cap_ratio(text: str, threshold: float = 0.5) -> bool:
# ex. world_tokenize("ITEM 1. Financial Statements (Unaudited)")
# = ['ITEM', '1', '.', 'Financial', 'Statements', '(', 'Unaudited', ')'],
# however, "ITEM 1. Financial Statements (Unaudited)" is Title, not NarrativeText
tokens = [tk for tk in word_tokenize(text) if tk.isalpha()]
tokens = [tk for tk in word_tokenize(text) if _is_word(tk)]

# NOTE(jay-ylee) - If word_tokenize(text) is empty, return must be True to
# avoid being misclassified as Narrative Text.
Expand Down