Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions test_unstructured/partition/test_text_type.py
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,21 @@ def test_is_possible_narrative_text(text, expected, monkeypatch):
assert is_possible_narrative is expected


@pytest.mark.parametrize(
"text",
["----------------------------------------", "$ 1,234.56 $ 7,890.12", "*** === +++ ###"],
)
def test_is_possible_narrative_text_rejects_low_alpha_text_without_nlp(text, monkeypatch):
def fail(*args, **kwargs):
raise AssertionError("low-alpha text should be rejected before the NLP pipeline runs")

monkeypatch.setattr(text_type, "word_tokenize", fail)
monkeypatch.setattr(text_type, "pos_tag", fail)
monkeypatch.setattr(text_type, "sent_tokenize", fail)

assert text_type.is_possible_narrative_text(text) is False


def test_narrative_text_language_checks():
# NOTE(robinson) - This is true because we don't check english vocab if language checks
# are set to False
Expand Down
14 changes: 8 additions & 6 deletions unstructured/partition/text_type.py
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,14 @@ def is_possible_narrative_text(
if "eng" in languages and language_checks and not contains_english_word(text):
return False

# -- Cheap character scan first: `exceeds_cap_ratio` runs the spaCy pipeline, and both checks
# -- only ever return False, so their order doesn't change the result. --
non_alpha_threshold = float(
os.environ.get("UNSTRUCTURED_NARRATIVE_TEXT_NON_ALPHA_THRESHOLD", non_alpha_threshold),
)
if under_non_alpha_ratio(text, threshold=non_alpha_threshold):
return False

# NOTE(robinson): it gets read in from the environment as a string so we need to
# cast it to a float
cap_threshold = float(
Expand All @@ -75,12 +83,6 @@ def is_possible_narrative_text(
trace_logger.detail(f"Not narrative. Text exceeds cap ratio {cap_threshold}:\n\n{text}") # type: ignore # noqa: E501
return False

non_alpha_threshold = float(
os.environ.get("UNSTRUCTURED_NARRATIVE_TEXT_NON_ALPHA_THRESHOLD", non_alpha_threshold),
)
if under_non_alpha_ratio(text, threshold=non_alpha_threshold):
return False

if "eng" in languages and (sentence_count(text, 3) < 2) and (not contains_verb(text)):
trace_logger.detail(f"Not narrative. Text does not contain a verb:\n\n{text}") # type: ignore # noqa: E501
return False
Expand Down