From c70dc5dfaf2991898f27da150d8e929d4b1c5397 Mon Sep 17 00:00:00 2001 From: Kevin Turcios Date: Mon, 9 Nov 2026 03:17:08 -0500 Subject: [PATCH 1/2] perf(text_type): run the non-alpha check before the spaCy-backed cap-ratio check --- unstructured/partition/text_type.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/unstructured/partition/text_type.py b/unstructured/partition/text_type.py index 79f45019d6..bf2cca0438 100644 --- a/unstructured/partition/text_type.py +++ b/unstructured/partition/text_type.py @@ -66,6 +66,14 @@ def is_possible_narrative_text( if "eng" in languages and language_checks and not contains_english_word(text): return False + # -- Cheap character scan first: `exceeds_cap_ratio` runs the spaCy pipeline, and both checks + # -- only ever return False, so their order doesn't change the result. -- + non_alpha_threshold = float( + os.environ.get("UNSTRUCTURED_NARRATIVE_TEXT_NON_ALPHA_THRESHOLD", non_alpha_threshold), + ) + if under_non_alpha_ratio(text, threshold=non_alpha_threshold): + return False + # NOTE(robinson): it gets read in from the environment as a string so we need to # cast it to a float cap_threshold = float( @@ -75,12 +83,6 @@ def is_possible_narrative_text( trace_logger.detail(f"Not narrative. Text exceeds cap ratio {cap_threshold}:\n\n{text}") # type: ignore # noqa: E501 return False - non_alpha_threshold = float( - os.environ.get("UNSTRUCTURED_NARRATIVE_TEXT_NON_ALPHA_THRESHOLD", non_alpha_threshold), - ) - if under_non_alpha_ratio(text, threshold=non_alpha_threshold): - return False - if "eng" in languages and (sentence_count(text, 3) < 2) and (not contains_verb(text)): trace_logger.detail(f"Not narrative. Text does not contain a verb:\n\n{text}") # type: ignore # noqa: E501 return False From 3f2c850b9af4095ae8b48f669b4f1e580ce4aa05 Mon Sep 17 00:00:00 2001 From: Kevin Turcios Date: Sat, 21 Nov 2026 03:52:11 -0500 Subject: [PATCH 2/2] test(text_type): pin that low-alpha text is rejected before the NLP pipeline runs --- test_unstructured/partition/test_text_type.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/test_unstructured/partition/test_text_type.py b/test_unstructured/partition/test_text_type.py index 98443df080..0721856e8a 100644 --- a/test_unstructured/partition/test_text_type.py +++ b/test_unstructured/partition/test_text_type.py @@ -55,6 +55,21 @@ def test_is_possible_narrative_text(text, expected, monkeypatch): assert is_possible_narrative is expected +@pytest.mark.parametrize( + "text", + ["----------------------------------------", "$ 1,234.56 $ 7,890.12", "*** === +++ ###"], +) +def test_is_possible_narrative_text_rejects_low_alpha_text_without_nlp(text, monkeypatch): + def fail(*args, **kwargs): + raise AssertionError("low-alpha text should be rejected before the NLP pipeline runs") + + monkeypatch.setattr(text_type, "word_tokenize", fail) + monkeypatch.setattr(text_type, "pos_tag", fail) + monkeypatch.setattr(text_type, "sent_tokenize", fail) + + assert text_type.is_possible_narrative_text(text) is False + + def test_narrative_text_language_checks(): # NOTE(robinson) - This is true because we don't check english vocab if language checks # are set to False