From 2a99c2d0bb9859a2cc14947ad7388ed4eaacb37c Mon Sep 17 00:00:00 2001 From: Afloat Date: Sat, 21 Nov 2026 10:12:33 +0800 Subject: [PATCH] fix(ocr): retain Google Vision paragraph tails and break hints --- .../test_google_vision_paragraphs.py | 33 +++++++++++++++++++ .../utils/ocr_models/google_vision_ocr.py | 30 +++++++++-------- 2 files changed, 50 insertions(+), 13 deletions(-) create mode 100644 test_unstructured/partition/test_google_vision_paragraphs.py diff --git a/test_unstructured/partition/test_google_vision_paragraphs.py b/test_unstructured/partition/test_google_vision_paragraphs.py new file mode 100644 index 0000000000..6d9011c45b --- /dev/null +++ b/test_unstructured/partition/test_google_vision_paragraphs.py @@ -0,0 +1,33 @@ +import pytest +from google.cloud.vision import Paragraph, TextAnnotation + +from unstructured.partition.utils.ocr_models.google_vision_ocr import OCRAgentGoogleVision + + +@pytest.mark.parametrize( + ("symbols", "expected"), + [ + ([("final", "UNKNOWN", False)], "final"), + ([("first", "SPACE", False), ("last", "UNKNOWN", False)], "first last"), + ([("first", "SURE_SPACE", False), ("last", "UNKNOWN", False)], "first last"), + ([("first", "EOL_SURE_SPACE", False), ("last", "UNKNOWN", False)], "first last"), + ([("first", "LINE_BREAK", False), ("last", "UNKNOWN", False)], "first\nlast"), + ([("hyphen", "HYPHEN", False), ("ated", "UNKNOWN", False)], "hyphen-\nated"), + ([("first", "UNKNOWN", False), ("last", "SPACE", True)], "first last"), + ([("whole", "LINE_BREAK", False)], "whole"), + ([], ""), + ], +) +def test_paragraph_keeps_all_symbols_and_detected_breaks(symbols, expected): + breaks = TextAnnotation.DetectedBreak.BreakType + words = [] + for text, break_, prefix in symbols: + characters = [{"text": char} for char in text] + characters[0 if prefix else -1]["property"] = { + "detected_break": {"type_": getattr(breaks, break_), "is_prefix": prefix} + } + words.append({"symbols": characters}) + paragraph = Paragraph(words=words) + agent = object.__new__(OCRAgentGoogleVision) + + assert agent._get_text_from_paragraph(paragraph) == expected diff --git a/unstructured/partition/utils/ocr_models/google_vision_ocr.py b/unstructured/partition/utils/ocr_models/google_vision_ocr.py index 852f973ba4..c3de8b70b3 100644 --- a/unstructured/partition/utils/ocr_models/google_vision_ocr.py +++ b/unstructured/partition/utils/ocr_models/google_vision_ocr.py @@ -98,19 +98,23 @@ def _parse_regions(self, ocr_data: TextAnnotation) -> TextRegions: return TextRegions.from_list(text_regions) def _get_text_from_paragraph(self, paragraph: Paragraph) -> str: + """Retain every symbol, applying the response's leading or trailing break hints.""" breaks = TextAnnotation.DetectedBreak.BreakType - para = "" - line = "" + separators = { + breaks.SPACE: " ", + breaks.SURE_SPACE: " ", + breaks.EOL_SURE_SPACE: " ", + breaks.HYPHEN: "-\n", + breaks.LINE_BREAK: "\n", + } + parts: list[str] = [] for word in paragraph.words: for symbol in word.symbols: - line += symbol.text - if symbol.property.detected_break.type_ == breaks.SPACE: - line += " " - if symbol.property.detected_break.type_ == breaks.EOL_SURE_SPACE: - line += " " - para += line - line = "" - if symbol.property.detected_break.type_ == breaks.LINE_BREAK: - para += line - line = "" - return para + detected_break = symbol.property.detected_break + separator = separators.get(detected_break.type_, "") + if detected_break.is_prefix: + parts.append(separator) + parts.append(symbol.text) + if not detected_break.is_prefix: + parts.append(separator) + return "".join(parts).rstrip()