diff --git a/test_unstructured/partition/test_google_vision_image_modes.py b/test_unstructured/partition/test_google_vision_image_modes.py new file mode 100644 index 0000000000..f5f72d0be3 --- /dev/null +++ b/test_unstructured/partition/test_google_vision_image_modes.py @@ -0,0 +1,69 @@ +from io import BytesIO +from unittest.mock import Mock + +import pytest +from google.cloud.vision import AnnotateImageResponse, TextAnnotation +from PIL import Image + +from unstructured.partition.utils.ocr_models.google_vision_ocr import OCRAgentGoogleVision + + +@pytest.mark.parametrize( + "method", + ["get_text_from_image", "get_layout_from_image", "get_layout_elements_from_image"], +) +@pytest.mark.parametrize( + ("mode", "color"), + [ + ("CMYK", (0, 255, 255, 0)), + ("RGBX", (12, 34, 56, 255)), + ("YCbCr", (120, 90, 160)), + ("HSV", (90, 200, 210)), + ], +) +def test_non_png_color_modes_are_converted_without_mutating_the_callers_image(method, mode, color): + image = Image.new(mode, (3, 2), color) + original = image.tobytes() + agent = object.__new__(OCRAgentGoogleVision) + agent.language = "en" + agent.client = Mock() + agent.client.document_text_detection.return_value = AnnotateImageResponse( + full_text_annotation=TextAnnotation(text="recognized") + ) + + getattr(agent, method)(image) + + for call in agent.client.document_text_detection.call_args_list: + with Image.open(BytesIO(call.kwargs["image"].content)) as encoded: + assert encoded.format == "PNG" + assert encoded.mode == "RGB" + assert encoded.size == image.size + assert encoded.tobytes() == image.convert("RGB").tobytes() + assert list(call.kwargs["image_context"].language_hints) == ["en"] + assert image.mode == mode + assert image.tobytes() == original + + +@pytest.mark.parametrize("mode", ["L", "P", "RGBA"]) +@pytest.mark.parametrize( + "method", + ["get_text_from_image", "get_layout_from_image", "get_layout_elements_from_image"], +) +def test_png_native_modes_keep_their_pixels_and_alpha(mode, method): + image = Image.new(mode, (3, 2)) + if mode == "RGBA": + image.putpixel((0, 0), (12, 34, 56, 78)) + agent = object.__new__(OCRAgentGoogleVision) + agent.language = None + agent.client = Mock() + agent.client.document_text_detection.return_value = AnnotateImageResponse( + full_text_annotation=TextAnnotation() + ) + + getattr(agent, method)(image) + + with Image.open( + BytesIO(agent.client.document_text_detection.call_args.kwargs["image"].content) + ) as encoded: + assert encoded.mode == mode + assert encoded.tobytes() == image.tobytes() diff --git a/unstructured/partition/utils/ocr_models/google_vision_ocr.py b/unstructured/partition/utils/ocr_models/google_vision_ocr.py index 852f973ba4..3e42262fa6 100644 --- a/unstructured/partition/utils/ocr_models/google_vision_ocr.py +++ b/unstructured/partition/utils/ocr_models/google_vision_ocr.py @@ -16,6 +16,17 @@ from unstructured_inference.inference.layoutelement import LayoutElements +def _as_png_bytes(image: PILImage.Image) -> bytes: + """Encode a page without mutating it, converting non-PNG color modes to RGB first.""" + with BytesIO() as buffer: + if image.mode in {"CMYK", "RGBX", "YCbCr", "HSV"}: + with image.convert("RGB") as rgb_image: + rgb_image.save(buffer, format="PNG") + else: + image.save(buffer, format="PNG") + return buffer.getvalue() + + class OCRAgentGoogleVision(OCRAgent): """OCR service implementation for Google Vision API.""" @@ -35,11 +46,9 @@ def is_text_sorted(self) -> bool: def get_text_from_image(self, image: PILImage.Image) -> str: image_context = ImageContext(language_hints=[self.language]) if self.language else None - with BytesIO() as buffer: - image.save(buffer, format="PNG") - response = self.client.document_text_detection( - image=Image(content=buffer.getvalue()), image_context=image_context - ) + response = self.client.document_text_detection( + image=Image(content=_as_png_bytes(image)), image_context=image_context + ) document = response.full_text_annotation assert isinstance(document, TextAnnotation) return document.text @@ -47,11 +56,9 @@ def get_text_from_image(self, image: PILImage.Image) -> str: def get_layout_from_image(self, image: PILImage.Image) -> TextRegions: trace_logger.detail("Processing entire page OCR with Google Vision API...") image_context = ImageContext(language_hints=[self.language]) if self.language else None - with BytesIO() as buffer: - image.save(buffer, format="PNG") - response = self.client.document_text_detection( - image=Image(content=buffer.getvalue()), image_context=image_context - ) + response = self.client.document_text_detection( + image=Image(content=_as_png_bytes(image)), image_context=image_context + ) document = response.full_text_annotation assert isinstance(document, TextAnnotation) regions = self._parse_regions(document)