Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
69 changes: 69 additions & 0 deletions test_unstructured/partition/test_google_vision_image_modes.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
from io import BytesIO
from unittest.mock import Mock

import pytest
from google.cloud.vision import AnnotateImageResponse, TextAnnotation
from PIL import Image

from unstructured.partition.utils.ocr_models.google_vision_ocr import OCRAgentGoogleVision


@pytest.mark.parametrize(
"method",
["get_text_from_image", "get_layout_from_image", "get_layout_elements_from_image"],
)
@pytest.mark.parametrize(
("mode", "color"),
[
("CMYK", (0, 255, 255, 0)),
("RGBX", (12, 34, 56, 255)),
("YCbCr", (120, 90, 160)),
("HSV", (90, 200, 210)),
],
)
def test_non_png_color_modes_are_converted_without_mutating_the_callers_image(method, mode, color):
image = Image.new(mode, (3, 2), color)
original = image.tobytes()
agent = object.__new__(OCRAgentGoogleVision)
agent.language = "en"
agent.client = Mock()
agent.client.document_text_detection.return_value = AnnotateImageResponse(
full_text_annotation=TextAnnotation(text="recognized")
)

getattr(agent, method)(image)

for call in agent.client.document_text_detection.call_args_list:
with Image.open(BytesIO(call.kwargs["image"].content)) as encoded:
assert encoded.format == "PNG"
assert encoded.mode == "RGB"
assert encoded.size == image.size
assert encoded.tobytes() == image.convert("RGB").tobytes()
assert list(call.kwargs["image_context"].language_hints) == ["en"]
assert image.mode == mode
assert image.tobytes() == original


@pytest.mark.parametrize("mode", ["L", "P", "RGBA"])
@pytest.mark.parametrize(
"method",
["get_text_from_image", "get_layout_from_image", "get_layout_elements_from_image"],
)
def test_png_native_modes_keep_their_pixels_and_alpha(mode, method):
image = Image.new(mode, (3, 2))
if mode == "RGBA":
image.putpixel((0, 0), (12, 34, 56, 78))
agent = object.__new__(OCRAgentGoogleVision)
agent.language = None
agent.client = Mock()
agent.client.document_text_detection.return_value = AnnotateImageResponse(
full_text_annotation=TextAnnotation()
)

getattr(agent, method)(image)

with Image.open(
BytesIO(agent.client.document_text_detection.call_args.kwargs["image"].content)
) as encoded:
assert encoded.mode == mode
assert encoded.tobytes() == image.tobytes()
27 changes: 17 additions & 10 deletions unstructured/partition/utils/ocr_models/google_vision_ocr.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,17 @@
from unstructured_inference.inference.layoutelement import LayoutElements


def _as_png_bytes(image: PILImage.Image) -> bytes:
"""Encode a page without mutating it, converting non-PNG color modes to RGB first."""
with BytesIO() as buffer:
if image.mode in {"CMYK", "RGBX", "YCbCr", "HSV"}:
with image.convert("RGB") as rgb_image:
rgb_image.save(buffer, format="PNG")
else:
image.save(buffer, format="PNG")
return buffer.getvalue()


class OCRAgentGoogleVision(OCRAgent):
"""OCR service implementation for Google Vision API."""

Expand All @@ -35,23 +46,19 @@ def is_text_sorted(self) -> bool:

def get_text_from_image(self, image: PILImage.Image) -> str:
image_context = ImageContext(language_hints=[self.language]) if self.language else None
with BytesIO() as buffer:
image.save(buffer, format="PNG")
response = self.client.document_text_detection(
image=Image(content=buffer.getvalue()), image_context=image_context
)
response = self.client.document_text_detection(
image=Image(content=_as_png_bytes(image)), image_context=image_context
)
document = response.full_text_annotation
assert isinstance(document, TextAnnotation)
return document.text

def get_layout_from_image(self, image: PILImage.Image) -> TextRegions:
trace_logger.detail("Processing entire page OCR with Google Vision API...")
image_context = ImageContext(language_hints=[self.language]) if self.language else None
with BytesIO() as buffer:
image.save(buffer, format="PNG")
response = self.client.document_text_detection(
image=Image(content=buffer.getvalue()), image_context=image_context
)
response = self.client.document_text_detection(
image=Image(content=_as_png_bytes(image)), image_context=image_context
)
document = response.full_text_annotation
assert isinstance(document, TextAnnotation)
regions = self._parse_regions(document)
Expand Down