Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,9 @@
## 0.1.17

### Fixes

- **Honor the `OCR_AGENT` environment variable**: The API called `partition()` without `ocr_agent`, so the library default (Tesseract) was always used and an image built with PaddleOCR and `OCR_AGENT` set still ran Tesseract. The agent named by `OCR_AGENT` is now passed to `partition()`, resolved the same way the library resolves it, so the legacy names `tesseract` and `paddle` keep working. Requests without `OCR_AGENT` still use Tesseract. Parallel-mode workers resolve the variable from their own environment.

## 0.1.16

### Fixes
Expand Down
2 changes: 1 addition & 1 deletion prepline_general/api/__version__.py
Original file line number Diff line number Diff line change
@@ -1 +1 @@
__version__ = "0.1.16" # pragma: no cover
__version__ = "0.1.17" # pragma: no cover
9 changes: 9 additions & 0 deletions prepline_general/api/general.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@
convert_to_isd,
elements_from_json,
)
from unstructured.partition.utils.ocr_models.ocr_interface import OCRAgent
from unstructured_inference.models.base import UnknownModelException

app = FastAPI()
Expand Down Expand Up @@ -156,6 +157,8 @@ def partition_file_via_api(
)
if "combine_text_under_n_chars" in worker_kwargs:
worker_kwargs["combine_under_n_chars"] = worker_kwargs.pop("combine_text_under_n_chars")
# -- not a form field; each worker resolves OCR_AGENT from its own environment --
worker_kwargs.pop("ocr_agent", None)

result = call_api(
request_url,
Expand Down Expand Up @@ -324,6 +327,10 @@ def pipeline_api(

extract_image_block_to_payload = bool(extract_image_block_types)

# -- partition() defaults to Tesseract unless `ocr_agent` is passed, so resolve the agent
# -- named by the OCR_AGENT env var (legacy names like "paddle" included) here.
ocr_agent = OCRAgent._get_ocr_agent_cls_qname()

try:
logger.debug(
"partition input data: {}".format(
Expand All @@ -332,6 +339,7 @@ def pipeline_api(
"content_type": file_content_type,
"strategy": strategy,
"ocr_languages": ocr_languages_str,
"ocr_agent": ocr_agent,
"coordinates": coordinates,
"pdf_infer_table_structure": pdf_infer_table_structure,
"include_page_breaks": include_page_breaks,
Expand Down Expand Up @@ -366,6 +374,7 @@ def pipeline_api(
"include_page_breaks": include_page_breaks,
"hi_res_model_name": hi_res_model_name,
"ocr_languages": ocr_languages_str,
"ocr_agent": ocr_agent,
"pdf_infer_table_structure": pdf_infer_table_structure,
"skip_infer_table_types": skip_infer_table_types,
"strategy": strategy,
Expand Down
3 changes: 3 additions & 0 deletions test_general/api/test_app.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@

from prepline_general.api import general
from prepline_general.api.app import app
from unstructured.partition.utils.constants import OCR_AGENT_PADDLE

MAIN_API_ROUTE = "general/v0/general"

Expand Down Expand Up @@ -720,6 +721,7 @@ def test_parallel_mode_passes_params(monkeypatch):
TODO - do the same test when params are sent back to the api
"""
monkeypatch.setenv("UNSTRUCTURED_PARALLEL_MODE_ENABLED", "true")
monkeypatch.setenv("OCR_AGENT", OCR_AGENT_PADDLE)

# Make this really big so we just call partition
monkeypatch.setenv("UNSTRUCTURED_PARALLEL_MODE_SPLIT_SIZE", "500")
Expand Down Expand Up @@ -775,6 +777,7 @@ def test_parallel_mode_passes_params(monkeypatch):
encoding="foo",
include_page_breaks=True,
ocr_languages=None,
ocr_agent=OCR_AGENT_PADDLE,
languages=["eng"],
# NOTE(robinson) - pdf_infer_table_structure is False because
# skip_infer_table_type=["pdf"] superceded pdf_infer_table_structure
Expand Down
64 changes: 64 additions & 0 deletions test_general/api/test_ocr_agent.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
import io
from pathlib import Path
from unittest.mock import Mock

import pytest
from fastapi.testclient import TestClient
from starlette.requests import Request

from prepline_general.api import general
from prepline_general.api.app import app
from unstructured.partition.utils.constants import OCR_AGENT_PADDLE, OCR_AGENT_TESSERACT

MAIN_API_ROUTE = "general/v0/general"


@pytest.mark.parametrize(
("env_value", "expected_agent"),
[
(None, OCR_AGENT_TESSERACT),
(OCR_AGENT_PADDLE, OCR_AGENT_PADDLE),
# -- legacy key-name form is mapped to the qualified name --
("paddle", OCR_AGENT_PADDLE),
],
)
def test_ocr_agent_env_var_is_passed_to_partition(monkeypatch, env_value, expected_agent):
if env_value is None:
monkeypatch.delenv("OCR_AGENT", raising=False)
else:
monkeypatch.setenv("OCR_AGENT", env_value)
mock_partition = Mock(return_value=[])
monkeypatch.setattr(general, "partition", mock_partition)

client = TestClient(app)
test_file = Path("sample-docs") / "layout-parser-paper-fast.jpg"
with open(test_file, "rb") as f:
response = client.post(
MAIN_API_ROUTE,
files=[("files", (str(test_file), f, "image/jpeg"))],
data={"strategy": "hi_res"},
)

assert response.status_code == 200
mock_partition.assert_called_once()
assert mock_partition.call_args.kwargs["ocr_agent"] == expected_agent


def test_ocr_agent_is_not_sent_to_parallel_workers(monkeypatch):
monkeypatch.setenv("UNSTRUCTURED_PARALLEL_MODE_URL", "https://worker.example/partition")
post = Mock(return_value=Mock(status_code=200, text="[]"))
monkeypatch.setattr(general.requests, "post", post)
request = Request({"type": "http", "headers": []})

general.partition_file_via_api(
(io.BytesIO(b"pdf"), 0),
request,
"document.pdf",
"application/pdf",
strategy="hi_res",
ocr_agent=OCR_AGENT_PADDLE,
)

sent_form = post.call_args.kwargs["data"]
assert sent_form["strategy"] == "hi_res"
assert "ocr_agent" not in sent_form