diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e7394c9..fa600875 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## 0.1.17 + +### Fixes + +- **Honor the `OCR_AGENT` environment variable**: The API called `partition()` without `ocr_agent`, so the library default (Tesseract) was always used and an image built with PaddleOCR and `OCR_AGENT` set still ran Tesseract. The agent named by `OCR_AGENT` is now passed to `partition()`, resolved the same way the library resolves it, so the legacy names `tesseract` and `paddle` keep working. Requests without `OCR_AGENT` still use Tesseract. Parallel-mode workers resolve the variable from their own environment. + ## 0.1.16 ### Fixes diff --git a/prepline_general/api/__version__.py b/prepline_general/api/__version__.py index d7e5d581..2daeb707 100644 --- a/prepline_general/api/__version__.py +++ b/prepline_general/api/__version__.py @@ -1 +1 @@ -__version__ = "0.1.16" # pragma: no cover +__version__ = "0.1.17" # pragma: no cover diff --git a/prepline_general/api/general.py b/prepline_general/api/general.py index 04142dcc..8e3ee36c 100644 --- a/prepline_general/api/general.py +++ b/prepline_general/api/general.py @@ -41,6 +41,7 @@ convert_to_isd, elements_from_json, ) +from unstructured.partition.utils.ocr_models.ocr_interface import OCRAgent from unstructured_inference.models.base import UnknownModelException app = FastAPI() @@ -156,6 +157,8 @@ def partition_file_via_api( ) if "combine_text_under_n_chars" in worker_kwargs: worker_kwargs["combine_under_n_chars"] = worker_kwargs.pop("combine_text_under_n_chars") + # -- not a form field; each worker resolves OCR_AGENT from its own environment -- + worker_kwargs.pop("ocr_agent", None) result = call_api( request_url, @@ -324,6 +327,10 @@ def pipeline_api( extract_image_block_to_payload = bool(extract_image_block_types) + # -- partition() defaults to Tesseract unless `ocr_agent` is passed, so resolve the agent + # -- named by the OCR_AGENT env var (legacy names like "paddle" included) here. + ocr_agent = OCRAgent._get_ocr_agent_cls_qname() + try: logger.debug( "partition input data: {}".format( @@ -332,6 +339,7 @@ def pipeline_api( "content_type": file_content_type, "strategy": strategy, "ocr_languages": ocr_languages_str, + "ocr_agent": ocr_agent, "coordinates": coordinates, "pdf_infer_table_structure": pdf_infer_table_structure, "include_page_breaks": include_page_breaks, @@ -366,6 +374,7 @@ def pipeline_api( "include_page_breaks": include_page_breaks, "hi_res_model_name": hi_res_model_name, "ocr_languages": ocr_languages_str, + "ocr_agent": ocr_agent, "pdf_infer_table_structure": pdf_infer_table_structure, "skip_infer_table_types": skip_infer_table_types, "strategy": strategy, diff --git a/test_general/api/test_app.py b/test_general/api/test_app.py index ac9cb6fb..8ed1704d 100644 --- a/test_general/api/test_app.py +++ b/test_general/api/test_app.py @@ -15,6 +15,7 @@ from prepline_general.api import general from prepline_general.api.app import app +from unstructured.partition.utils.constants import OCR_AGENT_PADDLE MAIN_API_ROUTE = "general/v0/general" @@ -720,6 +721,7 @@ def test_parallel_mode_passes_params(monkeypatch): TODO - do the same test when params are sent back to the api """ monkeypatch.setenv("UNSTRUCTURED_PARALLEL_MODE_ENABLED", "true") + monkeypatch.setenv("OCR_AGENT", OCR_AGENT_PADDLE) # Make this really big so we just call partition monkeypatch.setenv("UNSTRUCTURED_PARALLEL_MODE_SPLIT_SIZE", "500") @@ -775,6 +777,7 @@ def test_parallel_mode_passes_params(monkeypatch): encoding="foo", include_page_breaks=True, ocr_languages=None, + ocr_agent=OCR_AGENT_PADDLE, languages=["eng"], # NOTE(robinson) - pdf_infer_table_structure is False because # skip_infer_table_type=["pdf"] superceded pdf_infer_table_structure diff --git a/test_general/api/test_ocr_agent.py b/test_general/api/test_ocr_agent.py new file mode 100644 index 00000000..419c054c --- /dev/null +++ b/test_general/api/test_ocr_agent.py @@ -0,0 +1,64 @@ +import io +from pathlib import Path +from unittest.mock import Mock + +import pytest +from fastapi.testclient import TestClient +from starlette.requests import Request + +from prepline_general.api import general +from prepline_general.api.app import app +from unstructured.partition.utils.constants import OCR_AGENT_PADDLE, OCR_AGENT_TESSERACT + +MAIN_API_ROUTE = "general/v0/general" + + +@pytest.mark.parametrize( + ("env_value", "expected_agent"), + [ + (None, OCR_AGENT_TESSERACT), + (OCR_AGENT_PADDLE, OCR_AGENT_PADDLE), + # -- legacy key-name form is mapped to the qualified name -- + ("paddle", OCR_AGENT_PADDLE), + ], +) +def test_ocr_agent_env_var_is_passed_to_partition(monkeypatch, env_value, expected_agent): + if env_value is None: + monkeypatch.delenv("OCR_AGENT", raising=False) + else: + monkeypatch.setenv("OCR_AGENT", env_value) + mock_partition = Mock(return_value=[]) + monkeypatch.setattr(general, "partition", mock_partition) + + client = TestClient(app) + test_file = Path("sample-docs") / "layout-parser-paper-fast.jpg" + with open(test_file, "rb") as f: + response = client.post( + MAIN_API_ROUTE, + files=[("files", (str(test_file), f, "image/jpeg"))], + data={"strategy": "hi_res"}, + ) + + assert response.status_code == 200 + mock_partition.assert_called_once() + assert mock_partition.call_args.kwargs["ocr_agent"] == expected_agent + + +def test_ocr_agent_is_not_sent_to_parallel_workers(monkeypatch): + monkeypatch.setenv("UNSTRUCTURED_PARALLEL_MODE_URL", "https://worker.example/partition") + post = Mock(return_value=Mock(status_code=200, text="[]")) + monkeypatch.setattr(general.requests, "post", post) + request = Request({"type": "http", "headers": []}) + + general.partition_file_via_api( + (io.BytesIO(b"pdf"), 0), + request, + "document.pdf", + "application/pdf", + strategy="hi_res", + ocr_agent=OCR_AGENT_PADDLE, + ) + + sent_form = post.call_args.kwargs["data"] + assert sent_form["strategy"] == "hi_res" + assert "ocr_agent" not in sent_form