diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 07d4439d..d608c676 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -25,4 +25,4 @@ jobs: - name: Run E2E tests env: REDUCTO_API_KEY: ${{ secrets.REDUCTO_API_KEY }} - run: uv run pytest tests/e2e/ -v -x -o "addopts=" + run: uv run pytest tests/e2e/ -v -x -o "addopts=" -n auto --dist loadscope diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a17461e6..20ce0a86 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -71,6 +71,12 @@ $ pip install ./path-to-wheel-file.whl $ ./scripts/test ``` +End-to-end tests in `tests/e2e/` call the live Reducto API and are excluded by default. They need `REDUCTO_API_KEY`. Run them in parallel, one worker per test class: + +```sh +$ REDUCTO_API_KEY=... uv run pytest tests/e2e/ -o "addopts=" -n auto --dist loadscope +``` + ## Linting and formatting This repository uses [ruff](https://github.com/astral-sh/ruff) to format the code. diff --git a/api.md b/api.md index 2b7460ac..7b0f6575 100644 --- a/api.md +++ b/api.md @@ -11,12 +11,16 @@ from reducto.types import ( AsyncPipelineResponse, AsyncSplitResponse, BaseProcessingOptions, + ChartResponse, Chunking, ChunkingConfig, ClassifyResponse, DirectWebhookConfig, + DocumentProperties, EditResponse, EnrichConfig, + ErrorCode, + ErrorDetail, ExperimentalProcessingOptions, ExtractResponse, FigureAgentic, @@ -31,6 +35,7 @@ from reducto.types import ( TableAgentic, TableSummaryConfig, TextAgentic, + URLResult, Upload, WebhookConfigNew, ) @@ -41,12 +46,13 @@ from reducto.types import ( Types: ```python -from reducto.types import APIVersionResponse +from reducto.types import APIVersionResponse, ClientDeleteUploadResponse ``` Methods: - client.api_version() -> str +- client.delete_upload(file_id) -> ClientDeleteUploadResponse - client.upload(\*\*params) -> Upload # Parse @@ -154,11 +160,12 @@ Methods: Types: ```python -from reducto.types import JobGetResponse, JobGetAllResponse +from reducto.types import JobDeleteResponse, JobGetResponse, JobGetAllResponse ``` Methods: - client.job.cancel(job_id) -> object +- client.job.delete(job_id, \*\*params) -> JobDeleteResponse - client.job.get(job_id) -> JobGetResponse - client.job.get_all(\*\*params) -> JobGetAllResponse diff --git a/scripts/spec_drift.py b/scripts/spec_drift.py index 637436b5..db2688b1 100755 --- a/scripts/spec_drift.py +++ b/scripts/spec_drift.py @@ -7,6 +7,10 @@ and walked in parallel, comparing JSON property names, types, enum values, and required-ness. +`spec/drift-allowlist.json` lists known, intentional deviations with a reason. +Matching items are reported as allowed and do not fail the check. Entries that +match nothing are flagged as stale. + The check runs against the committed snapshot `spec/openapi.json` by default, so it is reproducible and needs no network. `--live` checks against the public URL. `--update-snapshot` fetches the live spec, rewrites the snapshot, then checks. @@ -31,7 +35,7 @@ import typing_extensions from typing import Any, Dict, List, Tuple, Union, Literal, Optional from pathlib import Path -from dataclasses import field, asdict, dataclass +from dataclasses import field, asdict, replace, dataclass from typing_extensions import Required, Annotated, NotRequired, get_args, get_origin, is_typeddict REPO = Path(__file__).resolve().parent.parent @@ -43,6 +47,7 @@ LIVE_SPEC = "https://reducto.ai/openapi.json" SNAPSHOT = REPO / "spec" / "openapi.json" +ALLOWLIST = REPO / "spec" / "drift-allowlist.json" HTTP_VERBS = ("get", "post", "put", "patch", "delete") HTTP_METHODS = {name: verb for verb in HTTP_VERBS for name in (verb, "_" + verb)} SUCCESS_CODES = ("200", "201", "202") @@ -64,6 +69,7 @@ class Shape: items: Optional[Shape] = None members: List[Shape] = field(default_factory=lambda: []) label: str = "" + has_default: bool = False ANY = Shape("any") @@ -117,6 +123,12 @@ def resolve(self, node: JsonDict) -> JsonDict: return node def shape(self, node: JsonDict, seen: typing.FrozenSet[str] = frozenset()) -> Shape: + shape = self.shape_of(node, seen) + if "default" in node and not shape.has_default: + shape = replace(shape, has_default=True) + return shape + + def shape_of(self, node: JsonDict, seen: typing.FrozenSet[str]) -> Shape: label = "" if "$ref" in node: label = node["$ref"].rsplit("/", 1)[-1] @@ -485,9 +497,13 @@ def compare_required(self, spec: Shape, sdk: Shape, name: str, loc: str) -> None if spec_req == sdk_req: return # Response models express nullable fields as Optional[...] = None, so - # "required but nullable" in the spec is not drift there. + # "required but nullable" in the spec is not drift there. A response + # field with a server default is always present, so the sdk may + # require it. if self.ignore_null and spec_req and is_nullable(spec.props[name]): return + if self.ignore_null and sdk_req and spec.props[name].has_default: + return self.report(loc, "required", f"spec {req_word(spec_req)}, sdk {req_word(sdk_req)}") def compare_union(self, spec: Shape, sdk: Shape, loc: str) -> None: @@ -682,9 +698,59 @@ def display(source: str) -> str: return source -def print_report(drifts: List[Drift], endpoints: List[Endpoint], spec: Spec, source: str) -> None: +@dataclass +class Allowed: + endpoint: str + location: str + kind: str + reason: str + detail: Optional[str] = None + + def matches(self, d: Drift) -> bool: + if (self.endpoint, self.location, self.kind) != (d.endpoint, d.location, d.kind): + return False + return self.detail is None or self.detail == d.detail + + +def load_allowlist(path: Path) -> List[Allowed]: + if not path.exists(): + return [] + entries = typing.cast(List[JsonDict], json.loads(path.read_text())) + return [Allowed(**e) for e in entries] + + +def split_allowed(drifts: List[Drift], allowlist: List[Allowed]) -> Tuple[List[Drift], List[Drift], List[Allowed]]: + active: List[Drift] = [] + allowed: List[Drift] = [] + used: typing.Set[int] = set() + for d in drifts: + hit = next((i for i, a in enumerate(allowlist) if a.matches(d)), None) + if hit is None: + active.append(d) + else: + allowed.append(d) + used.add(hit) + stale = [a for i, a in enumerate(allowlist) if i not in used] + return active, allowed, stale + + +def print_report( + drifts: List[Drift], + allowed: List[Drift], + stale: List[Allowed], + endpoints: List[Endpoint], + spec: Spec, + source: str, +) -> None: print(f"Spec: {display(source)} (version {spec.version})") print(f"Checked {len(endpoints)} sdk endpoints.") + if allowed: + print(f"{len(allowed)} allowed drift item(s), see {ALLOWLIST.relative_to(REPO)}:") + for d in allowed: + loc = f" {d.location}" if d.location else "" + print(f" {d.endpoint} [{d.kind}]{loc}: {d.detail}") + for a in stale: + print(f"Stale allowlist entry matches nothing: {a.endpoint} [{a.kind}] {a.location}", file=sys.stderr) if not drifts: print("No drift found.") return @@ -735,13 +801,19 @@ def main() -> int: spec = Spec(load_doc(source)) endpoints = discover_endpoints() - drifts = [d for d in run(spec, endpoints) if d.kind not in args.ignore] + found = [d for d in run(spec, endpoints) if d.kind not in args.ignore] + drifts, allowed, stale = split_allowed(found, load_allowlist(ALLOWLIST)) if args.json: - payload = {"spec": source, "spec_version": spec.version, "drifts": [asdict(d) for d in drifts]} + payload = { + "spec": source, + "spec_version": spec.version, + "drifts": [asdict(d) for d in drifts], + "allowed": [asdict(d) for d in allowed], + } print(json.dumps(payload, indent=2)) else: - print_report(drifts, endpoints, spec, source) + print_report(drifts, allowed, stale, endpoints, spec, source) return 0 if not drifts or args.warn_only else 1 diff --git a/spec/drift-allowlist.json b/spec/drift-allowlist.json new file mode 100644 index 00000000..86b096b1 --- /dev/null +++ b/spec/drift-allowlist.json @@ -0,0 +1,9 @@ +[ + { + "endpoint": "POST /extract", + "location": "response", + "kind": "extra", + "detail": "sdk union member object(7 fields) has no spec counterpart", + "reason": "The API returns ExtractResponse (response_type 'extract') for accounts not on V3 extract. The spec lists only V3ExtractResponse and AsyncExtractResponse for this endpoint. ExtractRunResponse includes ExtractResponse so those responses parse. Remove once the spec adds ExtractResponse to POST /extract." + } +] diff --git a/src/reducto/_client.py b/src/reducto/_client.py index 68a52cfb..cfd21d09 100644 --- a/src/reducto/_client.py +++ b/src/reducto/_client.py @@ -26,6 +26,7 @@ from ._utils import ( is_given, extract_files, + path_template, maybe_transform, deepcopy_minimal, get_async_library, @@ -49,6 +50,7 @@ make_request_options, ) from .types.shared.upload import Upload +from .types.client_delete_upload_response import ClientDeleteUploadResponse if TYPE_CHECKING: from .resources import job, edit, parse, split, extract, webhook, classify, pipeline @@ -312,6 +314,39 @@ def api_version( cast_to=str, ) + def delete_upload( + self, + file_id: str, + *, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = not_given, + ) -> ClientDeleteUploadResponse: + """ + Delete Uploaded File + + Args: + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if not file_id: + raise ValueError(f"Expected a non-empty value for `file_id` but received {file_id!r}") + return self.delete( + path_template("/upload/{file_id}", file_id=file_id), + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=ClientDeleteUploadResponse, + ) + def upload( self, *, @@ -623,6 +658,39 @@ async def api_version( cast_to=str, ) + async def delete_upload( + self, + file_id: str, + *, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = not_given, + ) -> ClientDeleteUploadResponse: + """ + Delete Uploaded File + + Args: + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if not file_id: + raise ValueError(f"Expected a non-empty value for `file_id` but received {file_id!r}") + return await self.delete( + path_template("/upload/{file_id}", file_id=file_id), + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=ClientDeleteUploadResponse, + ) + async def upload( self, *, @@ -711,6 +779,9 @@ def __init__(self, client: Reducto) -> None: self.api_version = to_raw_response_wrapper( client.api_version, ) + self.delete_upload = to_raw_response_wrapper( + client.delete_upload, + ) self.upload = to_raw_response_wrapper( client.upload, ) @@ -773,6 +844,9 @@ def __init__(self, client: AsyncReducto) -> None: self.api_version = async_to_raw_response_wrapper( client.api_version, ) + self.delete_upload = async_to_raw_response_wrapper( + client.delete_upload, + ) self.upload = async_to_raw_response_wrapper( client.upload, ) @@ -835,6 +909,9 @@ def __init__(self, client: Reducto) -> None: self.api_version = to_streamed_response_wrapper( client.api_version, ) + self.delete_upload = to_streamed_response_wrapper( + client.delete_upload, + ) self.upload = to_streamed_response_wrapper( client.upload, ) @@ -897,6 +974,9 @@ def __init__(self, client: AsyncReducto) -> None: self.api_version = async_to_streamed_response_wrapper( client.api_version, ) + self.delete_upload = async_to_streamed_response_wrapper( + client.delete_upload, + ) self.upload = async_to_streamed_response_wrapper( client.upload, ) diff --git a/src/reducto/resources/classify.py b/src/reducto/resources/classify.py index 98910c82..eb8ce07c 100644 --- a/src/reducto/resources/classify.py +++ b/src/reducto/resources/classify.py @@ -1,11 +1,12 @@ from __future__ import annotations -from typing import Iterable, Optional +from typing import Dict, Iterable, Optional +from typing_extensions import Literal import httpx from ..types import classify_run_params -from .._types import Body, Omit, Query, Headers, NotGiven, omit, not_given +from .._types import Body, Omit, Query, Headers, NotGiven, SequenceNotStr, omit, not_given from .._utils import maybe_transform, async_maybe_transform from .._compat import cached_property from .._resource import SyncAPIResource, AsyncAPIResource @@ -45,10 +46,13 @@ def run( self, *, input: classify_run_params.Input, + category_groups: Dict[str, SequenceNotStr[str]] | Omit = omit, classification_schema: Iterable[classify_run_params.ClassificationSchema] | Omit = omit, document_metadata: Optional[str] | Omit = omit, + force_url_result: bool | Omit = omit, + model: Literal["default", "accurate"] | Omit = omit, page_range: Optional[classify_run_params.PageRange] | Omit = omit, - persist_results: bool | Omit = omit, + priority: bool | Omit = omit, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. extra_headers: Headers | None = None, @@ -69,15 +73,27 @@ def run( For edit pipelines, this should be a string containing the edit instructions + category_groups: A mapping of higher-level classify groups to the category labels that belong + to each group. When provided, the response includes `extra_metadata.grouping` with + the matched group name, or `ungrouped` if the selected category is not in any + group. + classification_schema: A list of classification categories and their matching criteria. document_metadata: Optional document-level metadata to include in classification prompts. + force_url_result: Force the endpoint result to be returned in URL form. + + model: The classification model to use. Set to "accurate" to run Deep Classify for + higher accuracy on hard documents. Defaults to "default". + page_range: The page range to process (1-indexed). By default, the first 5 pages are used. If more than 25 pages are selected, only the first 25 (after sorting) are used. Only applies to PDFs; ignored for other document types. - persist_results: If True, persist the results indefinitely. Defaults to False. + priority: Workers poll the priority queue ahead of the standard queue, so priority jobs + start sooner when there is queued work; sync jobs are prioritized above async + jobs by default. extra_headers: Send extra headers @@ -92,10 +108,13 @@ def run( body=maybe_transform( { "input": input, + "category_groups": category_groups, "classification_schema": classification_schema, "document_metadata": document_metadata, + "force_url_result": force_url_result, + "model": model, "page_range": page_range, - "persist_results": persist_results, + "priority": priority, }, classify_run_params.ClassifyRunParams, ), @@ -130,10 +149,13 @@ async def run( self, *, input: classify_run_params.Input, + category_groups: Dict[str, SequenceNotStr[str]] | Omit = omit, classification_schema: Iterable[classify_run_params.ClassificationSchema] | Omit = omit, document_metadata: Optional[str] | Omit = omit, + force_url_result: bool | Omit = omit, + model: Literal["default", "accurate"] | Omit = omit, page_range: Optional[classify_run_params.PageRange] | Omit = omit, - persist_results: bool | Omit = omit, + priority: bool | Omit = omit, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. extra_headers: Headers | None = None, @@ -154,15 +176,27 @@ async def run( For edit pipelines, this should be a string containing the edit instructions + category_groups: A mapping of higher-level classify groups to the category labels that belong + to each group. When provided, the response includes `extra_metadata.grouping` with + the matched group name, or `ungrouped` if the selected category is not in any + group. + classification_schema: A list of classification categories and their matching criteria. document_metadata: Optional document-level metadata to include in classification prompts. + force_url_result: Force the endpoint result to be returned in URL form. + + model: The classification model to use. Set to "accurate" to run Deep Classify for + higher accuracy on hard documents. Defaults to "default". + page_range: The page range to process (1-indexed). By default, the first 5 pages are used. If more than 25 pages are selected, only the first 25 (after sorting) are used. Only applies to PDFs; ignored for other document types. - persist_results: If True, persist the results indefinitely. Defaults to False. + priority: Workers poll the priority queue ahead of the standard queue, so priority jobs + start sooner when there is queued work; sync jobs are prioritized above async + jobs by default. extra_headers: Send extra headers @@ -177,10 +211,13 @@ async def run( body=await async_maybe_transform( { "input": input, + "category_groups": category_groups, "classification_schema": classification_schema, "document_metadata": document_metadata, + "force_url_result": force_url_result, + "model": model, "page_range": page_range, - "persist_results": persist_results, + "priority": priority, }, classify_run_params.ClassifyRunParams, ), diff --git a/src/reducto/resources/extract.py b/src/reducto/resources/extract.py index e827481f..bc110f36 100644 --- a/src/reducto/resources/extract.py +++ b/src/reducto/resources/extract.py @@ -1,7 +1,7 @@ from __future__ import annotations from typing import Any, cast -from typing_extensions import overload +from typing_extensions import Literal, overload import httpx @@ -103,6 +103,7 @@ def run( async_: AsyncConfigV3Param | Omit = omit, instructions: InstructionsParam | Omit = omit, parsing: ParseOptionsParam | Omit = omit, + queue_priority: Literal["auto", "standard", "batch"] | Omit = omit, settings: ExtractSettingsParam | Omit = omit, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. @@ -126,6 +127,9 @@ def run( async_: The configuration options for asynchronous processing (default synchronous). + queue_priority: Queue priority. 'batch' places the job in a lower-priority queue for + non-urgent bulk work. 'auto' (alias: 'standard') uses the default queue. + instructions: The instructions to use for the extraction. parsing: The configuration options for parsing the document. If you are passing in a @@ -152,6 +156,7 @@ def run( parsing: ParseOptionsParam | Omit = omit, settings: ExtractSettingsParam | Omit = omit, async_: AsyncConfigV3Param | Omit = omit, + queue_priority: Literal["auto", "standard", "batch"] | Omit = omit, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. extra_headers: Headers | None = None, @@ -170,6 +175,7 @@ def run( "parsing": parsing, "settings": settings, "async_": async_, + "queue_priority": queue_priority, }, extract_run_params.ExtractRunParams, ), @@ -189,6 +195,7 @@ def run_job( async_: AsyncConfigV3Param | Omit = omit, instructions: InstructionsParam | Omit = omit, parsing: ParseOptionsParam | Omit = omit, + queue_priority: Literal["auto", "standard", "batch"] | Omit = omit, settings: ExtractSettingsParam | Omit = omit, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. @@ -212,6 +219,9 @@ def run_job( async_: The configuration options for asynchronous processing (default synchronous). + queue_priority: Queue priority. 'batch' places the job in a lower-priority queue for + non-urgent bulk work. 'auto' (alias: 'standard') uses the default queue. + instructions: The instructions to use for the extraction. parsing: The configuration options for parsing the document. If you are passing in a @@ -236,6 +246,7 @@ def run_job( "instructions": instructions, "parsing": parsing, "settings": settings, + "queue_priority": queue_priority, }, extract_run_job_params.ExtractRunJobParams, ), @@ -319,6 +330,7 @@ async def run( async_: AsyncConfigV3Param | Omit = omit, instructions: InstructionsParam | Omit = omit, parsing: ParseOptionsParam | Omit = omit, + queue_priority: Literal["auto", "standard", "batch"] | Omit = omit, settings: ExtractSettingsParam | Omit = omit, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. @@ -342,6 +354,9 @@ async def run( async_: The configuration options for asynchronous processing (default synchronous). + queue_priority: Queue priority. 'batch' places the job in a lower-priority queue for + non-urgent bulk work. 'auto' (alias: 'standard') uses the default queue. + instructions: The instructions to use for the extraction. parsing: The configuration options for parsing the document. If you are passing in a @@ -368,6 +383,7 @@ async def run( parsing: ParseOptionsParam | Omit = omit, settings: ExtractSettingsParam | Omit = omit, async_: AsyncConfigV3Param | Omit = omit, + queue_priority: Literal["auto", "standard", "batch"] | Omit = omit, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. extra_headers: Headers | None = None, @@ -386,6 +402,7 @@ async def run( "parsing": parsing, "settings": settings, "async_": async_, + "queue_priority": queue_priority, }, extract_run_params.ExtractRunParams, ), @@ -405,6 +422,7 @@ async def run_job( async_: AsyncConfigV3Param | Omit = omit, instructions: InstructionsParam | Omit = omit, parsing: ParseOptionsParam | Omit = omit, + queue_priority: Literal["auto", "standard", "batch"] | Omit = omit, settings: ExtractSettingsParam | Omit = omit, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. @@ -428,6 +446,9 @@ async def run_job( async_: The configuration options for asynchronous processing (default synchronous). + queue_priority: Queue priority. 'batch' places the job in a lower-priority queue for + non-urgent bulk work. 'auto' (alias: 'standard') uses the default queue. + instructions: The instructions to use for the extraction. parsing: The configuration options for parsing the document. If you are passing in a @@ -452,6 +473,7 @@ async def run_job( "instructions": instructions, "parsing": parsing, "settings": settings, + "queue_priority": queue_priority, }, extract_run_job_params.ExtractRunJobParams, ), diff --git a/src/reducto/resources/job.py b/src/reducto/resources/job.py index d5ce5a68..0554d139 100644 --- a/src/reducto/resources/job.py +++ b/src/reducto/resources/job.py @@ -4,7 +4,7 @@ import httpx -from ..types import job_get_all_params +from ..types import job_delete_params, job_get_all_params from .._types import Body, Omit, Query, Headers, NotGiven, omit, not_given from .._utils import path_template, maybe_transform, async_maybe_transform from .._compat import cached_property @@ -17,6 +17,7 @@ ) from .._base_client import make_request_options from ..types.job_get_response import JobGetResponse +from ..types.job_delete_response import JobDeleteResponse from ..types.job_get_all_response import JobGetAllResponse __all__ = ["JobResource", "AsyncJobResource"] @@ -75,6 +76,50 @@ def cancel( cast_to=object, ) + def delete( + self, + job_id: str, + *, + include_persisted: bool | Omit = omit, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = not_given, + ) -> JobDeleteResponse: + """ + Delete Parse Job + + Asynchronously delete a job's stored artifacts. Tags the job with the deletion + marker. Retrieval returns 409 until artifact cleanup finishes, then 410 once the + deletion has completed. + + Args: + include_persisted: Also delete long-retention persisted artifacts for this job. + + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if not job_id: + raise ValueError(f"Expected a non-empty value for `job_id` but received {job_id!r}") + return self._delete( + path_template("/job/{job_id}", job_id=job_id), + options=make_request_options( + extra_headers=extra_headers, + extra_query=extra_query, + extra_body=extra_body, + timeout=timeout, + query=maybe_transform({"include_persisted": include_persisted}, job_delete_params.JobDeleteParams), + ), + cast_to=JobDeleteResponse, + ) + def get( self, job_id: str, @@ -217,6 +262,52 @@ async def cancel( cast_to=object, ) + async def delete( + self, + job_id: str, + *, + include_persisted: bool | Omit = omit, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = not_given, + ) -> JobDeleteResponse: + """ + Delete Parse Job + + Asynchronously delete a job's stored artifacts. Tags the job with the deletion + marker. Retrieval returns 409 until artifact cleanup finishes, then 410 once the + deletion has completed. + + Args: + include_persisted: Also delete long-retention persisted artifacts for this job. + + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if not job_id: + raise ValueError(f"Expected a non-empty value for `job_id` but received {job_id!r}") + return await self._delete( + path_template("/job/{job_id}", job_id=job_id), + options=make_request_options( + extra_headers=extra_headers, + extra_query=extra_query, + extra_body=extra_body, + timeout=timeout, + query=await async_maybe_transform( + {"include_persisted": include_persisted}, job_delete_params.JobDeleteParams + ), + ), + cast_to=JobDeleteResponse, + ) + async def get( self, job_id: str, @@ -313,6 +404,9 @@ def __init__(self, job: JobResource) -> None: self.cancel = to_raw_response_wrapper( job.cancel, ) + self.delete = to_raw_response_wrapper( + job.delete, + ) self.get = to_raw_response_wrapper( job.get, ) @@ -328,6 +422,9 @@ def __init__(self, job: AsyncJobResource) -> None: self.cancel = async_to_raw_response_wrapper( job.cancel, ) + self.delete = async_to_raw_response_wrapper( + job.delete, + ) self.get = async_to_raw_response_wrapper( job.get, ) @@ -343,6 +440,9 @@ def __init__(self, job: JobResource) -> None: self.cancel = to_streamed_response_wrapper( job.cancel, ) + self.delete = to_streamed_response_wrapper( + job.delete, + ) self.get = to_streamed_response_wrapper( job.get, ) @@ -358,6 +458,9 @@ def __init__(self, job: AsyncJobResource) -> None: self.cancel = async_to_streamed_response_wrapper( job.cancel, ) + self.delete = async_to_streamed_response_wrapper( + job.delete, + ) self.get = async_to_streamed_response_wrapper( job.get, ) diff --git a/src/reducto/resources/parse.py b/src/reducto/resources/parse.py index e7a4b26a..a5a463d4 100644 --- a/src/reducto/resources/parse.py +++ b/src/reducto/resources/parse.py @@ -100,7 +100,7 @@ def run( async_: AsyncConfigV3Param | Omit = omit, enhance: EnhanceParam | Omit = omit, formatting: FormattingParam | Omit = omit, - queue_priority: Literal["auto", "batch"] | Omit = omit, + queue_priority: Literal["auto", "standard", "batch"] | Omit = omit, retrieval: RetrievalParam | Omit = omit, settings: SettingsParam | Omit = omit, spreadsheet: SpreadsheetParam | Omit = omit, @@ -126,8 +126,8 @@ def run( async_: The configuration options for asynchronous processing (default synchronous). - queue_priority: Queue priority. 'batch' for non-urgent work that processes when spare GPU - capacity is available. + queue_priority: Queue priority. 'batch' places the job in a lower-priority queue for + non-urgent bulk work. 'auto' (alias: 'standard') uses the default queue. extra_headers: Send extra headers @@ -150,7 +150,7 @@ def run( settings: SettingsParam | Omit = omit, spreadsheet: SpreadsheetParam | Omit = omit, async_: AsyncConfigV3Param | Omit = omit, - queue_priority: Literal["auto", "batch"] | Omit = omit, + queue_priority: Literal["auto", "standard", "batch"] | Omit = omit, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. extra_headers: Headers | None = None, @@ -189,7 +189,7 @@ def run_job( async_: AsyncConfigV3Param | Omit = omit, enhance: EnhanceParam | Omit = omit, formatting: FormattingParam | Omit = omit, - queue_priority: Literal["auto", "batch"] | Omit = omit, + queue_priority: Literal["auto", "standard", "batch"] | Omit = omit, retrieval: RetrievalParam | Omit = omit, settings: SettingsParam | Omit = omit, spreadsheet: SpreadsheetParam | Omit = omit, @@ -215,8 +215,8 @@ def run_job( async_: The configuration options for asynchronous processing (default synchronous). - queue_priority: Queue priority. 'batch' for non-urgent work that processes when spare GPU - capacity is available. + queue_priority: Queue priority. 'batch' places the job in a lower-priority queue for + non-urgent bulk work. 'auto' (alias: 'standard') uses the default queue. extra_headers: Send extra headers @@ -316,7 +316,7 @@ async def run( async_: AsyncConfigV3Param | Omit = omit, enhance: EnhanceParam | Omit = omit, formatting: FormattingParam | Omit = omit, - queue_priority: Literal["auto", "batch"] | Omit = omit, + queue_priority: Literal["auto", "standard", "batch"] | Omit = omit, retrieval: RetrievalParam | Omit = omit, settings: SettingsParam | Omit = omit, spreadsheet: SpreadsheetParam | Omit = omit, @@ -342,8 +342,8 @@ async def run( async_: The configuration options for asynchronous processing (default synchronous). - queue_priority: Queue priority. 'batch' for non-urgent work that processes when spare GPU - capacity is available. + queue_priority: Queue priority. 'batch' places the job in a lower-priority queue for + non-urgent bulk work. 'auto' (alias: 'standard') uses the default queue. extra_headers: Send extra headers @@ -366,7 +366,7 @@ async def run( settings: SettingsParam | Omit = omit, spreadsheet: SpreadsheetParam | Omit = omit, async_: AsyncConfigV3Param | Omit = omit, - queue_priority: Literal["auto", "batch"] | Omit = omit, + queue_priority: Literal["auto", "standard", "batch"] | Omit = omit, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. extra_headers: Headers | None = None, @@ -405,7 +405,7 @@ async def run_job( async_: AsyncConfigV3Param | Omit = omit, enhance: EnhanceParam | Omit = omit, formatting: FormattingParam | Omit = omit, - queue_priority: Literal["auto", "batch"] | Omit = omit, + queue_priority: Literal["auto", "standard", "batch"] | Omit = omit, retrieval: RetrievalParam | Omit = omit, settings: SettingsParam | Omit = omit, spreadsheet: SpreadsheetParam | Omit = omit, @@ -431,8 +431,8 @@ async def run_job( async_: The configuration options for asynchronous processing (default synchronous). - queue_priority: Queue priority. 'batch' for non-urgent work that processes when spare GPU - capacity is available. + queue_priority: Queue priority. 'batch' places the job in a lower-priority queue for + non-urgent bulk work. 'auto' (alias: 'standard') uses the default queue. extra_headers: Send extra headers diff --git a/src/reducto/types/__init__.py b/src/reducto/types/__init__.py index 5001b6c4..6528b138 100644 --- a/src/reducto/types/__init__.py +++ b/src/reducto/types/__init__.py @@ -3,11 +3,15 @@ from .shared import ( Upload as Upload, Chunking as Chunking, + ErrorCode as ErrorCode, PageRange as PageRange, + URLResult as URLResult, + ErrorDetail as ErrorDetail, TextAgentic as TextAgentic, EditResponse as EditResponse, EnrichConfig as EnrichConfig, TableAgentic as TableAgentic, + ChartResponse as ChartResponse, FigureAgentic as FigureAgentic, ParseResponse as ParseResponse, SplitResponse as SplitResponse, @@ -22,6 +26,7 @@ ArrayExtractConfig as ArrayExtractConfig, AsyncParseResponse as AsyncParseResponse, AsyncSplitResponse as AsyncSplitResponse, + DocumentProperties as DocumentProperties, TableSummaryConfig as TableSummaryConfig, DirectWebhookConfig as DirectWebhookConfig, FigureSummaryConfig as FigureSummaryConfig, @@ -47,6 +52,7 @@ from .parse_run_params import ParseRunParams as ParseRunParams from .split_run_params import SplitRunParams as SplitRunParams from .edit_widget_param import EditWidgetParam as EditWidgetParam +from .job_delete_params import JobDeleteParams as JobDeleteParams from .spreadsheet_param import SpreadsheetParam as SpreadsheetParam from .bounding_box_param import BoundingBoxParam as BoundingBoxParam from .edit_options_param import EditOptionsParam as EditOptionsParam @@ -56,6 +62,7 @@ from .parse_run_response import ParseRunResponse as ParseRunResponse from .classify_run_params import ClassifyRunParams as ClassifyRunParams from .edit_run_job_params import EditRunJobParams as EditRunJobParams +from .job_delete_response import JobDeleteResponse as JobDeleteResponse from .parse_options_param import ParseOptionsParam as ParseOptionsParam from .pipeline_run_params import PipelineRunParams as PipelineRunParams from .api_version_response import APIVersionResponse as APIVersionResponse @@ -75,3 +82,4 @@ from .deep_split_page_evidence import DeepSplitPageEvidence as DeepSplitPageEvidence from .split_table_options_param import SplitTableOptionsParam as SplitTableOptionsParam from .async_extract_config_param import AsyncExtractConfigParam as AsyncExtractConfigParam +from .client_delete_upload_response import ClientDeleteUploadResponse as ClientDeleteUploadResponse diff --git a/src/reducto/types/async_parse_config_param.py b/src/reducto/types/async_parse_config_param.py index 8c0441cb..53399257 100644 --- a/src/reducto/types/async_parse_config_param.py +++ b/src/reducto/types/async_parse_config_param.py @@ -42,10 +42,11 @@ class AsyncParseConfigParam(_AsyncParseConfigParamReservedKeywords, total=False) formatting: FormattingParam - queue_priority: Literal["auto", "batch"] + queue_priority: Literal["auto", "standard", "batch"] """Queue priority. - 'batch' for non-urgent work that processes when spare GPU capacity is available. + 'batch' places the job in a lower-priority queue for non-urgent bulk work. + 'auto' (alias: 'standard') uses the default queue. """ retrieval: RetrievalParam diff --git a/src/reducto/types/classify_run_params.py b/src/reducto/types/classify_run_params.py index a89a6af6..15a9dc61 100644 --- a/src/reducto/types/classify_run_params.py +++ b/src/reducto/types/classify_run_params.py @@ -1,7 +1,7 @@ from __future__ import annotations -from typing import Union, Iterable, Optional -from typing_extensions import Required, TypeAlias, TypedDict +from typing import Dict, Union, Iterable, Optional +from typing_extensions import Literal, Required, TypeAlias, TypedDict from .._types import SequenceNotStr from .shared_params import page_range @@ -23,12 +23,30 @@ class ClassifyRunParams(TypedDict, total=False): For edit pipelines, this should be a string containing the edit instructions """ + category_groups: Dict[str, SequenceNotStr[str]] + """ + A mapping of higher-level classify groups to the category labels that belong to + each group. When provided, the response includes `extra_metadata.grouping` with + the matched group name, or `ungrouped` if the selected category is not in any + group. + """ + classification_schema: Iterable[ClassificationSchema] """A list of classification categories and their matching criteria.""" document_metadata: Optional[str] """Optional document-level metadata to include in classification prompts.""" + force_url_result: bool + """Force the endpoint result to be returned in URL form.""" + + model: Literal["default", "accurate"] + """The classification model to use. + + Set to "accurate" to run Deep Classify for higher accuracy on hard documents. + Defaults to "default". + """ + page_range: Optional[PageRange] """The page range to process (1-indexed). @@ -37,8 +55,12 @@ class ClassifyRunParams(TypedDict, total=False): document types. """ - persist_results: bool - """If True, persist the results indefinitely. Defaults to False.""" + priority: bool + """ + Workers poll the priority queue ahead of the standard queue, so priority jobs + start sooner when there is queued work; sync jobs are prioritized above async + jobs by default. + """ Input: TypeAlias = Union[str, SequenceNotStr[str], Upload] diff --git a/src/reducto/types/client_delete_upload_response.py b/src/reducto/types/client_delete_upload_response.py new file mode 100644 index 00000000..84b0cd21 --- /dev/null +++ b/src/reducto/types/client_delete_upload_response.py @@ -0,0 +1,7 @@ +from .._models import BaseModel + +__all__ = ["ClientDeleteUploadResponse"] + + +class ClientDeleteUploadResponse(BaseModel): + file_id: str diff --git a/src/reducto/types/client_upload_params.py b/src/reducto/types/client_upload_params.py index 4be33b32..b13fee82 100644 --- a/src/reducto/types/client_upload_params.py +++ b/src/reducto/types/client_upload_params.py @@ -1,14 +1,17 @@ from __future__ import annotations -from typing import Optional -from typing_extensions import TypedDict +from typing import Union, Optional +from typing_extensions import TypeAlias, TypedDict from .._types import FileTypes -__all__ = ["ClientUploadParams"] +__all__ = ["ClientUploadParams", "File"] class ClientUploadParams(TypedDict, total=False): extension: Optional[str] - file: Optional[FileTypes] + file: Optional[File] + + +File: TypeAlias = Union[FileTypes, str] diff --git a/src/reducto/types/enhance_param.py b/src/reducto/types/enhance_param.py index 5cf183b9..a13643da 100644 --- a/src/reducto/types/enhance_param.py +++ b/src/reducto/types/enhance_param.py @@ -13,6 +13,14 @@ class EnhanceParam(TypedDict, total=False): + advanced_chart_agent: bool + """ + If True, run advanced chart extraction on figures classified as charts, without + requiring a figure-scoped agentic entry. Returns full structured series data + (chart_data) plus a reconstruction image re-drawn from that data. Higher + latency. Defaults to False. + """ + agentic: Iterable[Agentic] """ Agentic uses vision language models to enhance the accuracy of the output of diff --git a/src/reducto/types/extract_run_job_params.py b/src/reducto/types/extract_run_job_params.py index bbc07977..13949773 100644 --- a/src/reducto/types/extract_run_job_params.py +++ b/src/reducto/types/extract_run_job_params.py @@ -1,7 +1,7 @@ from __future__ import annotations from typing import Union -from typing_extensions import Required, Annotated, TypeAlias, TypedDict +from typing_extensions import Literal, Required, Annotated, TypeAlias, TypedDict from .._types import SequenceNotStr from .._utils import PropertyInfo @@ -40,6 +40,13 @@ class ExtractRunJobParams(TypedDict, total=False): be ignored. """ + queue_priority: Literal["auto", "standard", "batch"] + """Queue priority. + + 'batch' places the job in a lower-priority queue for non-urgent bulk work. + 'auto' (alias: 'standard') uses the default queue. + """ + settings: ExtractSettingsParam """The settings to use for the extraction.""" diff --git a/src/reducto/types/extract_run_params.py b/src/reducto/types/extract_run_params.py index 87e6c468..be7b65b2 100644 --- a/src/reducto/types/extract_run_params.py +++ b/src/reducto/types/extract_run_params.py @@ -1,7 +1,7 @@ from __future__ import annotations from typing import Union -from typing_extensions import Required, Annotated, TypeAlias, TypedDict +from typing_extensions import Literal, Required, Annotated, TypeAlias, TypedDict from .._types import SequenceNotStr from .._utils import PropertyInfo @@ -76,6 +76,13 @@ class AsyncExtractConfig(TypedDict, total=False): be ignored. """ + queue_priority: Literal["auto", "standard", "batch"] + """Queue priority. + + 'batch' places the job in a lower-priority queue for non-urgent bulk work. + 'auto' (alias: 'standard') uses the default queue. + """ + settings: ExtractSettingsParam """The settings to use for the extraction.""" diff --git a/src/reducto/types/extract_run_response.py b/src/reducto/types/extract_run_response.py index 025ca197..22184d75 100644 --- a/src/reducto/types/extract_run_response.py +++ b/src/reducto/types/extract_run_response.py @@ -1,9 +1,15 @@ from typing import Union -from typing_extensions import TypeAlias +from typing_extensions import Annotated, TypeAlias +from .._utils import PropertyInfo from .v3_extract import V3Extract +from .shared.extract_response import ExtractResponse from .shared.async_extract_response import AsyncExtractResponse __all__ = ["ExtractRunResponse"] -ExtractRunResponse: TypeAlias = Union[V3Extract, AsyncExtractResponse] +# The API returns ExtractResponse (response_type "extract") for accounts not on +# V3 extract, though the spec lists only V3ExtractResponse for this endpoint. +ExtractRunResponse: TypeAlias = Annotated[ + Union[ExtractResponse, V3Extract, AsyncExtractResponse], PropertyInfo(discriminator="response_type") +] diff --git a/src/reducto/types/extract_settings_param.py b/src/reducto/types/extract_settings_param.py index 9064eb29..99912f4f 100644 --- a/src/reducto/types/extract_settings_param.py +++ b/src/reducto/types/extract_settings_param.py @@ -1,8 +1,12 @@ from __future__ import annotations -from typing_extensions import TypedDict +from typing import Union, Iterable, Optional +from typing_extensions import Literal, TypeAlias, TypedDict -__all__ = ["ExtractSettingsParam", "Citations"] +from .._types import SequenceNotStr +from .shared_params import page_range + +__all__ = ["ExtractSettingsParam", "Citations", "PageRange"] class Citations(TypedDict, total=False): @@ -14,6 +18,17 @@ class Citations(TypedDict, total=False): numerical_confidence: bool """If True, enable numeric citation confidence scores. Defaults to True.""" + parent_block: Literal["full", "bbox_only"] + """How much of the source parse block to embed on each citation's parentBlock. + + 'full' (default) embeds the verbatim source-block HTML in parentBlock.content. + 'bbox_only' suppresses parentBlock.content (returned as an empty string) while + keeping parentBlock.bbox and all citation-level fields. + """ + + +PageRange: TypeAlias = Union[page_range.PageRange, Iterable[page_range.PageRange], Iterable[int], SequenceNotStr[str]] + class ExtractSettingsParam(TypedDict, total=False): array_extract: bool @@ -29,6 +44,9 @@ class ExtractSettingsParam(TypedDict, total=False): accuracy is critical. """ + force_url_result: bool + """Force the endpoint result to be returned in URL form.""" + include_images: bool """If True, include images in the extraction.""" @@ -37,3 +55,10 @@ class ExtractSettingsParam(TypedDict, total=False): If True, jobs will be processed with a higher throughput and priority at a higher cost. Defaults to False. """ + + page_range: Optional[PageRange] + """The page range to extract from (1-indexed). + + By default, the entire document is used. For spreadsheets, you can also provide + a list of sheet names. + """ diff --git a/src/reducto/types/extract_usage.py b/src/reducto/types/extract_usage.py index 9c6dba45..381026ad 100644 --- a/src/reducto/types/extract_usage.py +++ b/src/reducto/types/extract_usage.py @@ -3,7 +3,23 @@ from .._models import BaseModel -__all__ = ["ExtractUsage"] +__all__ = ["ExtractUsage", "UsageBreakdown"] + + +class UsageBreakdown(BaseModel): + extract_model: Literal["Extract", "Deep Extract"] + + charts: Optional[int] = None + + extract_fields: Optional[int] = None + + extract_pages: Optional[int] = None + + ocr_pages: Optional[int] = None + + prompted_blocks: Optional[int] = None + + tier: Optional[Literal["Default", "Batch"]] = None class ExtractUsage(BaseModel): @@ -14,3 +30,10 @@ class ExtractUsage(BaseModel): credits: Optional[float] = None extract_mode: Optional[Literal["super_agent", "extract", "spreadsheet_agent"]] = None + + usage_breakdown: Optional[UsageBreakdown] = None + """Raw usage quantities. + + Only set for accounts on the new pricing model; credit fields are omitted for + those accounts. + """ diff --git a/src/reducto/types/job_delete_params.py b/src/reducto/types/job_delete_params.py new file mode 100644 index 00000000..ad081425 --- /dev/null +++ b/src/reducto/types/job_delete_params.py @@ -0,0 +1,10 @@ +from __future__ import annotations + +from typing_extensions import TypedDict + +__all__ = ["JobDeleteParams"] + + +class JobDeleteParams(TypedDict, total=False): + include_persisted: bool + """Also delete long-retention persisted artifacts for this job.""" diff --git a/src/reducto/types/job_delete_response.py b/src/reducto/types/job_delete_response.py new file mode 100644 index 00000000..fa0a7f90 --- /dev/null +++ b/src/reducto/types/job_delete_response.py @@ -0,0 +1,7 @@ +from .._models import BaseModel + +__all__ = ["JobDeleteResponse"] + + +class JobDeleteResponse(BaseModel): + job_id: str diff --git a/src/reducto/types/job_get_all_response.py b/src/reducto/types/job_get_all_response.py index b061af13..1eef27e8 100644 --- a/src/reducto/types/job_get_all_response.py +++ b/src/reducto/types/job_get_all_response.py @@ -22,7 +22,7 @@ class Job(BaseModel): total_pages: Optional[int] = None - type: Literal["Parse", "Extract", "Split", "Edit", "Pipeline", "Classify"] + type: Literal["Parse", "Extract", "Split", "Edit", "Pipeline", "Classify", "Chart"] bucket: Optional[object] = None diff --git a/src/reducto/types/job_get_response.py b/src/reducto/types/job_get_response.py index 9a712020..fec9a6b5 100644 --- a/src/reducto/types/job_get_response.py +++ b/src/reducto/types/job_get_response.py @@ -1,10 +1,13 @@ from typing import Union, Optional from datetime import datetime -from typing_extensions import Literal, TypeAlias +from typing_extensions import Literal, Annotated, TypeAlias +from .._utils import PropertyInfo from .._models import BaseModel from .v3_extract import V3Extract +from .shared.error_detail import ErrorDetail from .shared.edit_response import EditResponse +from .shared.chart_response import ChartResponse from .shared.parse_response import ParseResponse from .shared.split_response import SplitResponse from .shared.extract_response import ExtractResponse @@ -19,24 +22,47 @@ "EnhancedAsyncJobResponseResult", ] -AsyncJobResponseResult: TypeAlias = Union[ - ParseResponse, ExtractResponse, SplitResponse, EditResponse, PipelineResponse, V3Extract, ClassifyResponse, None +AsyncJobResponseResult: TypeAlias = Annotated[ + Union[ + ParseResponse, + ExtractResponse, + SplitResponse, + EditResponse, + PipelineResponse, + V3Extract, + ClassifyResponse, + ChartResponse, + None, + ], + PropertyInfo(discriminator="response_type"), ] class AsyncJobResponse(BaseModel): status: Literal["Pending", "Completed", "Failed", "Idle"] + error: Optional[ErrorDetail] = None + progress: Optional[float] = None reason: Optional[str] = None result: Optional[AsyncJobResponseResult] = None - """Response from classify job - returned when polling /job/{job_id}""" -EnhancedAsyncJobResponseResult: TypeAlias = Union[ - ParseResponse, ExtractResponse, SplitResponse, EditResponse, PipelineResponse, V3Extract, ClassifyResponse, None +EnhancedAsyncJobResponseResult: TypeAlias = Annotated[ + Union[ + ParseResponse, + ExtractResponse, + SplitResponse, + EditResponse, + PipelineResponse, + V3Extract, + ClassifyResponse, + ChartResponse, + None, + ], + PropertyInfo(discriminator="response_type"), ] @@ -49,6 +75,8 @@ class EnhancedAsyncJobResponse(BaseModel): duration: Optional[float] = None + error: Optional[ErrorDetail] = None + num_pages: Optional[int] = None progress: Optional[float] = None @@ -58,13 +86,12 @@ class EnhancedAsyncJobResponse(BaseModel): reason: Optional[str] = None result: Optional[EnhancedAsyncJobResponseResult] = None - """Response from classify job - returned when polling /job/{job_id}""" source: Optional[object] = None total_pages: Optional[int] = None - type: Optional[Literal["Parse", "Extract", "Split", "Edit", "Pipeline", "Classify"]] = None + type: Optional[Literal["Parse", "Extract", "Split", "Edit", "Pipeline", "Classify", "Chart"]] = None JobGetResponse: TypeAlias = Union[AsyncJobResponse, EnhancedAsyncJobResponse] diff --git a/src/reducto/types/parse_run_job_params.py b/src/reducto/types/parse_run_job_params.py index 905575dd..4b23b67a 100644 --- a/src/reducto/types/parse_run_job_params.py +++ b/src/reducto/types/parse_run_job_params.py @@ -36,10 +36,11 @@ class ParseRunJobParams(TypedDict, total=False): formatting: FormattingParam - queue_priority: Literal["auto", "batch"] + queue_priority: Literal["auto", "standard", "batch"] """Queue priority. - 'batch' for non-urgent work that processes when spare GPU capacity is available. + 'batch' places the job in a lower-priority queue for non-urgent bulk work. + 'auto' (alias: 'standard') uses the default queue. """ retrieval: RetrievalParam diff --git a/src/reducto/types/parse_run_params.py b/src/reducto/types/parse_run_params.py index bd83ad14..ee4c6b9d 100644 --- a/src/reducto/types/parse_run_params.py +++ b/src/reducto/types/parse_run_params.py @@ -63,10 +63,11 @@ class AsyncParseConfig(TypedDict, total=False): formatting: FormattingParam - queue_priority: Literal["auto", "batch"] + queue_priority: Literal["auto", "standard", "batch"] """Queue priority. - 'batch' for non-urgent work that processes when spare GPU capacity is available. + 'batch' places the job in a lower-priority queue for non-urgent bulk work. + 'auto' (alias: 'standard') uses the default queue. """ retrieval: RetrievalParam diff --git a/src/reducto/types/parse_run_response.py b/src/reducto/types/parse_run_response.py index 18997793..0116e297 100644 --- a/src/reducto/types/parse_run_response.py +++ b/src/reducto/types/parse_run_response.py @@ -1,9 +1,12 @@ from typing import Union -from typing_extensions import TypeAlias +from typing_extensions import Annotated, TypeAlias +from .._utils import PropertyInfo from .shared.parse_response import ParseResponse from .shared.async_parse_response import AsyncParseResponse __all__ = ["ParseRunResponse"] -ParseRunResponse: TypeAlias = Union[ParseResponse, AsyncParseResponse] +ParseRunResponse: TypeAlias = Annotated[ + Union[ParseResponse, AsyncParseResponse], PropertyInfo(discriminator="response_type") +] diff --git a/src/reducto/types/parse_usage.py b/src/reducto/types/parse_usage.py index 60e09721..12c8c29f 100644 --- a/src/reducto/types/parse_usage.py +++ b/src/reducto/types/parse_usage.py @@ -1,8 +1,77 @@ -from typing import Dict, Optional +from typing import Dict, List, Union, Optional +from typing_extensions import Literal, TypeAlias from .._models import BaseModel -__all__ = ["ParseUsage"] +__all__ = [ + "ParseUsage", + "PageBillingFeature", + "UsageBreakdown", + "UsageBreakdownParseUsageBreakdown", + "UsageBreakdownSplitUsageBreakdown", + "UsageBreakdownEditUsageBreakdown", +] + +PageBillingFeature: TypeAlias = Literal[ + "page", + "html_page", + "docx_native_page", + "agentic", + "complex", + "chart_agent", + "spreadsheet_cells", + "billable_spreadsheet_pages", + "enrich_table", + "figure_summary", + "table_summary", + "key_value", + "agentic_text", + "promptable_agentic_text", + "reducto_lite_page", +] + + +class UsageBreakdownParseUsageBreakdown(BaseModel): + parse_model: Literal["R-1", "Legacy"] + + tier: Literal["Default", "Batch"] + + charts: Optional[int] = None + + legacy_parse_credits: Optional[float] = None + + ocr_pages: Optional[int] = None + + parse_native_pages: Optional[int] = None + + parse_pages: Optional[int] = None + + prompted_blocks: Optional[int] = None + + +class UsageBreakdownSplitUsageBreakdown(BaseModel): + split_model: Literal["Split", "Deep Split"] + + charts: Optional[int] = None + + ocr_pages: Optional[int] = None + + prompted_blocks: Optional[int] = None + + split_pages: Optional[int] = None + + +class UsageBreakdownEditUsageBreakdown(BaseModel): + edit_model: Literal["Normal", "Prefill"] + + edit_pages: Optional[int] = None + + prefill_pages: Optional[int] = None + + +UsageBreakdown: TypeAlias = Union[ + UsageBreakdownParseUsageBreakdown, UsageBreakdownSplitUsageBreakdown, UsageBreakdownEditUsageBreakdown +] class ParseUsage(BaseModel): @@ -11,3 +80,20 @@ class ParseUsage(BaseModel): credit_breakdown: Optional[Dict[str, float]] = None credits: Optional[float] = None + + non_empty_cell_count: Optional[int] = None + """Total non-empty cells across all sheets. Only set for spreadsheet inputs.""" + + page_billing_breakdown: Optional[Dict[str, List[PageBillingFeature]]] = None + """Per-page breakdown of features used. + + Maps 1-indexed page numbers (as strings) to the list of billing features applied + on that page (e.g. 'page', 'complex', 'chart_agent'). + """ + + usage_breakdown: Optional[UsageBreakdown] = None + """Raw usage quantities. + + Only set for accounts on the new pricing model; credit fields are omitted for + those accounts. + """ diff --git a/src/reducto/types/settings_param.py b/src/reducto/types/settings_param.py index e819c2e3..ab5b7675 100644 --- a/src/reducto/types/settings_param.py +++ b/src/reducto/types/settings_param.py @@ -1,16 +1,43 @@ from __future__ import annotations from typing import List, Union, Iterable, Optional -from typing_extensions import Literal, TypeAlias, TypedDict +from typing_extensions import Literal, Required, TypeAlias, TypedDict from .._types import SequenceNotStr from .shared_params import page_range -__all__ = ["SettingsParam", "PageRange"] +__all__ = ["SettingsParam", "HybridVpc", "PageRange", "TenantThrottling"] + + +class HybridVpc(TypedDict, total=False): + """Hybrid VPC request-scoped settings.""" + + environment: Optional[str] + """Named Hybrid VPC environment to use for this request. + + Only applies when your organization has Hybrid VPC environments configured. + """ + PageRange: TypeAlias = Union[page_range.PageRange, Iterable[page_range.PageRange], Iterable[int], SequenceNotStr[str]] +class TenantThrottling(TypedDict, total=False): + """Per-tenant throttling for multi-tenant applications.""" + + tenant_id: Required[str] + """ + Your identifier for the tenant (customer, workspace, organization) this request + belongs to. Used only for noisy-neighbor throttling inside your account. + """ + + max_share: float + """ + Maximum fraction of your account's concurrency ceiling this tenant may use, + between 0 (exclusive) and 1. Defaults to 0.5. + """ + + class SettingsParam(TypedDict, total=False): document_password: Optional[str] """Password to decrypt password-protected documents.""" @@ -18,6 +45,20 @@ class SettingsParam(TypedDict, total=False): embed_pdf_metadata: bool """If True, embed OCR metadata into the returned PDF. Defaults to False.""" + embed_pdf_metadata_dpi: int + """ + Render DPI used when rasterizing the source PDF before embedding the OCR text + layer (only applies when `embed_pdf_metadata` is True). Lower values produce + dramatically smaller output PDFs; higher values preserve more detail when zoomed + past 200%. Defaults to 100. + """ + + extract_document_properties: bool + """If True, return properties embedded in the original document. + + Defaults to False. + """ + extraction_mode: Literal["ocr", "hybrid"] """The mode to use for text extraction from PDFs. @@ -31,6 +72,9 @@ class SettingsParam(TypedDict, total=False): force_url_result: bool """Force the result to be returned in URL form.""" + hybrid_vpc: HybridVpc + """Hybrid VPC request-scoped settings.""" + model: Optional[Literal["r-1", "legacy"]] """The parse model used for the request. @@ -64,5 +108,12 @@ class SettingsParam(TypedDict, total=False): return_ocr_data: bool """If True, return OCR data in the result. Defaults to False.""" + tenant_throttling: Optional[TenantThrottling] + """Per-tenant throttling for multi-tenant applications. + + Tag each request with your tenant's id to bound how much of your account's + concurrency a single tenant can consume. Account-level throttles still apply. + """ + timeout: Optional[float] """The timeout for the job in seconds.""" diff --git a/src/reducto/types/shared/__init__.py b/src/reducto/types/shared/__init__.py index 710bec6d..82e71f36 100644 --- a/src/reducto/types/shared/__init__.py +++ b/src/reducto/types/shared/__init__.py @@ -1,10 +1,13 @@ from .upload import Upload as Upload from .chunking import Chunking as Chunking from .page_range import PageRange as PageRange +from .url_result import URLResult as URLResult +from .error_detail import ErrorCode as ErrorCode, ErrorDetail as ErrorDetail from .text_agentic import TextAgentic as TextAgentic from .edit_response import EditResponse as EditResponse from .enrich_config import EnrichConfig as EnrichConfig from .table_agentic import TableAgentic as TableAgentic +from .chart_response import ChartResponse as ChartResponse from .figure_agentic import FigureAgentic as FigureAgentic from .parse_response import ParseResponse as ParseResponse from .split_response import SplitResponse as SplitResponse @@ -15,6 +18,7 @@ from .split_large_tables import SplitLargeTables as SplitLargeTables from .webhook_config_new import WebhookConfigNew as WebhookConfigNew from .async_edit_response import AsyncEditResponse as AsyncEditResponse +from .document_properties import DocumentProperties as DocumentProperties from .svix_webhook_config import SvixWebhookConfig as SvixWebhookConfig from .array_extract_config import ArrayExtractConfig as ArrayExtractConfig from .async_parse_response import AsyncParseResponse as AsyncParseResponse diff --git a/src/reducto/types/shared/chart_response.py b/src/reducto/types/shared/chart_response.py new file mode 100644 index 00000000..52f9a48d --- /dev/null +++ b/src/reducto/types/shared/chart_response.py @@ -0,0 +1,34 @@ +from typing import Dict, Optional +from typing_extensions import Literal + +from ..._models import BaseModel + +__all__ = ["ChartResponse", "Result", "Usage"] + + +class Result(BaseModel): + chart_data: Dict[str, object] + + reconstruction_url: str + + summary: str + + verified: bool + + +class Usage(BaseModel): + credits: Optional[float] = None + + num_charts: Optional[Literal[1]] = None + + +class ChartResponse(BaseModel): + job_id: str + + result: Result + + duration: Optional[float] = None + + response_type: Literal["chart"] + + usage: Optional[Usage] = None diff --git a/src/reducto/types/shared/classify_response.py b/src/reducto/types/shared/classify_response.py index e86ef464..1e99a564 100644 --- a/src/reducto/types/shared/classify_response.py +++ b/src/reducto/types/shared/classify_response.py @@ -1,21 +1,28 @@ -from typing import List, Optional -from typing_extensions import Literal +from typing import Dict, List, Union, Optional +from typing_extensions import Literal, TypeAlias from ..._models import BaseModel +from .url_result import URLResult __all__ = [ "ClassifyResponse", "Result", + "ResultCategory", "ResponseConfidence", "ResponseConfidenceCategory", "ResponseConfidenceCategoryCriteriaConfidence", + "Usage", + "UsageUsageBreakdown", ] -class Result(BaseModel): +class ResultCategory(BaseModel): category: str +Result: TypeAlias = Union[ResultCategory, URLResult] + + class ResponseConfidenceCategoryCriteriaConfidence(BaseModel): """Confidence result for a single criterion.""" @@ -40,15 +47,50 @@ class ResponseConfidence(BaseModel): categories: List[ResponseConfidenceCategory] +class UsageUsageBreakdown(BaseModel): + classify_model: Literal["Classify", "Deep Classify"] + + classify_pages: Optional[int] = None + + +class Usage(BaseModel): + num_categories: int + + num_pages: int + + credits: Optional[float] = None + + usage_breakdown: Optional[UsageUsageBreakdown] = None + """Raw usage quantities. + + Only set for accounts on the new pricing model; credit fields are omitted for + those accounts. + """ + + class ClassifyResponse(BaseModel): """Response from classify job - returned when polling /job/{job_id}""" job_id: str result: Result + """The classify result. + + If force_url_result is True, this is returned as a URL result. + """ duration: Optional[float] = None """The duration of the classify request in seconds.""" + extra_metadata: Optional[Dict[str, str]] = None + """Additional metadata for the classify response. + + Contains `grouping` when the request set `category_groups`. Omitted when empty. + """ + response_confidence: Optional[ResponseConfidence] = None """Overall confidence breakdown for classification response.""" + + response_type: Literal["classify"] + + usage: Optional[Usage] = None diff --git a/src/reducto/types/shared/document_properties.py b/src/reducto/types/shared/document_properties.py new file mode 100644 index 00000000..1f7ea18d --- /dev/null +++ b/src/reducto/types/shared/document_properties.py @@ -0,0 +1,27 @@ +from typing import Optional + +from ..._models import BaseModel + +__all__ = ["DocumentProperties"] + + +class DocumentProperties(BaseModel): + """Properties embedded in the customer's original document.""" + + author: Optional[str] = None + + created_at: Optional[str] = None + + creator: Optional[str] = None + + keywords: Optional[str] = None + + last_modified_by: Optional[str] = None + + modified_at: Optional[str] = None + + producer: Optional[str] = None + + subject: Optional[str] = None + + title: Optional[str] = None diff --git a/src/reducto/types/shared/edit_response.py b/src/reducto/types/shared/edit_response.py index b094c381..36324a2a 100644 --- a/src/reducto/types/shared/edit_response.py +++ b/src/reducto/types/shared/edit_response.py @@ -1,4 +1,5 @@ from typing import List, Optional +from typing_extensions import Literal from ..._models import BaseModel from ..edit_widget import EditWidget @@ -17,6 +18,11 @@ class EditResponse(BaseModel): List of widgets with their types, descriptions, and bounding boxes. """ + job_id: Optional[str] = None + """The unique identifier for the edit job.""" + + response_type: Literal["edit"] + usage: Optional[ParseUsage] = None """ Usage information for the edit operation, including number of pages and credits diff --git a/src/reducto/types/shared/error_detail.py b/src/reducto/types/shared/error_detail.py new file mode 100644 index 00000000..854a00e7 --- /dev/null +++ b/src/reducto/types/shared/error_detail.py @@ -0,0 +1,63 @@ +from typing import Optional +from typing_extensions import Literal + +from ..._models import BaseModel + +__all__ = ["ErrorDetail", "ErrorCode"] + +ErrorCode = Literal[ + "TIMEOUT", + "CAPACITY_TIMEOUT", + "CUSTOMER_TIMEOUT", + "INTERNAL_ERROR", + "SERVICE_UNAVAILABLE", + "GPU_ALLOCATION_ERROR", + "GPU_POOL_SATURATED", + "BATCH_QUEUE_FULL", + "QUEUE_FULL", + "JOB_STATE_ERROR", + "DOCUMENT_CORRUPT", + "DOCUMENT_EMPTY", + "DOCUMENT_UNSUPPORTED", + "DOCUMENT_TOO_LARGE", + "OFFICE_CONVERSION_TOO_LARGE", + "IMAGE_TOO_LARGE", + "IMAGE_TOO_SMALL", + "IMAGE_INVALID_ASPECT_RATIO", + "DOCUMENT_PASSWORD_PROTECTED", + "FORM_FILL_FAILED", + "INTERNAL_INVARIANT_VIOLATION", + "GPU_UNAVAILABLE", + "CONTEXT_WINDOW_EXCEEDED", + "OUTPUT_LIMIT_EXCEEDED", + "PROCESSING_FAILED", + "INFERENCE_METHOD_UNSUPPORTED", + "SUBPROCESS_CRASHED", + "BATCH_ORPHANED", + "OVERSIZED_RESULT", + "LLM_OUTPUT_PARSE_FAILED", + "LLM_PROVIDER_ERROR", + "INVALID_CONFIG", + "INVALID_SCHEMA", + "AUTH_ERROR", + "NOT_APPLICABLE", + "REGION_UNAVAILABLE", + "NOT_FOUND", + "JOB_DELETION_IN_PROGRESS", + "JOB_DELETED", + "JOB_NOT_COMPLETE", + "JOB_CANCELLED", + "RATE_LIMIT", + "CELL_COUNT_EXCEEDED", + "URL_NOT_ALLOWED", +] + + +class ErrorDetail(BaseModel): + code: int + + message: str + + name: ErrorCode + + job_id: Optional[str] = None diff --git a/src/reducto/types/shared/extract_response.py b/src/reducto/types/shared/extract_response.py index 4f56b190..d2c7af38 100644 --- a/src/reducto/types/shared/extract_response.py +++ b/src/reducto/types/shared/extract_response.py @@ -1,25 +1,44 @@ -from typing import List, Optional +from typing import Dict, List, Union, Optional +from typing_extensions import Literal, TypeAlias from ..._models import BaseModel +from .url_result import URLResult from ..extract_usage import ExtractUsage -__all__ = ["ExtractResponse"] +__all__ = ["ExtractResponse", "Citations", "Result"] + +Citations: TypeAlias = Union[List[object], URLResult, None] + +Result: TypeAlias = Union[List[object], URLResult] class ExtractResponse(BaseModel): - citations: Optional[List[object]] = None - """The citations corresponding to the extracted response.""" + citations: Optional[Citations] = None + """The citations corresponding to the extracted response. + + If force_url_result is True and citations are present, this is returned as a + URL result. + """ - result: List[object] + result: Result """The extracted response in your provided schema. This is a list of dictionaries. If disable_chunking is True (default), then it - will be a list of length one. + will be a list of length one. If force_url_result is True, this is returned as a + URL result. """ usage: ExtractUsage job_id: Optional[str] = None + response_confidence: Optional[Dict[str, object]] = None + """ + Optional deep extract confidence metadata containing document-level confidence + plus a mirrored leaf-level confidence tree. + """ + + response_type: Literal["extract"] + studio_link: Optional[str] = None """The link to the studio pipeline for the document.""" diff --git a/src/reducto/types/shared/parse_response.py b/src/reducto/types/shared/parse_response.py index 5ddd8ea2..403086a7 100644 --- a/src/reducto/types/shared/parse_response.py +++ b/src/reducto/types/shared/parse_response.py @@ -4,6 +4,7 @@ from ..._models import BaseModel from ..parse_usage import ParseUsage from ..bounding_box import BoundingBox +from .document_properties import DocumentProperties __all__ = [ "ParseResponse", @@ -12,6 +13,7 @@ "ResultFullResultChunk", "ResultFullResultChunkBlock", "ResultFullResultChunkBlockGranularConfidence", + "ResultFullResultChunkBlockMergedTable", "ResultFullResultOcr", "ResultFullResultOcrLine", "ResultFullResultOcrWord", @@ -30,6 +32,15 @@ class ResultFullResultChunkBlockGranularConfidence(BaseModel): parse_confidence: Optional[float] = None +class ResultFullResultChunkBlockMergedTable(BaseModel): + bbox: BoundingBox + """The bounding box of the block extracted from the document.""" + + content: str + + image_url: Optional[str] = None + + class ResultFullResultChunkBlock(BaseModel): bbox: BoundingBox """The bounding box of the block extracted from the document.""" @@ -82,6 +93,9 @@ class ResultFullResultChunkBlock(BaseModel): image_url: Optional[str] = None """(Experimental) The URL of the image associated with the block.""" + merged_tables: Optional[List[ResultFullResultChunkBlockMergedTable]] = None + """Original table fragments that were combined into this table by merge_tables.""" + class ResultFullResultChunk(BaseModel): blocks: List[ResultFullResultChunkBlock] @@ -175,8 +189,13 @@ class ParseResponse(BaseModel): usage: ParseUsage + document_properties: Optional[DocumentProperties] = None + """Properties embedded in the customer's original document.""" + pdf_url: Optional[str] = None """The storage URL of the converted PDF file.""" + response_type: Literal["parse"] + studio_link: Optional[str] = None """The link to the studio pipeline for the document.""" diff --git a/src/reducto/types/shared/pipeline_response.py b/src/reducto/types/shared/pipeline_response.py index 6b2847ba..4bb53fc9 100644 --- a/src/reducto/types/shared/pipeline_response.py +++ b/src/reducto/types/shared/pipeline_response.py @@ -1,5 +1,5 @@ from typing import List, Union, Optional -from typing_extensions import TypeAlias +from typing_extensions import Literal, TypeAlias from ..._models import BaseModel from ..v3_extract import V3Extract @@ -54,3 +54,5 @@ class PipelineResponse(BaseModel): result: Result usage: ParseUsage + + response_type: Literal["pipeline"] diff --git a/src/reducto/types/shared/split_response.py b/src/reducto/types/shared/split_response.py index 83d95112..11e1ecb8 100644 --- a/src/reducto/types/shared/split_response.py +++ b/src/reducto/types/shared/split_response.py @@ -2,6 +2,7 @@ from typing_extensions import Literal, TypeAlias from ..._models import BaseModel +from .url_result import URLResult from ..parse_usage import ParseUsage from ..deep_split_page_evidence import DeepSplitPageEvidence @@ -59,11 +60,22 @@ class ResultDeepSplitResult(BaseModel): splits: List[ResultDeepSplitResultSplit] -Result: TypeAlias = Union[ResultSplitResult, ResultDeepSplitResult] +Result: TypeAlias = Union[ResultSplitResult, ResultDeepSplitResult, URLResult] class SplitResponse(BaseModel): result: Result - """The split result.""" + """The split result. + + If force_url_result is True, this is returned as a URL result. + """ usage: ParseUsage + + duration: Optional[float] = None + """The duration of the split request in seconds.""" + + job_id: Optional[str] = None + """The unique identifier for the split job.""" + + response_type: Literal["split"] diff --git a/src/reducto/types/shared/url_result.py b/src/reducto/types/shared/url_result.py new file mode 100644 index 00000000..de6683cd --- /dev/null +++ b/src/reducto/types/shared/url_result.py @@ -0,0 +1,14 @@ +from typing_extensions import Literal + +from ..._models import BaseModel + +__all__ = ["URLResult"] + + +class URLResult(BaseModel): + result_id: str + + type: Literal["url"] + """type = 'url'""" + + url: str diff --git a/src/reducto/types/shared_params/table_agentic.py b/src/reducto/types/shared_params/table_agentic.py index 46255f92..4764787d 100644 --- a/src/reducto/types/shared_params/table_agentic.py +++ b/src/reducto/types/shared_params/table_agentic.py @@ -9,5 +9,12 @@ class TableAgentic(TypedDict, total=False): scope: Required[Literal["table"]] + mode: Literal["default", "auto", "max"] + """Mode for table agentic. + + 'default' selectively applies enrichment only to tables likely to benefit, and + 'max' runs enrichment on all tables. + """ + prompt: Optional[str] """Custom prompt for table agentic.""" diff --git a/src/reducto/types/split_table_options_param.py b/src/reducto/types/split_table_options_param.py index 3cae31a3..b5571e0e 100644 --- a/src/reducto/types/split_table_options_param.py +++ b/src/reducto/types/split_table_options_param.py @@ -6,6 +6,29 @@ class SplitTableOptionsParam(TypedDict, total=False): + allow_page_overlap: bool + """If True, a page can belong to multiple categories/partitions. + + If False, each page must belong to exactly one category. Defaults to True. + """ + + auto_partition: bool + """ + If True (default), deep split may split a category into partitions even when + that category has no configured partition_key. If False, categories without a + partition_key are never partitioned, so partitioning happens only where you + explicitly configured a partition_key. + """ + + deep_split: bool + """If True, uses the deep split agent for higher-quality document splitting. + + Off by default. + """ + + force_url_result: bool + """Force the endpoint result to be returned in URL form.""" + table_cutoff: Literal["truncate", "preserve"] """ If tables should be truncated to the first few rows or if all content should be diff --git a/src/reducto/types/spreadsheet_param.py b/src/reducto/types/spreadsheet_param.py index 7f6b4ddd..b00c5ddb 100644 --- a/src/reducto/types/spreadsheet_param.py +++ b/src/reducto/types/spreadsheet_param.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import List +from typing import List, Optional from typing_extensions import Literal, TypedDict from .shared_params.split_large_tables import SplitLargeTables @@ -22,4 +22,11 @@ class SpreadsheetParam(TypedDict, total=False): include: List[Literal["cell_colors", "formula", "dropdowns"]] """Whether to include cell color, formula, and dropdown information in the output.""" + max_cell_count: Optional[int] + """Maximum total non-empty cells allowed across all sheets. + + If exceeded, the request is rejected with a 422 error. Set to null to disable + the limit. Defaults to null. + """ + split_large_tables: SplitLargeTables diff --git a/src/reducto/types/v3_extract.py b/src/reducto/types/v3_extract.py index e86fbe6a..56a1d27a 100644 --- a/src/reducto/types/v3_extract.py +++ b/src/reducto/types/v3_extract.py @@ -1,4 +1,5 @@ from typing import List, Union, Optional +from typing_extensions import Literal from .._models import BaseModel from .extract_usage import ExtractUsage @@ -16,7 +17,15 @@ class V3Extract(BaseModel): usage: ExtractUsage + confidence: Optional[Literal["high", "low"]] = None + """Optional document-level deep extract confidence label.""" + + confidence_reason: Optional[str] = None + """Optional explanation for the document-level confidence label.""" + job_id: Optional[str] = None + response_type: Literal["v3_extract"] + studio_link: Optional[str] = None """The link to the studio pipeline for the document.""" diff --git a/tests/api_resources/test_classify.py b/tests/api_resources/test_classify.py index 7b58a5ea..2ef969d9 100644 --- a/tests/api_resources/test_classify.py +++ b/tests/api_resources/test_classify.py @@ -28,6 +28,7 @@ def test_method_run(self, client: Reducto) -> None: def test_method_run_with_all_params(self, client: Reducto) -> None: classify = client.classify.run( input="string", + category_groups={"foo": ["string"]}, classification_schema=[ { "category": "category", @@ -35,11 +36,13 @@ def test_method_run_with_all_params(self, client: Reducto) -> None: } ], document_metadata="document_metadata", + force_url_result=True, + model="default", page_range={ "end": 0, "start": 0, }, - persist_results=True, + priority=True, ) assert_matches_type(ClassifyResponse, classify, path=["response"]) @@ -88,6 +91,7 @@ async def test_method_run(self, async_client: AsyncReducto) -> None: async def test_method_run_with_all_params(self, async_client: AsyncReducto) -> None: classify = await async_client.classify.run( input="string", + category_groups={"foo": ["string"]}, classification_schema=[ { "category": "category", @@ -95,11 +99,13 @@ async def test_method_run_with_all_params(self, async_client: AsyncReducto) -> N } ], document_metadata="document_metadata", + force_url_result=True, + model="default", page_range={ "end": 0, "start": 0, }, - persist_results=True, + priority=True, ) assert_matches_type(ClassifyResponse, classify, path=["response"]) diff --git a/tests/api_resources/test_extract.py b/tests/api_resources/test_extract.py index dee91042..7284e931 100644 --- a/tests/api_resources/test_extract.py +++ b/tests/api_resources/test_extract.py @@ -37,9 +37,11 @@ def test_method_run_with_all_params_overload_1(self, client: Reducto) -> None: }, parsing={ "enhance": { + "advanced_chart_agent": True, "agentic": [ { "scope": "table", + "mode": "default", "prompt": "prompt", } ], @@ -64,9 +66,12 @@ def test_method_run_with_all_params_overload_1(self, client: Reducto) -> None: "settings": { "document_password": "document_password", "embed_pdf_metadata": True, + "embed_pdf_metadata_dpi": 50, + "extract_document_properties": True, "extraction_mode": "ocr", "force_file_extension": "force_file_extension", "force_url_result": True, + "hybrid_vpc": {"environment": "environment"}, "ocr_system": "standard", "page_range": { "end": 0, @@ -75,12 +80,17 @@ def test_method_run_with_all_params_overload_1(self, client: Reducto) -> None: "persist_results": True, "return_images": ["figure"], "return_ocr_data": True, + "tenant_throttling": { + "tenant_id": "tenant_id", + "max_share": 0, + }, "timeout": 0, }, "spreadsheet": { "clustering": "accurate", "exclude": ["hidden_sheets"], "include": ["cell_colors"], + "max_cell_count": 0, "split_large_tables": { "enabled": True, "size": 0, @@ -92,10 +102,16 @@ def test_method_run_with_all_params_overload_1(self, client: Reducto) -> None: "citations": { "enabled": True, "numerical_confidence": True, + "parent_block": "full", }, "deep_extract": True, + "force_url_result": True, "include_images": True, "optimize_for_latency": True, + "page_range": { + "end": 0, + "start": 0, + }, }, ) assert_matches_type(ExtractRunResponse, extract, path=["response"]) @@ -153,9 +169,11 @@ def test_method_run_with_all_params_overload_2(self, client: Reducto) -> None: }, parsing={ "enhance": { + "advanced_chart_agent": True, "agentic": [ { "scope": "table", + "mode": "default", "prompt": "prompt", } ], @@ -180,9 +198,12 @@ def test_method_run_with_all_params_overload_2(self, client: Reducto) -> None: "settings": { "document_password": "document_password", "embed_pdf_metadata": True, + "embed_pdf_metadata_dpi": 50, + "extract_document_properties": True, "extraction_mode": "ocr", "force_file_extension": "force_file_extension", "force_url_result": True, + "hybrid_vpc": {"environment": "environment"}, "ocr_system": "standard", "page_range": { "end": 0, @@ -191,27 +212,39 @@ def test_method_run_with_all_params_overload_2(self, client: Reducto) -> None: "persist_results": True, "return_images": ["figure"], "return_ocr_data": True, + "tenant_throttling": { + "tenant_id": "tenant_id", + "max_share": 0, + }, "timeout": 0, }, "spreadsheet": { "clustering": "accurate", "exclude": ["hidden_sheets"], "include": ["cell_colors"], + "max_cell_count": 0, "split_large_tables": { "enabled": True, "size": 0, }, }, }, + queue_priority="auto", settings={ "array_extract": True, "citations": { "enabled": True, "numerical_confidence": True, + "parent_block": "full", }, "deep_extract": True, + "force_url_result": True, "include_images": True, "optimize_for_latency": True, + "page_range": { + "end": 0, + "start": 0, + }, }, ) assert_matches_type(ExtractRunResponse, extract, path=["response"]) @@ -269,9 +302,11 @@ def test_method_run_job_with_all_params(self, client: Reducto) -> None: }, parsing={ "enhance": { + "advanced_chart_agent": True, "agentic": [ { "scope": "table", + "mode": "default", "prompt": "prompt", } ], @@ -296,9 +331,12 @@ def test_method_run_job_with_all_params(self, client: Reducto) -> None: "settings": { "document_password": "document_password", "embed_pdf_metadata": True, + "embed_pdf_metadata_dpi": 50, + "extract_document_properties": True, "extraction_mode": "ocr", "force_file_extension": "force_file_extension", "force_url_result": True, + "hybrid_vpc": {"environment": "environment"}, "ocr_system": "standard", "page_range": { "end": 0, @@ -307,27 +345,39 @@ def test_method_run_job_with_all_params(self, client: Reducto) -> None: "persist_results": True, "return_images": ["figure"], "return_ocr_data": True, + "tenant_throttling": { + "tenant_id": "tenant_id", + "max_share": 0, + }, "timeout": 0, }, "spreadsheet": { "clustering": "accurate", "exclude": ["hidden_sheets"], "include": ["cell_colors"], + "max_cell_count": 0, "split_large_tables": { "enabled": True, "size": 0, }, }, }, + queue_priority="auto", settings={ "array_extract": True, "citations": { "enabled": True, "numerical_confidence": True, + "parent_block": "full", }, "deep_extract": True, + "force_url_result": True, "include_images": True, "optimize_for_latency": True, + "page_range": { + "end": 0, + "start": 0, + }, }, ) assert_matches_type(AsyncExtractResponse, extract, path=["response"]) @@ -383,9 +433,11 @@ async def test_method_run_with_all_params_overload_1(self, async_client: AsyncRe }, parsing={ "enhance": { + "advanced_chart_agent": True, "agentic": [ { "scope": "table", + "mode": "default", "prompt": "prompt", } ], @@ -410,9 +462,12 @@ async def test_method_run_with_all_params_overload_1(self, async_client: AsyncRe "settings": { "document_password": "document_password", "embed_pdf_metadata": True, + "embed_pdf_metadata_dpi": 50, + "extract_document_properties": True, "extraction_mode": "ocr", "force_file_extension": "force_file_extension", "force_url_result": True, + "hybrid_vpc": {"environment": "environment"}, "ocr_system": "standard", "page_range": { "end": 0, @@ -421,12 +476,17 @@ async def test_method_run_with_all_params_overload_1(self, async_client: AsyncRe "persist_results": True, "return_images": ["figure"], "return_ocr_data": True, + "tenant_throttling": { + "tenant_id": "tenant_id", + "max_share": 0, + }, "timeout": 0, }, "spreadsheet": { "clustering": "accurate", "exclude": ["hidden_sheets"], "include": ["cell_colors"], + "max_cell_count": 0, "split_large_tables": { "enabled": True, "size": 0, @@ -438,10 +498,16 @@ async def test_method_run_with_all_params_overload_1(self, async_client: AsyncRe "citations": { "enabled": True, "numerical_confidence": True, + "parent_block": "full", }, "deep_extract": True, + "force_url_result": True, "include_images": True, "optimize_for_latency": True, + "page_range": { + "end": 0, + "start": 0, + }, }, ) assert_matches_type(ExtractRunResponse, extract, path=["response"]) @@ -499,9 +565,11 @@ async def test_method_run_with_all_params_overload_2(self, async_client: AsyncRe }, parsing={ "enhance": { + "advanced_chart_agent": True, "agentic": [ { "scope": "table", + "mode": "default", "prompt": "prompt", } ], @@ -526,9 +594,12 @@ async def test_method_run_with_all_params_overload_2(self, async_client: AsyncRe "settings": { "document_password": "document_password", "embed_pdf_metadata": True, + "embed_pdf_metadata_dpi": 50, + "extract_document_properties": True, "extraction_mode": "ocr", "force_file_extension": "force_file_extension", "force_url_result": True, + "hybrid_vpc": {"environment": "environment"}, "ocr_system": "standard", "page_range": { "end": 0, @@ -537,27 +608,39 @@ async def test_method_run_with_all_params_overload_2(self, async_client: AsyncRe "persist_results": True, "return_images": ["figure"], "return_ocr_data": True, + "tenant_throttling": { + "tenant_id": "tenant_id", + "max_share": 0, + }, "timeout": 0, }, "spreadsheet": { "clustering": "accurate", "exclude": ["hidden_sheets"], "include": ["cell_colors"], + "max_cell_count": 0, "split_large_tables": { "enabled": True, "size": 0, }, }, }, + queue_priority="auto", settings={ "array_extract": True, "citations": { "enabled": True, "numerical_confidence": True, + "parent_block": "full", }, "deep_extract": True, + "force_url_result": True, "include_images": True, "optimize_for_latency": True, + "page_range": { + "end": 0, + "start": 0, + }, }, ) assert_matches_type(ExtractRunResponse, extract, path=["response"]) @@ -615,9 +698,11 @@ async def test_method_run_job_with_all_params(self, async_client: AsyncReducto) }, parsing={ "enhance": { + "advanced_chart_agent": True, "agentic": [ { "scope": "table", + "mode": "default", "prompt": "prompt", } ], @@ -642,9 +727,12 @@ async def test_method_run_job_with_all_params(self, async_client: AsyncReducto) "settings": { "document_password": "document_password", "embed_pdf_metadata": True, + "embed_pdf_metadata_dpi": 50, + "extract_document_properties": True, "extraction_mode": "ocr", "force_file_extension": "force_file_extension", "force_url_result": True, + "hybrid_vpc": {"environment": "environment"}, "ocr_system": "standard", "page_range": { "end": 0, @@ -653,27 +741,39 @@ async def test_method_run_job_with_all_params(self, async_client: AsyncReducto) "persist_results": True, "return_images": ["figure"], "return_ocr_data": True, + "tenant_throttling": { + "tenant_id": "tenant_id", + "max_share": 0, + }, "timeout": 0, }, "spreadsheet": { "clustering": "accurate", "exclude": ["hidden_sheets"], "include": ["cell_colors"], + "max_cell_count": 0, "split_large_tables": { "enabled": True, "size": 0, }, }, }, + queue_priority="auto", settings={ "array_extract": True, "citations": { "enabled": True, "numerical_confidence": True, + "parent_block": "full", }, "deep_extract": True, + "force_url_result": True, "include_images": True, "optimize_for_latency": True, + "page_range": { + "end": 0, + "start": 0, + }, }, ) assert_matches_type(AsyncExtractResponse, extract, path=["response"]) diff --git a/tests/api_resources/test_job.py b/tests/api_resources/test_job.py index 1c5c8bd3..6308556c 100644 --- a/tests/api_resources/test_job.py +++ b/tests/api_resources/test_job.py @@ -7,7 +7,7 @@ from reducto import Reducto, AsyncReducto from tests.utils import assert_matches_type -from reducto.types import JobGetResponse, JobGetAllResponse +from reducto.types import JobGetResponse, JobDeleteResponse, JobGetAllResponse base_url = os.environ.get("TEST_API_BASE_URL", "http://127.0.0.1:4010") @@ -57,6 +57,57 @@ def test_path_params_cancel(self, client: Reducto) -> None: "", ) + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_method_delete(self, client: Reducto) -> None: + job = client.job.delete( + "job_id", + ) + assert_matches_type(JobDeleteResponse, job, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_method_delete_with_all_params(self, client: Reducto) -> None: + job = client.job.delete( + "job_id", + include_persisted=True, + ) + assert_matches_type(JobDeleteResponse, job, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_raw_response_delete(self, client: Reducto) -> None: + response = client.job.with_raw_response.delete( + "job_id", + ) + + assert response.is_closed is True + assert response.http_request.headers.get("X-Reducto-Lang") == "python" + job = response.parse() + assert_matches_type(JobDeleteResponse, job, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_streaming_response_delete(self, client: Reducto) -> None: + with client.job.with_streaming_response.delete( + "job_id", + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Reducto-Lang") == "python" + + job = response.parse() + assert_matches_type(JobDeleteResponse, job, path=["response"]) + + assert cast(Any, response.is_closed) is True + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_path_params_delete(self, client: Reducto) -> None: + with pytest.raises(ValueError, match=r"Expected a non-empty value for `job_id` but received ''"): + client.job.with_raw_response.delete( + "", + ) + @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize def test_method_get(self, client: Reducto) -> None: @@ -185,6 +236,57 @@ async def test_path_params_cancel(self, async_client: AsyncReducto) -> None: "", ) + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_method_delete(self, async_client: AsyncReducto) -> None: + job = await async_client.job.delete( + "job_id", + ) + assert_matches_type(JobDeleteResponse, job, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_method_delete_with_all_params(self, async_client: AsyncReducto) -> None: + job = await async_client.job.delete( + "job_id", + include_persisted=True, + ) + assert_matches_type(JobDeleteResponse, job, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_raw_response_delete(self, async_client: AsyncReducto) -> None: + response = await async_client.job.with_raw_response.delete( + "job_id", + ) + + assert response.is_closed is True + assert response.http_request.headers.get("X-Reducto-Lang") == "python" + job = await response.parse() + assert_matches_type(JobDeleteResponse, job, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_streaming_response_delete(self, async_client: AsyncReducto) -> None: + async with async_client.job.with_streaming_response.delete( + "job_id", + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Reducto-Lang") == "python" + + job = await response.parse() + assert_matches_type(JobDeleteResponse, job, path=["response"]) + + assert cast(Any, response.is_closed) is True + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_path_params_delete(self, async_client: AsyncReducto) -> None: + with pytest.raises(ValueError, match=r"Expected a non-empty value for `job_id` but received ''"): + await async_client.job.with_raw_response.delete( + "", + ) + @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize async def test_method_get(self, async_client: AsyncReducto) -> None: diff --git a/tests/api_resources/test_parse.py b/tests/api_resources/test_parse.py index 2718d841..5eeecef2 100644 --- a/tests/api_resources/test_parse.py +++ b/tests/api_resources/test_parse.py @@ -32,9 +32,11 @@ def test_method_run_with_all_params_overload_1(self, client: Reducto) -> None: parse = client.parse.run( input="string", enhance={ + "advanced_chart_agent": True, "agentic": [ { "scope": "table", + "mode": "default", "prompt": "prompt", } ], @@ -59,9 +61,12 @@ def test_method_run_with_all_params_overload_1(self, client: Reducto) -> None: settings={ "document_password": "document_password", "embed_pdf_metadata": True, + "embed_pdf_metadata_dpi": 50, + "extract_document_properties": True, "extraction_mode": "ocr", "force_file_extension": "force_file_extension", "force_url_result": True, + "hybrid_vpc": {"environment": "environment"}, "ocr_system": "standard", "page_range": { "end": 0, @@ -70,12 +75,17 @@ def test_method_run_with_all_params_overload_1(self, client: Reducto) -> None: "persist_results": True, "return_images": ["figure"], "return_ocr_data": True, + "tenant_throttling": { + "tenant_id": "tenant_id", + "max_share": 0, + }, "timeout": 0, }, spreadsheet={ "clustering": "accurate", "exclude": ["hidden_sheets"], "include": ["cell_colors"], + "max_cell_count": 0, "split_large_tables": { "enabled": True, "size": 0, @@ -132,9 +142,11 @@ def test_method_run_with_all_params_overload_2(self, client: Reducto) -> None: }, }, enhance={ + "advanced_chart_agent": True, "agentic": [ { "scope": "table", + "mode": "default", "prompt": "prompt", } ], @@ -160,9 +172,12 @@ def test_method_run_with_all_params_overload_2(self, client: Reducto) -> None: settings={ "document_password": "document_password", "embed_pdf_metadata": True, + "embed_pdf_metadata_dpi": 50, + "extract_document_properties": True, "extraction_mode": "ocr", "force_file_extension": "force_file_extension", "force_url_result": True, + "hybrid_vpc": {"environment": "environment"}, "ocr_system": "standard", "page_range": { "end": 0, @@ -171,12 +186,17 @@ def test_method_run_with_all_params_overload_2(self, client: Reducto) -> None: "persist_results": True, "return_images": ["figure"], "return_ocr_data": True, + "tenant_throttling": { + "tenant_id": "tenant_id", + "max_share": 0, + }, "timeout": 0, }, spreadsheet={ "clustering": "accurate", "exclude": ["hidden_sheets"], "include": ["cell_colors"], + "max_cell_count": 0, "split_large_tables": { "enabled": True, "size": 0, @@ -233,9 +253,11 @@ def test_method_run_job_with_all_params(self, client: Reducto) -> None: }, }, enhance={ + "advanced_chart_agent": True, "agentic": [ { "scope": "table", + "mode": "default", "prompt": "prompt", } ], @@ -261,9 +283,12 @@ def test_method_run_job_with_all_params(self, client: Reducto) -> None: settings={ "document_password": "document_password", "embed_pdf_metadata": True, + "embed_pdf_metadata_dpi": 50, + "extract_document_properties": True, "extraction_mode": "ocr", "force_file_extension": "force_file_extension", "force_url_result": True, + "hybrid_vpc": {"environment": "environment"}, "ocr_system": "standard", "page_range": { "end": 0, @@ -272,12 +297,17 @@ def test_method_run_job_with_all_params(self, client: Reducto) -> None: "persist_results": True, "return_images": ["figure"], "return_ocr_data": True, + "tenant_throttling": { + "tenant_id": "tenant_id", + "max_share": 0, + }, "timeout": 0, }, spreadsheet={ "clustering": "accurate", "exclude": ["hidden_sheets"], "include": ["cell_colors"], + "max_cell_count": 0, "split_large_tables": { "enabled": True, "size": 0, @@ -332,9 +362,11 @@ async def test_method_run_with_all_params_overload_1(self, async_client: AsyncRe parse = await async_client.parse.run( input="string", enhance={ + "advanced_chart_agent": True, "agentic": [ { "scope": "table", + "mode": "default", "prompt": "prompt", } ], @@ -359,9 +391,12 @@ async def test_method_run_with_all_params_overload_1(self, async_client: AsyncRe settings={ "document_password": "document_password", "embed_pdf_metadata": True, + "embed_pdf_metadata_dpi": 50, + "extract_document_properties": True, "extraction_mode": "ocr", "force_file_extension": "force_file_extension", "force_url_result": True, + "hybrid_vpc": {"environment": "environment"}, "ocr_system": "standard", "page_range": { "end": 0, @@ -370,12 +405,17 @@ async def test_method_run_with_all_params_overload_1(self, async_client: AsyncRe "persist_results": True, "return_images": ["figure"], "return_ocr_data": True, + "tenant_throttling": { + "tenant_id": "tenant_id", + "max_share": 0, + }, "timeout": 0, }, spreadsheet={ "clustering": "accurate", "exclude": ["hidden_sheets"], "include": ["cell_colors"], + "max_cell_count": 0, "split_large_tables": { "enabled": True, "size": 0, @@ -432,9 +472,11 @@ async def test_method_run_with_all_params_overload_2(self, async_client: AsyncRe }, }, enhance={ + "advanced_chart_agent": True, "agentic": [ { "scope": "table", + "mode": "default", "prompt": "prompt", } ], @@ -460,9 +502,12 @@ async def test_method_run_with_all_params_overload_2(self, async_client: AsyncRe settings={ "document_password": "document_password", "embed_pdf_metadata": True, + "embed_pdf_metadata_dpi": 50, + "extract_document_properties": True, "extraction_mode": "ocr", "force_file_extension": "force_file_extension", "force_url_result": True, + "hybrid_vpc": {"environment": "environment"}, "ocr_system": "standard", "page_range": { "end": 0, @@ -471,12 +516,17 @@ async def test_method_run_with_all_params_overload_2(self, async_client: AsyncRe "persist_results": True, "return_images": ["figure"], "return_ocr_data": True, + "tenant_throttling": { + "tenant_id": "tenant_id", + "max_share": 0, + }, "timeout": 0, }, spreadsheet={ "clustering": "accurate", "exclude": ["hidden_sheets"], "include": ["cell_colors"], + "max_cell_count": 0, "split_large_tables": { "enabled": True, "size": 0, @@ -533,9 +583,11 @@ async def test_method_run_job_with_all_params(self, async_client: AsyncReducto) }, }, enhance={ + "advanced_chart_agent": True, "agentic": [ { "scope": "table", + "mode": "default", "prompt": "prompt", } ], @@ -561,9 +613,12 @@ async def test_method_run_job_with_all_params(self, async_client: AsyncReducto) settings={ "document_password": "document_password", "embed_pdf_metadata": True, + "embed_pdf_metadata_dpi": 50, + "extract_document_properties": True, "extraction_mode": "ocr", "force_file_extension": "force_file_extension", "force_url_result": True, + "hybrid_vpc": {"environment": "environment"}, "ocr_system": "standard", "page_range": { "end": 0, @@ -572,12 +627,17 @@ async def test_method_run_job_with_all_params(self, async_client: AsyncReducto) "persist_results": True, "return_images": ["figure"], "return_ocr_data": True, + "tenant_throttling": { + "tenant_id": "tenant_id", + "max_share": 0, + }, "timeout": 0, }, spreadsheet={ "clustering": "accurate", "exclude": ["hidden_sheets"], "include": ["cell_colors"], + "max_cell_count": 0, "split_large_tables": { "enabled": True, "size": 0, diff --git a/tests/api_resources/test_split.py b/tests/api_resources/test_split.py index 46ef6933..cd01a266 100644 --- a/tests/api_resources/test_split.py +++ b/tests/api_resources/test_split.py @@ -43,9 +43,11 @@ def test_method_run_with_all_params(self, client: Reducto) -> None: ], parsing={ "enhance": { + "advanced_chart_agent": True, "agentic": [ { "scope": "table", + "mode": "default", "prompt": "prompt", } ], @@ -70,9 +72,12 @@ def test_method_run_with_all_params(self, client: Reducto) -> None: "settings": { "document_password": "document_password", "embed_pdf_metadata": True, + "embed_pdf_metadata_dpi": 50, + "extract_document_properties": True, "extraction_mode": "ocr", "force_file_extension": "force_file_extension", "force_url_result": True, + "hybrid_vpc": {"environment": "environment"}, "ocr_system": "standard", "page_range": { "end": 0, @@ -81,19 +86,30 @@ def test_method_run_with_all_params(self, client: Reducto) -> None: "persist_results": True, "return_images": ["figure"], "return_ocr_data": True, + "tenant_throttling": { + "tenant_id": "tenant_id", + "max_share": 0, + }, "timeout": 0, }, "spreadsheet": { "clustering": "accurate", "exclude": ["hidden_sheets"], "include": ["cell_colors"], + "max_cell_count": 0, "split_large_tables": { "enabled": True, "size": 0, }, }, }, - settings={"table_cutoff": "truncate"}, + settings={ + "allow_page_overlap": True, + "auto_partition": True, + "deep_split": True, + "force_url_result": True, + "table_cutoff": "truncate", + }, split_rules="split_rules", ) assert_matches_type(SplitResponse, split, path=["response"]) @@ -172,9 +188,11 @@ def test_method_run_job_with_all_params(self, client: Reducto) -> None: }, parsing={ "enhance": { + "advanced_chart_agent": True, "agentic": [ { "scope": "table", + "mode": "default", "prompt": "prompt", } ], @@ -199,9 +217,12 @@ def test_method_run_job_with_all_params(self, client: Reducto) -> None: "settings": { "document_password": "document_password", "embed_pdf_metadata": True, + "embed_pdf_metadata_dpi": 50, + "extract_document_properties": True, "extraction_mode": "ocr", "force_file_extension": "force_file_extension", "force_url_result": True, + "hybrid_vpc": {"environment": "environment"}, "ocr_system": "standard", "page_range": { "end": 0, @@ -210,19 +231,30 @@ def test_method_run_job_with_all_params(self, client: Reducto) -> None: "persist_results": True, "return_images": ["figure"], "return_ocr_data": True, + "tenant_throttling": { + "tenant_id": "tenant_id", + "max_share": 0, + }, "timeout": 0, }, "spreadsheet": { "clustering": "accurate", "exclude": ["hidden_sheets"], "include": ["cell_colors"], + "max_cell_count": 0, "split_large_tables": { "enabled": True, "size": 0, }, }, }, - settings={"table_cutoff": "truncate"}, + settings={ + "allow_page_overlap": True, + "auto_partition": True, + "deep_split": True, + "force_url_result": True, + "table_cutoff": "truncate", + }, split_rules="split_rules", ) assert_matches_type(AsyncSplitResponse, split, path=["response"]) @@ -299,9 +331,11 @@ async def test_method_run_with_all_params(self, async_client: AsyncReducto) -> N ], parsing={ "enhance": { + "advanced_chart_agent": True, "agentic": [ { "scope": "table", + "mode": "default", "prompt": "prompt", } ], @@ -326,9 +360,12 @@ async def test_method_run_with_all_params(self, async_client: AsyncReducto) -> N "settings": { "document_password": "document_password", "embed_pdf_metadata": True, + "embed_pdf_metadata_dpi": 50, + "extract_document_properties": True, "extraction_mode": "ocr", "force_file_extension": "force_file_extension", "force_url_result": True, + "hybrid_vpc": {"environment": "environment"}, "ocr_system": "standard", "page_range": { "end": 0, @@ -337,19 +374,30 @@ async def test_method_run_with_all_params(self, async_client: AsyncReducto) -> N "persist_results": True, "return_images": ["figure"], "return_ocr_data": True, + "tenant_throttling": { + "tenant_id": "tenant_id", + "max_share": 0, + }, "timeout": 0, }, "spreadsheet": { "clustering": "accurate", "exclude": ["hidden_sheets"], "include": ["cell_colors"], + "max_cell_count": 0, "split_large_tables": { "enabled": True, "size": 0, }, }, }, - settings={"table_cutoff": "truncate"}, + settings={ + "allow_page_overlap": True, + "auto_partition": True, + "deep_split": True, + "force_url_result": True, + "table_cutoff": "truncate", + }, split_rules="split_rules", ) assert_matches_type(SplitResponse, split, path=["response"]) @@ -428,9 +476,11 @@ async def test_method_run_job_with_all_params(self, async_client: AsyncReducto) }, parsing={ "enhance": { + "advanced_chart_agent": True, "agentic": [ { "scope": "table", + "mode": "default", "prompt": "prompt", } ], @@ -455,9 +505,12 @@ async def test_method_run_job_with_all_params(self, async_client: AsyncReducto) "settings": { "document_password": "document_password", "embed_pdf_metadata": True, + "embed_pdf_metadata_dpi": 50, + "extract_document_properties": True, "extraction_mode": "ocr", "force_file_extension": "force_file_extension", "force_url_result": True, + "hybrid_vpc": {"environment": "environment"}, "ocr_system": "standard", "page_range": { "end": 0, @@ -466,19 +519,30 @@ async def test_method_run_job_with_all_params(self, async_client: AsyncReducto) "persist_results": True, "return_images": ["figure"], "return_ocr_data": True, + "tenant_throttling": { + "tenant_id": "tenant_id", + "max_share": 0, + }, "timeout": 0, }, "spreadsheet": { "clustering": "accurate", "exclude": ["hidden_sheets"], "include": ["cell_colors"], + "max_cell_count": 0, "split_large_tables": { "enabled": True, "size": 0, }, }, }, - settings={"table_cutoff": "truncate"}, + settings={ + "allow_page_overlap": True, + "auto_partition": True, + "deep_split": True, + "force_url_result": True, + "table_cutoff": "truncate", + }, split_rules="split_rules", ) assert_matches_type(AsyncSplitResponse, split, path=["response"]) diff --git a/tests/e2e/test_sdk_e2e.py b/tests/e2e/test_sdk_e2e.py index 3900cf26..64bc0a37 100644 --- a/tests/e2e/test_sdk_e2e.py +++ b/tests/e2e/test_sdk_e2e.py @@ -14,17 +14,32 @@ import os import time import tempfile +from typing import Any, Dict, List, Union from pathlib import Path import pytest +import reducto from reducto import Reducto -from reducto.types import V3Extract, ParseResponse, AsyncParseResponse, AsyncExtractResponse -from reducto.types.job_get_response import AsyncJobResponse, EnhancedAsyncJobResponse +from reducto.types import ( + URLResult, + V3Extract, + ErrorDetail, + ParseResponse, + SplitResponse, + ExtractResponse, + ClassifyResponse, + JobDeleteResponse, + AsyncParseResponse, + DocumentProperties, + AsyncExtractResponse, +) +from reducto.types.job_get_response import JobGetResponse, AsyncJobResponse, EnhancedAsyncJobResponse DOCUMENT_URL = "https://ci.reducto.ai/onepager.pdf" +MISSING_DOCUMENT_URL = "https://ci.reducto.ai/does-not-exist-e2e.pdf" -TRIVIAL_SCHEMA = { +TRIVIAL_SCHEMA: Dict[str, Any] = { "type": "object", "properties": { "title": { @@ -35,6 +50,14 @@ "required": ["title"], } +CLASSIFICATION_SCHEMA: List[Dict[str, Any]] = [ + {"category": "invoice", "criteria": ["an invoice or bill"]}, + {"category": "report", "criteria": ["a narrative report"]}, +] + +# The API returns ExtractResponse or V3Extract from /extract depending on the account. +SyncExtractResponse = Union[ExtractResponse, V3Extract] + @pytest.fixture(scope="module") def client() -> Reducto: @@ -44,6 +67,70 @@ def client() -> Reducto: return Reducto(api_key=api_key) +@pytest.fixture(scope="module") +def parse_response(client: Reducto) -> ParseResponse: + """One parse call that sends every newer parse option.""" + return client.parse.run( + input=DOCUMENT_URL, + enhance={ + "advanced_chart_agent": True, + "agentic": [{"scope": "table", "mode": "default"}], + }, + settings={ + "embed_pdf_metadata": True, + "embed_pdf_metadata_dpi": 72, + "extract_document_properties": True, + "force_url_result": True, + "tenant_throttling": {"tenant_id": "sdk-e2e", "max_share": 0.5}, + }, + spreadsheet={"max_cell_count": 100_000}, + ) + + +@pytest.fixture(scope="module") +def extract_response(client: Reducto) -> SyncExtractResponse: + """One extract call that sends every newer extract option.""" + response = client.extract.run( + input=DOCUMENT_URL, + instructions={"schema": TRIVIAL_SCHEMA}, + settings={ + "force_url_result": True, + "page_range": {"start": 1, "end": 1}, + "citations": {"enabled": True, "parent_block": "bbox_only"}, + }, + ) + assert isinstance(response, (ExtractResponse, V3Extract)) + return response + + +@pytest.fixture(scope="module") +def split_response(client: Reducto) -> SplitResponse: + return client.split.run( + input=DOCUMENT_URL, + split_description=[ + {"name": "invoice", "description": "An invoice or bill"}, + {"name": "other", "description": "Anything else"}, + ], + settings={ + "allow_page_overlap": True, + "auto_partition": True, + "deep_split": False, + "force_url_result": True, + }, + ) + + +def wait_for_job(client: Reducto, job_id: str, timeout_s: int = 120) -> JobGetResponse: + deadline = time.monotonic() + timeout_s + while time.monotonic() < deadline: + job = client.job.get(job_id) + assert isinstance(job, (AsyncJobResponse, EnhancedAsyncJobResponse)) + if job.status in ("Completed", "Failed"): + return job + time.sleep(2) + pytest.fail(f"Job {job_id} did not finish within {timeout_s}s") + + class TestParse: """Tests for the /parse endpoint (client.parse.run).""" @@ -61,6 +148,26 @@ def test_parse_returns_chunks(self, client: Reducto) -> None: assert result.type == "full" assert len(result.chunks) > 0 # type: ignore[union-attr] + def test_response_type(self, parse_response: ParseResponse) -> None: + assert parse_response.response_type == "parse" + + def test_force_url_result(self, parse_response: ParseResponse) -> None: + assert parse_response.result.type == "url" + assert parse_response.result.url.startswith("https://") + + def test_document_properties(self, parse_response: ParseResponse) -> None: + assert isinstance(parse_response.document_properties, DocumentProperties) + assert parse_response.document_properties.title + + def test_embed_pdf_metadata(self, parse_response: ParseResponse) -> None: + assert parse_response.pdf_url + + def test_usage_page_billing_breakdown(self, parse_response: ParseResponse) -> None: + breakdown = parse_response.usage.page_billing_breakdown + assert breakdown is not None + assert "1" in breakdown + assert "page" in breakdown["1"] + class TestParseAsync: """Tests for the /parse_async endpoint (client.parse.run_job).""" @@ -73,20 +180,15 @@ def test_parse_async_returns_job_id(self, client: Reducto) -> None: def test_parse_async_job_completes(self, client: Reducto) -> None: response = client.parse.run_job(input=DOCUMENT_URL) assert isinstance(response, AsyncParseResponse) - job_id = response.job_id + job = wait_for_job(client, response.job_id) + if job.status == "Failed": + pytest.fail(f"Parse async job failed: {job.reason}") + assert job.result is not None - # Poll until job completes (max 120s) - for _ in range(60): - job = client.job.get(job_id) - assert isinstance(job, (AsyncJobResponse, EnhancedAsyncJobResponse)) - if job.status == "Completed": - assert job.result is not None - return - if job.status == "Failed": - pytest.fail(f"Parse async job failed: {job.reason}") - time.sleep(2) - - pytest.fail("Parse async job did not complete within timeout") + def test_queue_priority_standard(self, client: Reducto) -> None: + response = client.parse.run_job(input=DOCUMENT_URL, queue_priority="standard") + assert isinstance(response, AsyncParseResponse) + assert response.job_id class TestExtract: @@ -97,7 +199,7 @@ def test_extract_sync(self, client: Reducto) -> None: input=DOCUMENT_URL, instructions={"schema": TRIVIAL_SCHEMA}, ) - assert isinstance(response, V3Extract) + assert isinstance(response, (ExtractResponse, V3Extract)) assert response.result is not None def test_extract_returns_result(self, client: Reducto) -> None: @@ -105,10 +207,32 @@ def test_extract_returns_result(self, client: Reducto) -> None: input=DOCUMENT_URL, instructions={"schema": TRIVIAL_SCHEMA}, ) - assert isinstance(response, V3Extract) + assert isinstance(response, (ExtractResponse, V3Extract)) assert isinstance(response.result, list) assert len(response.result) > 0 + def test_response_type_matches_class(self, extract_response: SyncExtractResponse) -> None: + expected = "extract" if isinstance(extract_response, ExtractResponse) else "v3_extract" + assert extract_response.response_type == expected + + def test_force_url_result(self, extract_response: SyncExtractResponse) -> None: + if isinstance(extract_response, ExtractResponse): + assert isinstance(extract_response.result, URLResult) + assert extract_response.result.url.startswith("https://") + else: + assert isinstance(extract_response.result, dict) + assert extract_response.result["type"] == "url" + assert str(extract_response.result["url"]).startswith("https://") + + def test_confidence_fields_present(self, extract_response: SyncExtractResponse) -> None: + if isinstance(extract_response, V3Extract): + assert extract_response.confidence in (None, "high", "low") + assert extract_response.confidence_reason is None or isinstance(extract_response.confidence_reason, str) + else: + assert extract_response.response_confidence is None or isinstance( + extract_response.response_confidence, dict + ) + class TestExtractAsync: """Tests for the /extract_async endpoint (client.extract.run_job).""" @@ -127,24 +251,68 @@ def test_extract_async_job_completes(self, client: Reducto) -> None: instructions={"schema": TRIVIAL_SCHEMA}, ) assert isinstance(response, AsyncExtractResponse) - job_id = response.job_id + job = wait_for_job(client, response.job_id) + if job.status == "Failed": + pytest.fail(f"Extract async job failed: {job.reason}") + assert job.result is not None + + def test_queue_priority_standard(self, client: Reducto) -> None: + response = client.extract.run_job( + input=DOCUMENT_URL, + instructions={"schema": TRIVIAL_SCHEMA}, + queue_priority="standard", + ) + assert isinstance(response, AsyncExtractResponse) + assert response.job_id + + +class TestSplit: + """Tests for the /split endpoint (client.split.run).""" - # Poll until job completes (max 120s) - for _ in range(60): - job = client.job.get(job_id) - assert isinstance(job, (AsyncJobResponse, EnhancedAsyncJobResponse)) - if job.status == "Completed": - assert job.result is not None - return - if job.status == "Failed": - pytest.fail(f"Extract async job failed: {job.reason}") - time.sleep(2) + def test_response_type(self, split_response: SplitResponse) -> None: + assert split_response.response_type == "split" - pytest.fail("Extract async job did not complete within timeout") + def test_job_id_and_duration(self, split_response: SplitResponse) -> None: + assert split_response.job_id + assert split_response.duration is not None and split_response.duration >= 0 + + def test_force_url_result(self, split_response: SplitResponse) -> None: + assert isinstance(split_response.result, URLResult) + assert split_response.result.url.startswith("https://") + + +class TestClassify: + """Tests for the /classify endpoint (client.classify.run).""" + + def test_category_groups_and_model(self, client: Reducto) -> None: + response = client.classify.run( + input=DOCUMENT_URL, + classification_schema=CLASSIFICATION_SCHEMA, + category_groups={"financial": ["invoice"]}, + model="default", + priority=True, + ) + assert isinstance(response, ClassifyResponse) + assert response.response_type == "classify" + assert not isinstance(response.result, URLResult) + assert response.result.category in ("invoice", "report") + assert response.extra_metadata is not None + assert response.extra_metadata["grouping"] in ("financial", "ungrouped") + assert response.usage is not None + assert response.usage.num_categories == len(CLASSIFICATION_SCHEMA) + + def test_force_url_result(self, client: Reducto) -> None: + response = client.classify.run( + input=DOCUMENT_URL, + classification_schema=CLASSIFICATION_SCHEMA, + force_url_result=True, + ) + assert isinstance(response.result, URLResult) + assert response.result.url.startswith("https://") class TestUpload: - """Tests for the /upload endpoint (client.upload).""" + """Tests for the /upload endpoints (client.upload, client.delete_upload).""" def test_upload_returns_file_id(self, client: Reducto) -> None: response = client.upload() @@ -171,9 +339,25 @@ def test_upload_with_file_path(self, client: Reducto) -> None: finally: tmp_path.unlink() + def test_delete_uploaded_file(self, client: Reducto) -> None: + with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as f: + f.write(b"%PDF-1.4 minimal test file") + tmp_path = Path(f.name) + try: + upload = client.upload(file=tmp_path) + finally: + tmp_path.unlink() + assert upload.file_id.startswith("reducto://") + + deleted = client.delete_upload(upload.file_id) + assert deleted.file_id == upload.file_id + + with pytest.raises(reducto.NotFoundError): + client.delete_upload(upload.file_id) + class TestJob: - """Tests for the /job/{job_id} endpoint (client.job.get).""" + """Tests for the /job/{job_id} endpoints (client.job.get, client.job.delete).""" def test_job_get_with_parse_job(self, client: Reducto) -> None: # Create an async parse job first @@ -187,19 +371,36 @@ def test_job_get_with_parse_job(self, client: Reducto) -> None: assert job.status in ("Pending", "Completed", "Idle") def test_job_get_completed(self, client: Reducto) -> None: - # Create an async parse job and wait for completion async_response = client.parse.run_job(input=DOCUMENT_URL) assert isinstance(async_response, AsyncParseResponse) - job_id = async_response.job_id - - for _ in range(60): - job = client.job.get(job_id) - assert isinstance(job, (AsyncJobResponse, EnhancedAsyncJobResponse)) - if job.status == "Completed": - assert job.result is not None - return - if job.status == "Failed": - pytest.fail(f"Job failed: {job.reason}") - time.sleep(2) - - pytest.fail("Job did not complete within timeout") + job = wait_for_job(client, async_response.job_id) + if job.status == "Failed": + pytest.fail(f"Job failed: {job.reason}") + assert job.result is not None + + def test_delete_job(self, client: Reducto) -> None: + job_id = client.parse.run_job(input=DOCUMENT_URL).job_id + assert wait_for_job(client, job_id).status == "Completed" + + deleted = client.job.delete(job_id) + assert isinstance(deleted, JobDeleteResponse) + assert deleted.job_id == job_id + + with pytest.raises(reducto.APIStatusError) as exc_info: + client.job.get(job_id) + assert exc_info.value.status_code in (409, 410) + + def test_delete_job_with_include_persisted(self, client: Reducto) -> None: + job_id = client.parse.run_job(input=DOCUMENT_URL).job_id + wait_for_job(client, job_id) + deleted = client.job.delete(job_id, include_persisted=True) + assert deleted.job_id == job_id + + def test_failed_job_has_error_detail(self, client: Reducto) -> None: + job_id = client.parse.run_job(input=MISSING_DOCUMENT_URL).job_id + job = wait_for_job(client, job_id) + assert job.status == "Failed" + assert isinstance(job.error, ErrorDetail) + assert job.error.name == "INVALID_CONFIG" + assert job.error.code == 400 + assert job.error.message