diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 17bd6262..e49f9ba3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,6 +9,23 @@ permissions: contents: read jobs: + native-wheels: + uses: ./.github/workflows/native-wheels.yml + + rust: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + with: + components: rustfmt, clippy + - uses: Swatinem/rust-cache@v2 + with: + workspaces: rust + - run: cargo fmt --manifest-path rust/Cargo.toml --all --check + - run: cargo clippy --manifest-path rust/Cargo.toml --all-targets --locked -- -D warnings + - run: cargo test --manifest-path rust/Cargo.toml --locked + lint-and-test: runs-on: ubuntu-latest timeout-minutes: 20 @@ -25,6 +42,11 @@ jobs: enable-cache: true python-version: ${{ matrix.python-version }} + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + with: + workspaces: rust + # Lint and validate runner-free pass/skip behavior before installing heavy deps. - name: Sync dependencies (dev only) run: uv sync --locked --extra dev @@ -49,7 +71,10 @@ jobs: steps: - uses: actions/checkout@v4 - uses: astral-sh/setup-uv@v5 + - uses: dtolnay/rust-toolchain@stable - run: uv build + - name: Build wheel from the source distribution + run: uv build dist/*.tar.gz --wheel --out-dir dist/from-sdist - name: Smoke-test the wheel in a clean environment run: | uv venv /tmp/smoke diff --git a/.github/workflows/native-wheels.yml b/.github/workflows/native-wheels.yml new file mode 100644 index 00000000..d18714ca --- /dev/null +++ b/.github/workflows/native-wheels.yml @@ -0,0 +1,52 @@ +name: Native wheels + +on: + workflow_call: + +permissions: + contents: read + +jobs: + wheels: + name: Wheel (${{ matrix.platform }}) + strategy: + fail-fast: false + matrix: + include: + - platform: linux-x86_64 + os: ubuntu-22.04 + - platform: linux-aarch64 + os: ubuntu-24.04-arm + - platform: macos-arm64 + os: macos-14 + - platform: macos-x86_64 + os: macos-15-intel + - platform: windows-x86_64 + os: windows-2022 + runs-on: ${{ matrix.os }} + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v5 + with: + python-version: "3.12" + - name: Build release wheel + uses: PyO3/maturin-action@v1 + with: + maturin-version: "1.14.1" + command: build + args: --release --locked --out dist + manylinux: auto + - name: Install and exercise the wheel + shell: bash + run: | + uv venv smoke + uv pip install --python smoke dist/*.whl pytest + uv run --no-project --python smoke python -c 'from extract_bench.evaluation.metrics.field_grounding.core import BBox, compute_standard_iou_metrics; assert compute_standard_iou_metrics([BBox(1, (0.0, 0.0, 1.0, 1.0))], [BBox(1, (0.0, 0.0, 1.0, 1.0))]).iou == 1.0' + uv run --no-project --python smoke pytest tests/extract_bench/evaluation/metrics/field_grounding/test_native_geometry.py -q + uv run --no-project --python smoke extract-bench version + uv run --no-project --python smoke extract-bench pipelines > /dev/null + - uses: actions/upload-artifact@v4 + with: + name: native-wheel-${{ matrix.platform }} + path: dist/*.whl diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 68e224dc..d6739a9c 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,7 +1,7 @@ name: Publish to PyPI # Publishes when a version tag (v0.3.0, v1.0.0, ...) is pushed. The tag must -# match `__version__` in src/extract_bench/__init__.py. +# match the workspace package version in rust/Cargo.toml. # # Uses PyPI trusted publishing (OIDC): no API token is stored in GitHub. # One-time setup on pypi.org: project "llama-extract-bench" -> Publishing -> add a @@ -13,6 +13,9 @@ on: tags: ["v*"] jobs: + native-wheels: + uses: ./.github/workflows/native-wheels.yml + build: runs-on: ubuntu-latest steps: @@ -20,20 +23,21 @@ jobs: - uses: astral-sh/setup-uv@v5 - name: Check tag matches package version run: | - pkg_version=$(uv run --no-project python -c "import re,pathlib; print(re.search(r'__version__ = \"([^\"]+)\"', pathlib.Path('src/extract_bench/__init__.py').read_text()).group(1))") + pkg_version=$(uv run --no-project python -c 'import pathlib,tomllib; print(tomllib.loads(pathlib.Path("rust/Cargo.toml").read_text())["workspace"]["package"]["version"])') tag_version="${GITHUB_REF_NAME#v}" if [ "$pkg_version" != "$tag_version" ]; then - echo "Tag $GITHUB_REF_NAME does not match __version__ $pkg_version" >&2 + echo "Tag $GITHUB_REF_NAME does not match package version $pkg_version" >&2 exit 1 fi - - run: uv build + - uses: dtolnay/rust-toolchain@stable + - run: uv build --sdist - uses: actions/upload-artifact@v4 with: name: dist path: dist/ publish: - needs: build + needs: [build, native-wheels] runs-on: ubuntu-latest environment: pypi permissions: @@ -44,6 +48,11 @@ jobs: with: name: dist path: dist/ + - uses: actions/download-artifact@v4 + with: + pattern: native-wheel-* + path: dist/ + merge-multiple: true - uses: pypa/gh-action-pypi-publish@release/v1 - name: Create GitHub release uses: softprops/action-gh-release@v2 diff --git a/README.md b/README.md index 1a039313..80f358d7 100644 --- a/README.md +++ b/README.md @@ -95,6 +95,13 @@ uv run extract-bench run llamaextract_agentic uv run extract-bench serve llamaextract_agentic ``` +Source installs require Rust 1.83 or newer and a C linker. Published wheels +include the native geometry extension. Numerical coverage runs in Rust; +Python retains text comparison, scoring decisions, and metric metadata. +The release version lives in `rust/Cargo.toml`; `extract_bench.__version__` +reads the installed package metadata. + + > [!WARNING] > A full run is **370 documents / 4,869 pages** against a metered API, and costs roughly > **$10 to $1,677** depending on which system you evaluate. Start with `--test`, which runs diff --git a/pyproject.toml b/pyproject.toml index 6a92f74c..4c592d32 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -117,28 +117,25 @@ Changelog = "https://github.com/run-llama/ExtractBench/blob/main/CHANGELOG.md" extract-bench = "extract_bench.cli:main" [build-system] -requires = ["hatchling"] -build-backend = "hatchling.build" - -[tool.hatch.version] -path = "src/extract_bench/__init__.py" - -[tool.hatch.build.targets.wheel] -# ``packages`` already carries analysis/static into the wheel; a force-include -# for the same directory makes hatchling add every asset twice and abort. -packages = ["src/extract_bench"] - -[tool.hatch.build.targets.sdist] -only-include = [ - "src/extract_bench", - "tests", - "README.md", - "LICENSE", - "CHANGELOG.md", -] -exclude = [ - "**/__pycache__", - "**/.gitkeep", +requires = ["maturin==1.14.1"] +build-backend = "maturin" + +[tool.maturin] +manifest-path = "rust/bindings/python/Cargo.toml" +python-source = "src" +module-name = "extract_bench._native" +features = ["pyo3/extension-module"] +profile = "release" +include = [{ path = "tests/**/*", format = "sdist" }] + +[tool.uv] +cache-keys = [ + { file = "pyproject.toml" }, + { file = "src/**/*.py" }, + { file = "src/**/*.pyi" }, + { file = "rust/**/*.toml" }, + { file = "rust/Cargo.lock" }, + { file = "rust/**/*.rs" }, ] [tool.ruff] diff --git a/rust/Cargo.lock b/rust/Cargo.lock new file mode 100644 index 00000000..4d42bad1 --- /dev/null +++ b/rust/Cargo.lock @@ -0,0 +1,137 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "extract-bench-geometry" +version = "1.0.0" + +[[package]] +name = "extract-bench-native" +version = "1.0.0" +dependencies = [ + "extract-bench-geometry", + "pyo3", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "libc" +version = "0.2.190" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78" + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "portable-atomic" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "pyo3" +version = "0.29.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "700d18fa267b73b9b521fd7e13580e2f446916f176cacee1ab63fcc8191f1655" +dependencies = [ + "libc", + "once_cell", + "portable-atomic", + "pyo3-build-config", + "pyo3-ffi", + "pyo3-macros", +] + +[[package]] +name = "pyo3-build-config" +version = "0.29.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b3fc0c4d08f6bb10e71fe39dfb9e2f59c6eb6854e22ec8092f50c69a4499adb" +dependencies = [ + "target-lexicon", +] + +[[package]] +name = "pyo3-ffi" +version = "0.29.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfc0b8e19df29aad7086cf977bb0c2a2f143e30567eb113e9cf72b62ca698330" +dependencies = [ + "libc", + "pyo3-build-config", +] + +[[package]] +name = "pyo3-macros" +version = "0.29.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6100e8a4b5eba53afaa5ed078364851a0b2499c553a44c31026b929049b49dc6" +dependencies = [ + "proc-macro2", + "pyo3-macros-backend", + "quote", + "syn", +] + +[[package]] +name = "pyo3-macros-backend" +version = "0.29.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6143877a16e82b5a727b7127ff4cd86858a24a28d745d72f43e6f227c7b1bdb3" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "target-lexicon" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca" + +[[package]] +name = "unicode-ident" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" diff --git a/rust/Cargo.toml b/rust/Cargo.toml new file mode 100644 index 00000000..96051f93 --- /dev/null +++ b/rust/Cargo.toml @@ -0,0 +1,13 @@ +[workspace] +members = ["crates/geometry", "bindings/python"] +resolver = "2" + +[workspace.package] +version = "1.0.0" +edition = "2021" +rust-version = "1.83" +license = "Apache-2.0" + +[workspace.dependencies] +extract-bench-geometry = { path = "crates/geometry" } +pyo3 = "0.29" diff --git a/rust/README.md b/rust/README.md new file mode 100644 index 00000000..78c2a948 --- /dev/null +++ b/rust/README.md @@ -0,0 +1,17 @@ +The geometry crate contains binding-free rectangle coverage calculations. +The PyO3 crate only marshals coordinate batches and releases the GIL while the +core runs. Python retains coordinate boundary ordering, page/group scopes, +text comparison, match selection, thresholds, and metric metadata. + +Build the optimized mixed Python/Rust package with `uv sync --extra dev` or +`uv build`. Source builds require Rust and a C linker. Run `cargo test +--manifest-path rust/Cargo.toml --locked`, `cargo clippy --manifest-path +rust/Cargo.toml --all-targets --locked -- -D warnings`, and `cargo fmt +--manifest-path rust/Cargo.toml --all --check`. Python tests compare the native +kernel with the frozen Python geometry implementation, including nonfinite +coordinates, degenerate boxes, and candidate thresholds. + +The union algorithm intentionally retains the grid traversal and floating +point accumulation order of the Python scorer. An algorithmic replacement +requires separate parity and performance evidence. Native batches use owned +coordinate arrays; they contain no Python objects or document text. diff --git a/rust/bindings/python/Cargo.toml b/rust/bindings/python/Cargo.toml new file mode 100644 index 00000000..d727c0a5 --- /dev/null +++ b/rust/bindings/python/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "extract-bench-native" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true + +[lib] +name = "_native" +crate-type = ["cdylib"] + +[dependencies] +extract-bench-geometry.workspace = true +pyo3 = { workspace = true, features = ["extension-module", "abi3-py312"] } diff --git a/rust/bindings/python/src/lib.rs b/rust/bindings/python/src/lib.rs new file mode 100644 index 00000000..aad30fa9 --- /dev/null +++ b/rust/bindings/python/src/lib.rs @@ -0,0 +1,31 @@ +//! Convert Python coordinate batches into owned inputs for the geometry core. +use extract_bench_geometry::{union_area, Rect}; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::*; + +#[pyfunction] +fn rect_union_areas( + py: Python<'_>, + rectangles: Vec>, + xs: Vec>, + ys: Vec>, +) -> PyResult> { + if rectangles.len() != xs.len() || rectangles.len() != ys.len() { + return Err(PyValueError::new_err( + "rectangle and boundary batches must have equal lengths", + )); + } + Ok(py.detach(|| { + rectangles + .iter() + .zip(&xs) + .zip(&ys) + .map(|((rects, x), y)| union_area(rects, x, y)) + .collect() + })) +} + +#[pymodule] +fn _native(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(rect_union_areas, module)?) +} diff --git a/rust/crates/geometry/Cargo.toml b/rust/crates/geometry/Cargo.toml new file mode 100644 index 00000000..2c219f88 --- /dev/null +++ b/rust/crates/geometry/Cargo.toml @@ -0,0 +1,6 @@ +[package] +name = "extract-bench-geometry" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true diff --git a/rust/crates/geometry/src/lib.rs b/rust/crates/geometry/src/lib.rs new file mode 100644 index 00000000..124f3cef --- /dev/null +++ b/rust/crates/geometry/src/lib.rs @@ -0,0 +1,39 @@ +//! Numerical rectangle coverage over host-ordered coordinate boundaries. +#![forbid(unsafe_code)] + +pub type Rect = [f64; 4]; + +/// Union area using the caller's sorted, distinct x and y boundaries. +/// +/// The host owns boundary ordering, including its treatment of nonfinite +/// coordinates and signed zero. Cell traversal and accumulation order match +/// the Python scorer so close candidate scores do not acquire new ties. +/// +/// ``` +/// use extract_bench_geometry::union_area; +/// assert_eq!(union_area(&[[0.0, 0.0, 2.0, 1.0]], &[0.0, 2.0], &[0.0, 1.0]), 2.0); +/// ``` +pub fn union_area(rectangles: &[Rect], xs: &[f64], ys: &[f64]) -> f64 { + let mut total = 0.0; + for x in xs.windows(2) { + let (left, right) = (x[0], x[1]); + if right <= left { + continue; + } + for y in ys.windows(2) { + let (top, bottom) = (y[0], y[1]); + if bottom <= top { + continue; + } + if rectangles.iter().any(|rect| { + rect[0] <= left && rect[2] >= right && rect[1] <= top && rect[3] >= bottom + }) { + total += (right - left) * (bottom - top); + } + } + } + total +} + +#[cfg(test)] +mod tests; diff --git a/rust/crates/geometry/src/tests.rs b/rust/crates/geometry/src/tests.rs new file mode 100644 index 00000000..8eb5f41e --- /dev/null +++ b/rust/crates/geometry/src/tests.rs @@ -0,0 +1,58 @@ +use super::union_area; + +#[test] +fn overlapping_and_duplicate_rectangles_count_once() { + let a = [0.0, 0.0, 2.0, 2.0]; + let b = [1.0, 1.0, 3.0, 3.0]; + assert_eq!( + union_area(&[a, a, b], &[0.0, 1.0, 2.0, 3.0], &[0.0, 1.0, 2.0, 3.0]), + 7.0 + ); +} + +#[test] +fn touching_and_disjoint_rectangles() { + let rects = [ + [0.0, 0.0, 1.0, 1.0], + [1.0, 0.0, 2.0, 1.0], + [3.0, 0.0, 4.0, 1.0], + ]; + assert_eq!( + union_area(&rects, &[0.0, 1.0, 2.0, 3.0, 4.0], &[0.0, 1.0]), + 3.0 + ); +} + +#[test] +fn empty_degenerate_and_reversed_rectangles() { + assert_eq!(union_area(&[], &[], &[]), 0.0); + assert_eq!( + union_area(&[[0.0, 0.0, 0.0, 1.0]], &[0.0], &[0.0, 1.0]), + 0.0 + ); + assert_eq!( + union_area(&[[2.0, 2.0, 0.0, 0.0]], &[0.0, 2.0], &[0.0, 2.0]), + 0.0 + ); +} + +#[test] +fn nonfinite_comparisons_and_signed_zero_follow_cell_traversal() { + let nan = f64::NAN; + assert_eq!( + union_area(&[[nan, 0.0, 1.0, 1.0]], &[nan, 1.0], &[0.0, 1.0]), + 0.0 + ); + assert_eq!( + union_area( + &[[0.0, 0.0, f64::INFINITY, 1.0]], + &[0.0, f64::INFINITY], + &[0.0, 1.0] + ), + f64::INFINITY + ); + assert_eq!( + union_area(&[[-0.0, -0.0, 1.0, 1.0]], &[-0.0, 1.0], &[-0.0, 1.0]), + 1.0 + ); +} diff --git a/src/extract_bench/__init__.py b/src/extract_bench/__init__.py index b7f796ab..38f60bb7 100644 --- a/src/extract_bench/__init__.py +++ b/src/extract_bench/__init__.py @@ -1,3 +1,5 @@ """ExtractBench: a benchmark and evaluation harness for schema-guided document extraction.""" -__version__ = "1.0.0" +from importlib.metadata import version + +__version__ = version("llama-extract-bench") diff --git a/src/extract_bench/_native.pyi b/src/extract_bench/_native.pyi new file mode 100644 index 00000000..aac7ad88 --- /dev/null +++ b/src/extract_bench/_native.pyi @@ -0,0 +1,7 @@ +from collections.abc import Sequence + +def rect_union_areas( + rectangles: Sequence[Sequence[Sequence[float]]], + xs: Sequence[Sequence[float]], + ys: Sequence[Sequence[float]], +) -> list[float]: ... diff --git a/src/extract_bench/evaluation/metrics/field_grounding/core.py b/src/extract_bench/evaluation/metrics/field_grounding/core.py index 55fe1d62..9a5a0cb9 100644 --- a/src/extract_bench/evaluation/metrics/field_grounding/core.py +++ b/src/extract_bench/evaluation/metrics/field_grounding/core.py @@ -5,6 +5,7 @@ import math import re import unicodedata +from collections.abc import Sequence from dataclasses import dataclass from datetime import date, datetime from typing import Any, cast @@ -12,6 +13,8 @@ from dateutil import parser as date_parser from rapidfuzz.distance import JaroWinkler +from extract_bench._native import rect_union_areas + STRING_MATCH_THRESHOLD = 0.90 NUMERIC_ABSOLUTE_TOLERANCE = 1e-6 NUMERIC_RELATIVE_TOLERANCE = 1e-6 @@ -223,15 +226,17 @@ def compute_standard_iou_metrics(gt_boxes: list[BBox], pred_boxes: list[BBox]) - gt_rects = [_xywh_to_xyxy(box.bbox) for box in scope_gt] pred_rects = [_xywh_to_xyxy(box.bbox) for box in scope_pred] - gt_area += _rect_union_area(gt_rects) - pred_area += _rect_union_area(pred_rects) - intersections: list[tuple[float, float, float, float]] = [] for gt_rect in gt_rects: for pred_rect in pred_rects: if (intersection := _intersect_xyxy(gt_rect, pred_rect)) is not None: intersections.append(intersection) - intersection_area += _rect_union_area(intersections) + # Batch the three unions within one scope. Keeping the batch local + # bounds temporary memory by one page/group rather than the document. + areas = _rect_union_areas((gt_rects, pred_rects, intersections)) + gt_area += areas[0] + pred_area += areas[1] + intersection_area += areas[2] union_area = gt_area + pred_area - intersection_area iou = intersection_area / union_area if union_area > 0.0 else 0.0 @@ -431,21 +436,15 @@ def _area_xyxy(bbox: tuple[float, float, float, float]) -> float: return max(0.0, bbox[2] - bbox[0]) * max(0.0, bbox[3] - bbox[1]) -def _rect_union_area(rectangles: list[tuple[float, float, float, float]]) -> float: - if not rectangles: - return 0.0 +def _rect_union_areas( + batches: Sequence[Sequence[tuple[float, float, float, float]]], +) -> list[float]: + # Python owns set/sort semantics for NaNs and signed zero. Rust traverses + # these exact boundaries in the same order as the reference scorer. + xs = [sorted({coord for rect in rectangles for coord in (rect[0], rect[2])}) for rectangles in batches] + ys = [sorted({coord for rect in rectangles for coord in (rect[1], rect[3])}) for rectangles in batches] + return rect_union_areas(batches, xs, ys) - xs = sorted({coord for rect in rectangles for coord in (rect[0], rect[2])}) - ys = sorted({coord for rect in rectangles for coord in (rect[1], rect[3])}) - total = 0.0 - for left, right in zip(xs, xs[1:], strict=False): - if right <= left: - continue - for top, bottom in zip(ys, ys[1:], strict=False): - if bottom <= top: - continue - if any( - rect[0] <= left and rect[2] >= right and rect[1] <= top and rect[3] >= bottom for rect in rectangles - ): - total += (right - left) * (bottom - top) - return total + +def _rect_union_area(rectangles: Sequence[tuple[float, float, float, float]]) -> float: + return _rect_union_areas((rectangles,))[0] diff --git a/tests/extract_bench/evaluation/metrics/field_grounding/_python_geometry_reference.py b/tests/extract_bench/evaluation/metrics/field_grounding/_python_geometry_reference.py new file mode 100644 index 00000000..c0157d99 --- /dev/null +++ b/tests/extract_bench/evaluation/metrics/field_grounding/_python_geometry_reference.py @@ -0,0 +1,71 @@ +"""Frozen geometry oracle from ExtractBench c972087, independent of native output.""" + +from extract_bench.evaluation.metrics.field_grounding.core import ( + BBox, + StandardIoUMetrics, + _intersect_xyxy, + _valid_xywh, + _xywh_to_xyxy, +) + + +def compute_standard_iou_metrics(gt_boxes: list[BBox], pred_boxes: list[BBox]) -> StandardIoUMetrics: + """Compute standard IoU between GT and predicted bbox sets. + + Rectangles are scoped by page and group. Within each scope, GT boxes and + predicted boxes are independently unioned before intersection/union area + are accumulated. This differs from :func:`compute_bbox_metrics`, whose + historic ``iou`` field is GT-coverage shaped. + """ + valid_gt = [box for box in gt_boxes if _valid_xywh(box.bbox)] + valid_pred = [box for box in pred_boxes if _valid_xywh(box.bbox)] + scopes = {(box.page, box.group) for box in valid_gt} | {(box.page, box.group) for box in valid_pred} + + gt_area = 0.0 + pred_area = 0.0 + intersection_area = 0.0 + for page, group in scopes: + scope_gt = [box for box in valid_gt if box.page == page and box.group == group] + scope_pred = [box for box in valid_pred if box.page == page and box.group == group] + gt_rects = [_xywh_to_xyxy(box.bbox) for box in scope_gt] + pred_rects = [_xywh_to_xyxy(box.bbox) for box in scope_pred] + + gt_area += _rect_union_area(gt_rects) + pred_area += _rect_union_area(pred_rects) + + intersections: list[tuple[float, float, float, float]] = [] + for gt_rect in gt_rects: + for pred_rect in pred_rects: + if (intersection := _intersect_xyxy(gt_rect, pred_rect)) is not None: + intersections.append(intersection) + intersection_area += _rect_union_area(intersections) + + union_area = gt_area + pred_area - intersection_area + iou = intersection_area / union_area if union_area > 0.0 else 0.0 + return StandardIoUMetrics( + iou=iou, + gt_area=gt_area, + pred_area=pred_area, + intersection_area=intersection_area, + union_area=union_area, + ) + + +def _rect_union_area(rectangles: list[tuple[float, float, float, float]]) -> float: + if not rectangles: + return 0.0 + + xs = sorted({coord for rect in rectangles for coord in (rect[0], rect[2])}) + ys = sorted({coord for rect in rectangles for coord in (rect[1], rect[3])}) + total = 0.0 + for left, right in zip(xs, xs[1:], strict=False): + if right <= left: + continue + for top, bottom in zip(ys, ys[1:], strict=False): + if bottom <= top: + continue + if any( + rect[0] <= left and rect[2] >= right and rect[1] <= top and rect[3] >= bottom for rect in rectangles + ): + total += (right - left) * (bottom - top) + return total diff --git a/tests/extract_bench/evaluation/metrics/field_grounding/test_native_geometry.py b/tests/extract_bench/evaluation/metrics/field_grounding/test_native_geometry.py new file mode 100644 index 00000000..c256459f --- /dev/null +++ b/tests/extract_bench/evaluation/metrics/field_grounding/test_native_geometry.py @@ -0,0 +1,109 @@ +from __future__ import annotations + +import math +import random +import struct +from collections.abc import Sequence + +import pytest +from _python_geometry_reference import _rect_union_area as reference_union +from _python_geometry_reference import compute_standard_iou_metrics as reference_iou + +from extract_bench._native import rect_union_areas +from extract_bench.evaluation.metrics.field_grounding.core import ( + BBox, + _rect_union_area, + _rect_union_areas, + compute_standard_iou_metrics, + field_grounding_localization_passes, +) + + +def assert_same_float(actual: float, expected: float) -> None: + if math.isnan(expected): + assert math.isnan(actual) + else: + assert struct.pack("!d", actual) == struct.pack("!d", expected) + + +@pytest.mark.parametrize( + "rectangles", + [ + [], + [(0.0, 0.0, 1.0, 1.0)], + [(0.0, 0.0, 2.0, 2.0)] * 3 + [(1.0, 1.0, 3.0, 3.0)], + [(0.0, 0.0, 1.0, 1.0), (1.0, 0.0, 2.0, 1.0)], + [(2.0, 2.0, 0.0, 0.0), (0.0, 0.0, 0.0, 1.0)], + [(-0.0, -0.0, 1.0, 1.0), (0.0, 0.0, 1.0, 1.0)], + [(float("nan"), 0.0, 1.0, 1.0), (0.0, 0.0, 1.0, 1.0)], + [(0.0, 0.0, float("inf"), 1.0)], + [(float("-inf"), 0.0, 0.0, 1.0), (0.0, 0.0, float("inf"), 1.0)], + [(0.0, 0.0, 1e-160, 1e-160)], + [(0.0, 0.0, 1e200, 1e200)], + ], +) +def test_union_matches_frozen_python(rectangles: Sequence[tuple[float, float, float, float]]) -> None: + assert_same_float(_rect_union_area(rectangles), reference_union(list(rectangles))) + + +def test_seeded_batches_match_python_including_order_and_duplicates() -> None: + rng = random.Random(20261004) + batches = [] + for _ in range(300): + rectangles = [] + for _ in range(rng.randrange(21)): + x, y, w, h = [rng.uniform(-1.0, 1.0) for _ in range(4)] + rectangles.append((x, y, x + w, y + h)) + if len(rectangles) > 0: + rectangles.append(rectangles[0]) + batches.append(rectangles) + expected = [reference_union(rects) for rects in batches] + for actual, reference in zip(_rect_union_areas(batches), expected, strict=True): + assert_same_float(actual, reference) + assert _rect_union_areas([]) == [] + + +@pytest.mark.parametrize("lengths", [(1, 0, 1), (1, 1, 0), (0, 1, 1)]) +def test_binding_rejects_misaligned_batches(lengths: tuple[int, int, int]) -> None: + with pytest.raises(ValueError, match="equal lengths"): + rect_union_areas([[]] * lengths[0], [[]] * lengths[1], [[]] * lengths[2]) + + +def test_page_and_group_scopes_match_frozen_python() -> None: + rng = random.Random(28075) + for _ in range(200): + collections = [] + for _ in range(2): + boxes = [] + for _ in range(rng.randrange(16)): + x, y, w, h = (rng.uniform(-0.5, 1.0) for _ in range(4)) + boxes.append( + BBox( + page=rng.randrange(1, 4), + group=rng.choice([None, "a", "b"]), + bbox=(x, y, w, h), + ) + ) + collections.append(boxes) + actual = compute_standard_iou_metrics(*collections) + reference = reference_iou(*collections) + for name in ("iou", "gt_area", "pred_area", "intersection_area", "union_area"): + assert_same_float(getattr(actual, name), getattr(reference, name)) + + +@pytest.mark.parametrize("coordinate", [float("nan"), float("inf"), float("-inf"), -0.0]) +def test_iou_nonfinite_coordinates_and_invalid_dimensions(coordinate: float) -> None: + gt = [BBox(1, (coordinate, 0.0, 1.0, 1.0)), BBox(2, (0.0, 0.0, -1.0, 1.0))] + pred = [BBox(1, (0.0, 0.0, 1.0, 1.0)), BBox(1, (0.0, 0.0, float("nan"), 1.0))] + actual, reference = compute_standard_iou_metrics(gt, pred), reference_iou(gt, pred) + for name in ("iou", "gt_area", "pred_area", "intersection_area", "union_area"): + assert_same_float(getattr(actual, name), getattr(reference, name)) + + +@pytest.mark.parametrize("overlap", [math.nextafter(0.5, 0.0), 0.5, math.nextafter(0.5, 1.0)]) +def test_localization_threshold_uses_identical_iou(overlap: float) -> None: + gt = [BBox(1, (0.0, 0.0, 1.0, 1.0))] + pred = [BBox(1, (0.0, 0.0, overlap, 1.0))] + actual, reference = compute_standard_iou_metrics(gt, pred), reference_iou(gt, pred) + assert_same_float(actual.iou, reference.iou) + assert field_grounding_localization_passes(iou=actual.iou, max_ioa=0.0, comparison=None) == (reference.iou >= 0.5) diff --git a/tests/extract_bench/test_extensions.py b/tests/extract_bench/test_extensions.py index a51332f8..5a8322cf 100644 --- a/tests/extract_bench/test_extensions.py +++ b/tests/extract_bench/test_extensions.py @@ -3,7 +3,9 @@ from __future__ import annotations import re +import tomllib from datetime import datetime +from importlib.metadata import version from pathlib import Path from typing import Any, Literal @@ -36,7 +38,9 @@ def test_version_is_the_single_source_of_truth() -> None: assert BenchCLI().version() == extract_bench.__version__ pyproject = (Path(__file__).resolve().parents[2] / "pyproject.toml").read_text() assert 'dynamic = ["version"]' in pyproject - assert 'path = "src/extract_bench/__init__.py"' in pyproject + assert extract_bench.__version__ == version("llama-extract-bench") + manifest = Path(__file__).resolve().parents[2] / "rust/Cargo.toml" + assert tomllib.loads(manifest.read_text())["workspace"]["package"]["version"] == extract_bench.__version__ def test_package_is_typed() -> None: