diff --git a/.github/PULL_REQUEST_TEMPLATE/benchmark-result.md b/.github/PULL_REQUEST_TEMPLATE/benchmark-result.md new file mode 100644 index 00000000..9624df64 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE/benchmark-result.md @@ -0,0 +1,24 @@ +## Benchmark result + +Benchmark: ExtractBench / ExtractBench Pro +System and configuration: +New entry or update: + +## Reproduction + +- Dataset repository and revision: +- Harness commit/version: +- Model/API version and evaluation date: +- Pipeline implementation, prompts, tools and configuration: +- Concurrency, timeout, retries and any infrastructure recovery: +- Document counts, failures and timeouts: +- Cost methodology, failed-attempt spend and missing usage: +- Evaluation reports, predictions/citations and execution errors (artifact link): + +## Validation + +- [ ] Complete benchmark cohort; failures, missing outputs and timeouts remain in the denominator. +- [ ] Updated the benchmark's CSV using the official scorer. +- [ ] Regenerated README tables with `uv run python scripts/update_readme.py`. +- [ ] `uv run python scripts/update_readme.py --check` passes. +- [ ] `git diff --check` passes. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 17bd6262..5e60f89e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,6 +35,9 @@ jobs: - name: Ruff lint run: uv run ruff check + - name: Check README leaderboards + run: uv run python scripts/update_readme.py --check + - name: Pytest without runners run: uv run pytest tests/ -q diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..ab5fcf52 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,46 @@ +# Contributing + +## Submitting benchmark results + +Open a pull request in [run-llama/ExtractBench](https://github.com/run-llama/ExtractBench/pulls) with scores and enough evidence to reproduce them. Both new systems and updates to existing systems are welcome. + +| Benchmark | Result file | Evaluation set | +|---|---|---| +| ExtractBench | [leaderboard.csv](leaderboard.csv) | All 370 documents | +| ExtractBench Pro | [leaderboard-pro.csv](leaderboard-pro.csv) | All 100 documents across six challenges | + +Use a distinct provider/configuration name when changing the model, harness, or effort setting. For an update to an existing configuration, include the previous and new run dates and describe what changed. + +### Run and score + +Install your provider extra and configure its API key as described in the [README](README.md#configuration). To add a provider, see [Evaluating Your Own System](README.md#usage). + +For original ExtractBench: + +```bash +uv run extract-bench run your_pipeline +``` + +For Pro, use a 30-minute deadline per document and no timeout retries: + +```bash +uv run extract-bench download --dataset pro +uv run extract-bench inference run your_pipeline data-pro --output_dir output-pro --per_file_timeout 1800 --timeout_retries 0 +uv run extract-bench run your_pipeline --dataset pro --skip_inference +uv run extract-bench serve output-pro/your_pipeline +``` + +Replace `your_pipeline` with a registered pipeline name. Inference calls your provider's paid API. The second `run` command evaluates the saved predictions without calling that API again. Execution failures, missing outputs, and documents that exceed the deadline remain in the denominator with zero scores. Do not select the best output from repeated attempts. Disclose any infrastructure recovery and preserve failed-attempt records. + +### Update the leaderboard + +1. Update the appropriate CSV. For Pro, include one `all` row and one row for each of the six challenges; see the [column definitions](docs/pro-leaderboard.md#csv-format). Use the scorer's per-document metrics, not rule pass rate. All scores are percentages and costs are USD per page. Leave unavailable metrics or costs blank. +2. Run `uv run python scripts/update_readme.py` to regenerate the README tables. +3. Run `uv run python scripts/update_readme.py --check` and `git diff --check`. +4. Include the following in your PR: + - Dataset repository and revision from `data/.extract_bench_manifest.json` or `data-pro/.extract_bench_manifest.json`, and the harness commit/version. + - Provider/model version, harness, effort settings, prompts, tools, concurrency, timeout and retry settings, and evaluation date. Include the pipeline implementation/configuration or a reproducible script. + - Evaluation reports (`_evaluation_report.json` for each split), per-document predictions/citations, and execution errors, as attachments or a durable downloadable artifact. Keep API keys and account credentials out of artifacts. + - Document counts, failures/timeouts, and how costs were measured, including failed attempts and any missing usage. Distinguish measured usage from an estimate based on published prices. + +Use the [benchmark result PR template](https://github.com/run-llama/ExtractBench/compare?expand=1&template=benchmark-result.md) when opening your PR. Maintainers review evidence before merging leaderboard updates. The benchmark websites are updated separately; a CSV edit does not automatically publish a website change. diff --git a/README.md b/README.md index 759bdd71..558496dd 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,11 @@ Read the [ExtractBench launch post](https://www.llamaindex.ai/blog/introducing-e ExtractBench: JSON Schema and document in; JSON plus evidence out; scored on value F1, grounding F1, challenge tags, and cost

-## Leaderboard +## Leaderboards + +[ExtractBench](#extractbench-leaderboard) · [ExtractBench Pro](#extractbench-pro-leaderboard) · [Submit results](CONTRIBUTING.md#submitting-benchmark-results) + +### ExtractBench leaderboard Models and prices reflect each provider's official documentation; each system uses its recommended configuration. @@ -75,6 +79,31 @@ Top 10 of 49 systems — full table in [leaderboard.csv](leaderboard.csv). +### ExtractBench Pro leaderboard + +100 documents across six challenges. **Perfect documents** is the percentage with 100% Value F1. Failed documents and documents exceeding the 30-minute limit score zero. Grounding is evaluated on the 81 documents with verified evidence boxes. + + +Scores are percentages; costs are USD per page, displayed in cents. Rows are ordered by perfect-document percentage, then Value F1, then lower cost. The best score in each metric is **bold**; the second-best distinct score is underlined. A dash means unreported, not zero. + +| Rank | Provider | Category | Perfect docs (%) | Value F1 | Box F1 | Page F1 | ¢ / Page | +|---:|---|---|---:|---:|---:|---:|---:| +| 1 | Codex GPT-6 Astra | Coding Agents | **58.00** | **94.53** | **70.97** | **79.10** | 95.20¢ | +| 2 | Claude Code Opus 5.5 | Coding Agents | 51.00 | 74.87 | 45.29 | 56.85 | 41.20¢ | +| 3 | LlamaExtract Agentic Plus | LlamaExtract | 38.00 | 77.70 | 60.39 | 66.01 | 7.50¢ | +| 4 | Extend Operator | Specialized APIs | 34.00 | 73.87 | 57.75 | 68.84 | 9.88¢ | +| 5 | Claude Code Sonnet 5.5 | Coding Agents | 32.00 | 73.62 | 40.47 | 53.00 | 19.98¢ | +| 6 | LlamaExtract Agentic | LlamaExtract | 20.00 | 71.32 | 58.01 | 63.77 | 3.12¢ | +| 7 | Pulse | Specialized APIs | 18.00 | 70.53 | 18.74 | 74.77 | 10.50¢ | +| 8 | Codex GPT-6.1 Sol | Coding Agents | 12.00 | 74.06 | 63.96 | 72.26 | 13.86¢ | +| 9 | Reducto Deep | Specialized APIs | 3.00 | 56.35 | 42.20 | 62.38 | 6.18¢ | +| 10 | Datalab Accurate | Specialized APIs | 2.00 | 43.93 | 0.32 | 4.66 | 15.75¢ | +| 11 | Codex GPT-6 Luna | Coding Agents | 0.00 | 48.75 | 44.27 | 60.78 | 1.14¢ | +| 12 | LlamaExtract Cost Effective | LlamaExtract | 0.00 | 47.19 | 49.40 | 56.46 | 1.00¢ | + + +The [Pro CSV](leaderboard-pro.csv) includes overall scores and all six challenge slices. See the [score definitions and launch snapshot](docs/pro-leaderboard.md), or [submit a new result](CONTRIBUTING.md#submitting-benchmark-results). + ## Quick Start **Prerequisites:** Create a `.env` file with the API key for the extraction system you want to evaluate (see [Configuration](#configuration)). @@ -208,7 +237,7 @@ uv run extract-bench status ## ExtractBench Pro -[ExtractBench Pro](https://huggingface.co/datasets/llamaindex/ExtractBenchPro) contains 100 documents across six challenges: document conventions, house vocabulary, house counting rules, drawn relations, charts inside documents, and long chart sheets. It uses the same evaluator as ExtractBench. Read the [ExtractBench Pro launch post](https://www.llamaindex.ai/blog/introducing-extractbench-pro) for the challenges and results. +[ExtractBench Pro](https://huggingface.co/datasets/llamaindex/ExtractBenchPro) contains 100 documents across six challenges: document conventions, house vocabulary, house counting rules, drawn relations, charts inside documents, and long chart sheets. It uses the same evaluator as ExtractBench. See the [Pro leaderboard](#extractbench-pro-leaderboard) or [submit results](CONTRIBUTING.md#submitting-benchmark-results). Read the [ExtractBench Pro launch post](https://www.llamaindex.ai/blog/introducing-extractbench-pro) for the challenges and results. Download the PDFs, schemas, ground truth, and scoring rules into the layout the harness reads: diff --git a/docs/pro-leaderboard.md b/docs/pro-leaderboard.md new file mode 100644 index 00000000..1cef1f67 --- /dev/null +++ b/docs/pro-leaderboard.md @@ -0,0 +1,47 @@ +# ExtractBench Pro leaderboard + +The [README table](../README.md#extractbench-pro-leaderboard) summarizes [leaderboard-pro.csv](../leaderboard-pro.csv). The CSV contains overall results and six challenge slices for each system. To submit or update scores, see [Contributing](../CONTRIBUTING.md#submitting-benchmark-results). + +## Scoring + +- **Perfect documents:** percentage of documents with 100% Value F1, before rounding the displayed score. Use a tolerance of `1e-9` on the scorer's 0–1 F1 scale. +- **Value F1:** unweighted mean of `extract_unified_value_f1` over documents. +- **Box F1:** unweighted mean of `extract_unified_grounded_f1`, requiring both the correct value and a correct evidence box at IoU 0.5. This is not Grounded AUC F1-IoU. +- **Page F1:** unweighted mean of `extract_unified_page_f1`, requiring the correct value and source page. + +The leaderboard applies a 30-minute limit per document. Failed, missing, and over-limit documents contribute zero rather than disappearing from the denominator. Grounding scores cover only documents with verified evidence boxes; the official cohort has 100 value-scored documents and 81 grounding-scored documents. Overall scores are document-weighted, not the average of the six challenge means. + +| Challenge key | Documents | Grounding documents | +|---|---:|---:| +| `document_conventions` | 22 | 22 | +| `house_vocabulary` | 14 | 14 | +| `house_counting_rules` | 12 | 9 | +| `drawn_relations` | 20 | 19 | +| `charts_inside_documents` | 25 | 10 | +| `long_chart_sheets` | 7 | 7 | +| `all` | 100 | 81 | + +## CSV format + +Each `(Provider, Challenge)` pair has one row. `all` is the overall result; other challenge keys match the download directories. + +| Column | Meaning | +|---|---| +| `Provider` | System name, including model/configuration where relevant | +| `Category` | LlamaExtract, Specialized APIs, or Coding Agents | +| `Challenge` | `all` or one of the six challenge keys above | +| `Documents` | Full value-scoring denominator, including failures | +| `Grounding_Documents` | Denominator for box and page grounding | +| `Perfect_Documents` | Percentage of documents with perfect Value F1 | +| `Value_F1` | Value F1 on a 0–100 scale | +| `Word_Grounding` | Box F1 on a 0–100 scale; name matches original ExtractBench's CSV | +| `Page_Grounding` | Page F1 on a 0–100 scale | +| `Cost_Per_Page` | USD per page for this slice; the README converts to cents | + +Blank grounding metrics and costs mean unreported, not zero. Perfect-document and Value F1 scores are required for the overall ranking. Rows are ordered by perfect-document percentage, then Value F1, then lower cost. Rank emphasis uses each metric independently. + +## Launch snapshot + +The initial twelve systems reproduce the 30-minute-limit results published in the [ExtractBench Pro launch post](https://www.llamaindex.ai/blog/introducing-extractbench-pro) and [benchmark website](https://extractbench.ai/pro/). The launch snapshot was assembled on October 14, 2026 and covers the released 100-document, 2,622-page cohort. Scores and costs retain the published precision; costs are the launch estimates, rather than newly measured API bills. No new inference is required to reproduce these table entries. + +The CSV is the editable source for this repository's table. `scripts/update_readme.py` renders it alongside the original ExtractBench tables; CI checks that generated README sections match both CSVs. diff --git a/leaderboard-pro.csv b/leaderboard-pro.csv new file mode 100644 index 00000000..177390cd --- /dev/null +++ b/leaderboard-pro.csv @@ -0,0 +1,85 @@ +Provider,Category,Challenge,Documents,Grounding_Documents,Perfect_Documents,Value_F1,Word_Grounding,Page_Grounding,Cost_Per_Page +LlamaExtract Agentic Plus,LlamaExtract,all,100,81,38.00,77.70,60.39,66.01,0.075000 +LlamaExtract Agentic Plus,LlamaExtract,document_conventions,22,22,22.73,97.45,73.21,83.57,0.075000 +LlamaExtract Agentic Plus,LlamaExtract,house_vocabulary,14,14,64.29,98.64,61.22,68.61,0.075000 +LlamaExtract Agentic Plus,LlamaExtract,house_counting_rules,12,9,41.67,67.64,33.23,33.23,0.075000 +LlamaExtract Agentic Plus,LlamaExtract,drawn_relations,20,19,10.00,49.27,71.42,75.52,0.075000 +LlamaExtract Agentic Plus,LlamaExtract,charts_inside_documents,25,10,68.00,94.35,76.75,78.00,0.075000 +LlamaExtract Agentic Plus,LlamaExtract,long_chart_sheets,7,7,0.00,12.73,0.00,4.76,0.075000 +LlamaExtract Agentic,LlamaExtract,all,100,81,20.00,71.32,58.01,63.77,0.031200 +LlamaExtract Agentic,LlamaExtract,document_conventions,22,22,13.64,95.54,76.03,87.92,0.031200 +LlamaExtract Agentic,LlamaExtract,house_vocabulary,14,14,35.71,95.39,55.62,65.16,0.031200 +LlamaExtract Agentic,LlamaExtract,house_counting_rules,12,9,16.67,53.95,32.80,32.80,0.031200 +LlamaExtract Agentic,LlamaExtract,drawn_relations,20,19,10.00,44.25,72.81,75.61,0.031200 +LlamaExtract Agentic,LlamaExtract,charts_inside_documents,25,10,32.00,86.14,56.88,58.75,0.031200 +LlamaExtract Agentic,LlamaExtract,long_chart_sheets,7,7,0.00,1.28,0.00,0.00,0.031200 +LlamaExtract Cost Effective,LlamaExtract,all,100,81,0.00,47.19,49.40,56.46,0.010000 +LlamaExtract Cost Effective,LlamaExtract,document_conventions,22,22,0.00,74.97,55.63,63.93,0.010000 +LlamaExtract Cost Effective,LlamaExtract,house_vocabulary,14,14,0.00,80.85,46.80,65.92,0.010000 +LlamaExtract Cost Effective,LlamaExtract,house_counting_rules,12,9,0.00,33.40,25.38,29.07,0.010000 +LlamaExtract Cost Effective,LlamaExtract,drawn_relations,20,19,0.00,29.85,62.41,65.74,0.010000 +LlamaExtract Cost Effective,LlamaExtract,charts_inside_documents,25,10,0.00,37.57,70.82,73.32,0.010000 +LlamaExtract Cost Effective,LlamaExtract,long_chart_sheets,7,7,0.00,0.10,0.00,0.00,0.010000 +Codex GPT-6 Astra,Coding Agents,all,100,81,58.00,94.53,70.97,79.10,0.952000 +Codex GPT-6 Astra,Coding Agents,document_conventions,22,22,27.27,92.58,87.67,88.87,0.170300 +Codex GPT-6 Astra,Coding Agents,house_vocabulary,14,14,64.29,99.51,63.53,63.62,0.141700 +Codex GPT-6 Astra,Coding Agents,house_counting_rules,12,9,58.33,83.07,33.33,33.33,0.175900 +Codex GPT-6 Astra,Coding Agents,drawn_relations,20,19,80.00,98.02,87.07,88.80,0.192700 +Codex GPT-6 Astra,Coding Agents,charts_inside_documents,25,10,64.00,94.72,87.78,87.78,0.150800 +Codex GPT-6 Astra,Coding Agents,long_chart_sheets,7,7,57.14,99.73,14.05,99.52,11.389700 +Claude Code Opus 5.5,Coding Agents,all,100,81,51.00,74.87,45.29,56.85,0.412000 +Claude Code Opus 5.5,Coding Agents,document_conventions,22,22,22.73,58.11,43.44,46.23,0.093700 +Claude Code Opus 5.5,Coding Agents,house_vocabulary,14,14,35.71,55.62,33.25,38.09,0.070000 +Claude Code Opus 5.5,Coding Agents,house_counting_rules,12,9,33.33,56.11,25.93,29.63,0.057100 +Claude Code Opus 5.5,Coding Agents,drawn_relations,20,19,80.00,88.19,61.34,67.55,0.127400 +Claude Code Opus 5.5,Coding Agents,charts_inside_documents,25,10,80.00,97.78,76.18,96.06,0.078000 +Claude Code Opus 5.5,Coding Agents,long_chart_sheets,7,7,14.29,78.28,12.36,77.65,4.665100 +Claude Code Sonnet 5.5,Coding Agents,all,100,81,32.00,73.62,40.47,53.00,0.199800 +Claude Code Sonnet 5.5,Coding Agents,document_conventions,22,22,22.73,58.17,44.74,46.93,0.043300 +Claude Code Sonnet 5.5,Coding Agents,house_vocabulary,14,14,7.14,81.73,33.41,66.55,0.038600 +Claude Code Sonnet 5.5,Coding Agents,house_counting_rules,12,9,33.33,64.57,25.40,29.10,0.026800 +Claude Code Sonnet 5.5,Coding Agents,drawn_relations,20,19,55.00,83.80,46.46,52.45,0.060600 +Claude Code Sonnet 5.5,Coding Agents,charts_inside_documents,25,10,44.00,88.04,64.87,78.41,0.047900 +Claude Code Sonnet 5.5,Coding Agents,long_chart_sheets,7,7,0.00,40.93,9.48,40.93,2.188200 +Codex GPT-6.1 Sol,Coding Agents,all,100,81,12.00,74.06,63.96,72.26,0.138600 +Codex GPT-6.1 Sol,Coding Agents,document_conventions,22,22,9.09,86.90,78.01,81.24,0.025000 +Codex GPT-6.1 Sol,Coding Agents,house_vocabulary,14,14,14.29,86.56,54.18,56.29,0.016300 +Codex GPT-6.1 Sol,Coding Agents,house_counting_rules,12,9,8.33,40.45,32.19,32.19,0.013600 +Codex GPT-6.1 Sol,Coding Agents,drawn_relations,20,19,25.00,75.11,80.28,83.35,0.022000 +Codex GPT-6.1 Sol,Coding Agents,charts_inside_documents,25,10,4.00,65.58,75.67,75.67,0.014500 +Codex GPT-6.1 Sol,Coding Agents,long_chart_sheets,7,7,14.29,93.64,19.17,92.60,1.730400 +Codex GPT-6 Luna,Coding Agents,all,100,81,0.00,48.75,44.27,60.78,0.011400 +Codex GPT-6 Luna,Coding Agents,document_conventions,22,22,0.00,68.05,50.91,63.08,0.003100 +Codex GPT-6 Luna,Coding Agents,house_vocabulary,14,14,0.00,54.72,33.41,55.82,0.002900 +Codex GPT-6 Luna,Coding Agents,house_counting_rules,12,9,0.00,30.37,18.20,30.05,0.003800 +Codex GPT-6 Luna,Coding Agents,drawn_relations,20,19,0.00,35.38,52.90,73.59,0.004100 +Codex GPT-6 Luna,Coding Agents,charts_inside_documents,25,10,0.00,55.05,82.96,90.88,0.005100 +Codex GPT-6 Luna,Coding Agents,long_chart_sheets,7,7,0.00,23.44,0.00,25.18,0.111100 +Extend Operator,Specialized APIs,all,100,81,34.00,73.87,57.75,68.84,0.098800 +Extend Operator,Specialized APIs,document_conventions,22,22,27.27,99.64,80.29,91.42,0.100100 +Extend Operator,Specialized APIs,house_vocabulary,14,14,64.29,97.55,78.24,91.14,0.087500 +Extend Operator,Specialized APIs,house_counting_rules,12,9,16.67,59.55,18.33,27.04,0.087500 +Extend Operator,Specialized APIs,drawn_relations,20,19,0.00,41.24,61.16,76.66,0.124700 +Extend Operator,Specialized APIs,charts_inside_documents,25,10,68.00,91.58,48.89,58.89,0.091900 +Extend Operator,Specialized APIs,long_chart_sheets,7,7,0.00,0.00,0.00,0.00,0.087500 +Pulse,Specialized APIs,all,100,81,18.00,70.53,18.74,74.77,0.105000 +Pulse,Specialized APIs,document_conventions,22,22,9.09,93.83,26.79,79.91,0.105000 +Pulse,Specialized APIs,house_vocabulary,14,14,50.00,82.13,23.30,77.34,0.105000 +Pulse,Specialized APIs,house_counting_rules,12,9,25.00,56.25,14.50,54.07,0.105000 +Pulse,Specialized APIs,drawn_relations,20,19,15.00,65.00,10.31,85.98,0.105000 +Pulse,Specialized APIs,charts_inside_documents,25,10,12.00,65.01,27.64,85.64,0.105000 +Pulse,Specialized APIs,long_chart_sheets,7,7,0.00,34.10,0.00,34.10,0.105000 +Reducto Deep,Specialized APIs,all,100,81,3.00,56.35,42.20,62.38,0.061800 +Reducto Deep,Specialized APIs,document_conventions,22,22,4.55,79.20,42.58,62.96,0.062800 +Reducto Deep,Specialized APIs,house_vocabulary,14,14,0.00,86.49,40.68,79.64,0.056300 +Reducto Deep,Specialized APIs,house_counting_rules,12,9,0.00,38.33,24.89,40.15,0.057600 +Reducto Deep,Specialized APIs,drawn_relations,20,19,5.00,39.00,58.32,67.58,0.060300 +Reducto Deep,Specialized APIs,charts_inside_documents,25,10,4.00,52.58,58.02,75.24,0.064900 +Reducto Deep,Specialized APIs,long_chart_sheets,7,7,0.00,18.16,0.00,22.15,0.070000 +Datalab Accurate,Specialized APIs,all,100,81,2.00,43.93,0.32,4.66,0.157500 +Datalab Accurate,Specialized APIs,document_conventions,22,22,4.55,67.25,0.04,1.10,0.150100 +Datalab Accurate,Specialized APIs,house_vocabulary,14,14,0.00,44.92,0.00,0.01,0.146100 +Datalab Accurate,Specialized APIs,house_counting_rules,12,9,0.00,45.78,0.00,0.00,0.027700 +Datalab Accurate,Specialized APIs,drawn_relations,20,19,0.00,33.48,1.32,18.57,0.038200 +Datalab Accurate,Specialized APIs,charts_inside_documents,25,10,4.00,42.63,0.00,0.00,0.019400 +Datalab Accurate,Specialized APIs,long_chart_sheets,7,7,0.00,0.00,0.00,0.00,1.260000 diff --git a/scripts/update_readme.py b/scripts/update_readme.py index 6ce3662e..7ff652be 100644 --- a/scripts/update_readme.py +++ b/scripts/update_readme.py @@ -1,20 +1,23 @@ -"""Regenerate the leaderboard tables in README.md from leaderboard.csv. +"""Regenerate README leaderboard tables from leaderboard.csv and leaderboard-pro.csv. -Two tables are generated: +Three tables are generated: LEADERBOARD unified value F1 (the headline metric) GROUNDING word-level and page-level grounding F1, side by side + PRO_LEADERBOARD perfect documents, value F1, grounding and cost on Pro Run: uv run python scripts/update_readme.py """ from __future__ import annotations +import argparse import csv from pathlib import Path REPO_ROOT = Path(__file__).resolve().parent.parent CSV_PATH = REPO_ROOT / "leaderboard.csv" +PRO_CSV_PATH = REPO_ROOT / "leaderboard-pro.csv" README_PATH = REPO_ROOT / "README.md" @@ -37,7 +40,7 @@ def column_ranks(rows: list[dict], columns: tuple[str, ...]) -> dict[str, dict[f """Return emphasis for the best and second-best distinct score in each column.""" ranks = {} for column in columns: - values = sorted({float(row[column]) for row in rows}, reverse=True) + values = sorted({float(row[column]) for row in rows if row[column] != ""}, reverse=True) ranks[column] = dict(zip(values[:2], ("best", "second"), strict=False)) return ranks @@ -138,6 +141,39 @@ def grounding_table(rows: list[dict]) -> str: ) +def pro_table(rows: list[dict[str, str]]) -> str: + """Rank the overall Pro slice, retaining challenge rows in the source CSV.""" + overall = [row for row in rows if row["Challenge"] == "all"] + ranked = sorted( + overall, + key=lambda row: ( + -float(row["Perfect_Documents"]), + -float(row["Value_F1"]), + float(row["Cost_Per_Page"]) if row["Cost_Per_Page"] != "" else float("inf"), + row["Provider"], + ), + ) + columns = ("Perfect_Documents", "Value_F1", "Word_Grounding", "Page_Grounding") + ranks = column_ranks(overall, columns) + return table( + "| Rank | Provider | Category | Perfect docs (%) | Value F1 | Box F1 | Page F1 | ¢ / Page |", + "|---:|---|---|---:|---:|---:|---:|---:|", + [ + [ + str(index), + row["Provider"], + row["Category"], + *( + fmt_ranked(row[column], ranks[column].get(float(row[column]))) if row[column] != "" else "—" + for column in columns + ), + fmt_cost(row["Cost_Per_Page"]), + ] + for index, row in enumerate(ranked, 1) + ], + ) + + def replace_block(readme: str, name: str, body: str) -> str: start_marker, end_marker = f"", f"" start, end = readme.find(start_marker), readme.find(end_marker) @@ -148,10 +184,17 @@ def replace_block(readme: str, name: str, body: str) -> str: def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--check", action="store_true", help="Fail if README tables are stale; do not write files") + args = parser.parse_args() with CSV_PATH.open() as f: rows = [r for r in csv.DictReader(f) if r.get("Provider")] - readme = README_PATH.read_text() + with PRO_CSV_PATH.open() as f: + pro_rows = [row for row in csv.DictReader(f) if row.get("Provider")] + + original = README_PATH.read_text() + readme = original readme = replace_block( readme, "LEADERBOARD", @@ -172,8 +215,21 @@ def main() -> None: + "\n\n" + grounding_table(rows), ) - README_PATH.write_text(readme) - print(f"Updated README.md from {CSV_PATH.name} ({len(rows)} systems)") + readme = replace_block( + readme, + "PRO_LEADERBOARD", + "Scores are percentages; costs are USD per page, displayed in cents. Rows are ordered by " + "perfect-document percentage, then Value F1, then lower cost. The best score in each " + "metric is **bold**; the second-best distinct score is underlined. A dash means " + "unreported, not zero.\n\n" + pro_table(pro_rows), + ) + if args.check: + if readme != original: + raise SystemExit("README leaderboard tables are stale. Run: uv run python scripts/update_readme.py") + print("README leaderboard tables are up to date") + else: + README_PATH.write_text(readme) + print(f"Updated README.md from {CSV_PATH.name} and {PRO_CSV_PATH.name}") if __name__ == "__main__": diff --git a/tests/test_readme_leaderboards.py b/tests/test_readme_leaderboards.py new file mode 100644 index 00000000..36f322f7 --- /dev/null +++ b/tests/test_readme_leaderboards.py @@ -0,0 +1,73 @@ +"""The checked-in tables must reflect comparable overall scores and CSV edits.""" + +import importlib.util +from pathlib import Path + +import pytest + +SCRIPT = Path(__file__).resolve().parents[1] / "scripts" / "update_readme.py" +spec = importlib.util.spec_from_file_location("update_readme", SCRIPT) +assert spec is not None and spec.loader is not None +leaderboards = importlib.util.module_from_spec(spec) +spec.loader.exec_module(leaderboards) + + +def pro_row(provider, perfect, value, cost, *, challenge="all", box="", page=""): + return { + "Provider": provider, + "Category": "Coding Agents", + "Challenge": challenge, + "Perfect_Documents": perfect, + "Value_F1": value, + "Word_Grounding": box, + "Page_Grounding": page, + "Cost_Per_Page": cost, + } + + +def test_challenge_scores_cannot_enter_overall_ranking_and_ties_use_value_then_cost(): + rows = [ + pro_row("Unpriced", "50", "80", ""), + pro_row("Priced", "50", "80", "0.02"), + pro_row("Lower value", "50", "79", "0.001"), + pro_row("Fewer perfect", "49", "99", "0.001"), + pro_row("Challenge only", "100", "100", "0.001", challenge="house_vocabulary"), + ] + rendered = leaderboards.pro_table(rows) + assert len({line.count("|") for line in rendered.splitlines()}) == 1 + names = [line.split(" | ")[1] for line in rendered.splitlines()[2:]] + assert names == ["Priced", "Unpriced", "Lower value", "Fewer perfect"] + assert "2.00¢" in rendered + + +def test_unreported_grounding_stays_missing_and_does_not_rank_above_reported_zero(): + rendered = leaderboards.pro_table( + [ + pro_row("Unknown", "0", "70", "", box="", page=""), + pro_row("Zero", "0", "60", "0", box="0", page="0"), + ] + ) + unknown, zero = rendered.splitlines()[2:] + assert "| — | — | — |" in unknown + assert "| **0** | **0** | 0.00¢ |" in zero + + +def test_check_detects_stale_tables_without_rewriting_then_passes_after_generation(tmp_path, monkeypatch): + readme = tmp_path / "README.md" + skeleton = "\n".join( + f"\nstale\n" for name in ("LEADERBOARD", "GROUNDING", "PRO_LEADERBOARD") + ) + readme.write_text(skeleton) + monkeypatch.setattr(leaderboards, "README_PATH", readme) + monkeypatch.setattr("sys.argv", [str(SCRIPT), "--check"]) + with pytest.raises(SystemExit, match="tables are stale"): + leaderboards.main() + assert readme.read_text() == skeleton + + monkeypatch.setattr("sys.argv", [str(SCRIPT)]) + leaderboards.main() + generated = readme.read_text() + assert generated != skeleton + monkeypatch.setattr("sys.argv", [str(SCRIPT), "--check"]) + leaderboards.main() + assert readme.read_text() == generated