diff --git a/README.md b/README.md index 1a039313..2e73c55c 100644 --- a/README.md +++ b/README.md @@ -24,16 +24,16 @@ Models and prices reflect each provider's official documentation; each system us | Rank | Provider | Category | Overall | Short | Medium | Long | ¢ / Page | |---:|---|---|---:|---:|---:|---:|---:| -| 1 | LlamaExtract Agentic Plus | LlamaExtract | **96.38** | 97.15 | 94.77 | 94.60 | 7.50¢ | -| 2 | Reducto Deep Extract | Specialized APIs | 96.08 | 96.84 | 94.80 | 92.85 | 6.35¢ | -| 3 | LlamaExtract Agentic | LlamaExtract | 96.03 | 96.81 | 94.21 | **95.05** | 3.12¢ | -| 4 | Pulse (Effort) | Specialized APIs | 95.91 | 96.46 | **95.01** | 93.51 | 10.50¢ | +| 1 | LlamaExtract Agentic Plus | LlamaExtract | **96.38** | 97.15 | 94.77 | 94.60 | 7.50¢ | +| 2 | Reducto Deep Extract | Specialized APIs | 96.08 | 96.84 | **94.80** | 92.85 | 6.35¢ | +| 3 | Pulse (Effort) | Specialized APIs | 96.07 | 96.83 | 94.70 | 93.24 | 10.50¢ | +| 4 | LlamaExtract Agentic | LlamaExtract | 96.03 | 96.81 | 94.21 | **95.05** | 3.12¢ | | 5 | Claude Code (Sonnet 5.5 Evidence) | Coding Agents | 94.68 | 96.22 | 92.97 | 83.69 | 5.69¢ | | 6 | Codex (GPT-6 Sol Evidence) | Coding Agents | 94.57 | 95.99 | 92.42 | 87.25 | 9.80¢ | -| 7 | Codex (GPT-5.6 Sol Evidence) | Coding Agents | 93.77 | 96.04 | 90.19 | 82.73 | 21.65¢ | -| 8 | LlamaExtract Cost-Effective | LlamaExtract | 93.67 | 96.07 | 90.21 | 80.48 | 1.00¢ | -| 9 | Codex (GPT-5.5) | Coding Agents | 93.57 | 95.68 | 91.15 | 78.88 | 27.83¢ | -| 10 | Codex (GPT-5.5 Evidence) | Coding Agents | 93.35 | 95.59 | 88.72 | 87.97 | 33.63¢ | +| 7 | Pulse (Non-effort) | Specialized APIs | 94.51 | 95.39 | 92.49 | 93.31 | 3.00¢ | +| 8 | Codex (GPT-5.6 Sol Evidence) | Coding Agents | 93.77 | 96.04 | 90.19 | 82.73 | 21.65¢ | +| 9 | LlamaExtract Cost-Effective | LlamaExtract | 93.67 | 96.07 | 90.21 | 80.48 | 1.00¢ | +| 10 | Codex (GPT-5.5) | Coding Agents | 93.57 | 95.68 | 91.15 | 78.88 | 27.83¢ | Top 10 of 49 systems — full table in [leaderboard.csv](leaderboard.csv). @@ -47,9 +47,9 @@ Top 10 of 49 systems — full table in [leaderboard.csv](leaderboard.csv). OverallShortMediumLongOverallShortMediumLong - 1LlamaExtract Agentic Plus81.2681.6381.1875.2289.9492.4586.5074.71 - 2LlamaExtract Agentic78.8977.6381.8680.8990.6492.5886.6887.09 - 3Codex (GPT-6 Sol Evidence)77.1177.6175.5378.3186.5087.2085.1284.82 + 1LlamaExtract Agentic Plus81.2681.6381.1875.2289.9492.4586.5074.71 + 2LlamaExtract Agentic78.8977.6381.8680.8990.6492.5886.6887.09 + 3Codex (GPT-6 Sol Evidence)77.1177.6175.5378.3186.5087.2085.1284.82 4Claude Code (Opus 5.5 Evidence)71.5573.5765.5474.8776.9579.2771.3676.47 5Codex (GPT-6 Luna Evidence)65.7062.9171.1174.3683.9384.4183.4780.17 6Claude Code (Sonnet 5.5 Evidence)64.6164.5963.1871.2275.9977.9371.5074.98 diff --git a/leaderboard.csv b/leaderboard.csv index e3e73dc6..96b4f6e0 100644 --- a/leaderboard.csv +++ b/leaderboard.csv @@ -1,12 +1,12 @@ Provider,Category,Overall,Short,Medium,Long,Cost_Per_Page,Cost_Short,Cost_Medium,Cost_Long,P_Short,P_Medium,P_Long,R_Short,R_Medium,R_Long,Word_Grounding,Word_Grounding_Short,Word_Grounding_Medium,Word_Grounding_Long,Page_Grounding,Page_Grounding_Short,Page_Grounding_Medium,Page_Grounding_Long,Latency_S_Per_Doc_Short,Latency_S_Per_Doc_Medium,Latency_S_Per_Doc_Long LlamaExtract Agentic Plus,LlamaExtract,96.38,97.15,94.77,94.60,0.0750,0.0750,0.0750,0.0750,97.17,95.31,94.35,97.18,94.29,94.86,81.26,81.63,81.18,75.22,89.94,92.45,86.50,74.71,167.8,243.9,821.6 -Pulse (Effort),Specialized APIs,95.91,96.46,95.01,93.51,0.1050,0.1050,0.1050,0.1050,96.44,95.28,94.07,96.50,94.89,93.19,1.49,1.17,3.03,0.04,79.70,77.40,86.19,77.74,125.7,206.3,453.1 +Pulse (Effort),Specialized APIs,96.07,96.83,94.70,93.24,0.1050,0.1050,0.1050,0.1050,96.85,94.93,93.71,96.86,94.60,92.96,14.94,12.58,20.80,15.96,83.62,81.32,87.95,90.08,118.0,249.7,507.3 Claude Code (Sonnet 5.5 Evidence),Coding Agents,94.68,96.22,92.97,83.69,0.0569,0.0735,0.0237,0.0091,96.68,94.47,83.36,95.93,91.83,84.05,64.61,64.59,63.18,71.22,75.99,77.93,71.50,74.98,82.5,87.7,187.0 Codex (GPT-6 Sol Evidence),Coding Agents,94.57,95.99,92.42,87.25,0.0980,0.1279,0.0388,0.0122,96.33,93.49,91.28,95.80,91.72,86.70,77.11,77.61,75.53,78.31,86.50,87.20,85.12,84.82,104.2,89.9,101.4 Codex (GPT-5.6 Sol Evidence),Coding Agents,93.77,96.04,90.19,82.73,0.2165,0.2813,0.0870,0.0300,96.28,91.37,86.24,95.84,89.21,82.42,54.66,50.87,64.58,69.01,83.62,85.12,80.77,78.84,95.2,106.6,197.2 Codex (GPT-5.5),Coding Agents,93.57,95.68,91.15,78.88,0.2783,0.3579,0.1214,0.0434,96.01,93.64,94.03,95.51,90.16,78.58,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,70.3,82.6,122.0 Codex (GPT-5.5 Evidence),Coding Agents,93.35,95.59,88.72,87.97,0.3363,0.4354,0.1406,0.0472,95.86,89.85,89.29,95.36,87.86,87.08,52.57,53.13,48.01,61.87,80.64,83.27,73.87,79.90,89.0,117.8,164.6 -Pulse (Non-effort),Specialized APIs,92.70,93.17,92.22,89.07,0.0300,0.0300,0.0300,0.0300,93.36,92.59,90.81,93.17,92.01,87.82,0.71,0.82,0.49,0.01,73.42,73.81,69.86,84.72,87.2,140.0,314.3 +Pulse (Non-effort),Specialized APIs,94.51,95.39,92.49,93.31,0.0300,0.0300,0.0300,0.0300,95.71,93.34,93.91,95.25,91.86,92.86,13.37,11.56,18.03,13.47,81.44,80.87,82.29,84.08,174.7,244.9,499.3 Codex (GPT-5.6 Terra Evidence),Coding Agents,92.26,95.60,86.69,77.36,0.0945,0.1231,0.0376,0.0121,95.99,89.59,92.33,95.38,85.20,76.71,43.68,40.48,53.32,51.17,80.52,83.51,75.18,69.32,69.4,92.0,123.3 OpenAI GPT-6 Astra,Commercial VLM,91.91,97.22,90.56,31.70,0.1109,0.1320,0.0677,0.0382,97.14,92.08,75.58,97.32,90.16,29.87,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,35.0,106.4,90.1 Codex (GPT-5.6 Luna Evidence),Coding Agents,91.16,93.50,87.43,80.05,0.0120,0.0156,0.0048,0.0016,94.06,89.54,85.29,93.25,86.66,79.28,40.03,37.18,47.66,50.89,79.81,82.85,74.07,69.50,110.4,138.1,256.1