Pricing: charge routers for the LLM calls they make themselves

README.md

@@ -218,6 +218,10 @@ To get your router on the leaderboard, you can open a Pull Request with your rou
    - `router_inference/config/<router_name>.json` - Your router configuration
    - `router_inference/predictions/<router_name>.json` - Your prediction file with `generated_result` fields populated
    - `router_inference/predictions/<router_name>-robustness.json` - Your prediction file for robustness evaluation, no `generated_result` fields needed
+
+   > [!NOTE]
+   > **Routers that call LLMs themselves.** If your router calls an LLM to make its decision (a routing model, a classifier, voters, a critic, or the earlier steps of a cascade), record each call in the row's `router_usage` list as `{"model": ..., "input_tokens": ..., "output_tokens": ...}`, with an optional `purpose`. Count reasoning tokens in `output_tokens`. The evaluator adds these calls to the row's cost at each model's price. Check your total with `uv run python scripts/pricing/compute_cost.py <router_name>`.
+
 2. **Open a Pull Request to `main` branch and call `/evaluate` in the PR comment**
    - When the PR is ready for evaluation, call `/evaluate` in the PR comment to trigger the evaluation workflow. See an example [here](https://github.com/RouteWorks/RouterArena/pull/71#issuecomment-3904936480).
    - The automated workflow will:

global_utils/router_usage.py

@@ -0,0 +1,88 @@
+# SPDX-FileCopyrightText: Copyright contributors to the RouterArena project
+# SPDX-License-Identifier: Apache-2.0
+
+"""Router calls: LLM calls a router makes itself, besides the call that produces the answer.
+
+A router that uses an LLM to decide (a routing model, a classifier, voters, a critic, the
+earlier steps of a cascade) records each such call in its prediction row:
+
+    "router_usage": [
+        {"model": "openai/gpt-4o-mini", "input_tokens": 812, "output_tokens": 37,
+         "purpose": "routing"}
+    ]
+
+``output_tokens`` counts every generated token, reasoning included, as the provider bills
+it. ``purpose`` is optional free text. Each call is priced like an answer: the model's
+price profile in ``model_cost/model_profiles.yaml``, or the table's highest price if the
+model has none. The row's cost is the answer's cost plus the cost of its router calls.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Callable, Dict, List
+
+__all__ = [
+    "ROUTER_USAGE_FIELD",
+    "validate_router_usage",
+    "router_usage_models",
+    "router_usage_cost",
+]
+
+ROUTER_USAGE_FIELD = "router_usage"
+
+# (model name, token usage) -> USD; ModelEvaluator.calculate_inference_cost fits.
+PriceFn = Callable[[str, Dict[str, int]], float]
+
+
+def validate_router_usage(value: Any) -> List[str]:
+    """Return a list of problems with a row's ``router_usage`` (empty if it is valid)."""
+    if not isinstance(value, list):
+        return [f"{ROUTER_USAGE_FIELD} must be a list of calls"]
+    errors: List[str] = []
+    for i, call in enumerate(value):
+        where = f"{ROUTER_USAGE_FIELD}[{i}]"
+        if not isinstance(call, dict):
+            errors.append(f"{where} must be an object")
+            continue
+        if not isinstance(call.get("model"), str) or not call["model"].strip():
+            errors.append(f"{where}.model must be a non-empty string")
+        for field in ("input_tokens", "output_tokens"):
+            count = call.get(field)
+            if not isinstance(count, int) or isinstance(count, bool) or count < 0:
+                errors.append(f"{where}.{field} must be an integer >= 0")
+        if "purpose" in call and not isinstance(call["purpose"], str):
+            errors.append(f"{where}.purpose must be a string")
+    return errors
+
+
+def router_usage_models(value: Any) -> List[str]:
+    """Model names used by a row's router calls (for price checks)."""
+    if not isinstance(value, list):
+        return []
+    return [
+        c["model"]
+        for c in value
+        if isinstance(c, dict) and isinstance(c.get("model"), str)
+    ]
+
+
+def router_usage_cost(value: Any, price: PriceFn) -> float:
+    """Total cost of a row's router calls. A missing field costs nothing.
+
+    Raises ValueError for a malformed field, so a bad record is never priced as free."""
+    if value is None:
+        return 0.0
+    errors = validate_router_usage(value)
+    if errors:
+        raise ValueError("; ".join(errors))
+    return sum(
+        price(
+            call["model"],
+            {
+                "input_tokens": call["input_tokens"],
+                "output_tokens": call["output_tokens"],
+                "total_tokens": call["input_tokens"] + call["output_tokens"],
+            },
+        )
+        for call in value
+    )

llm_evaluation/run.py

@@ -27,6 +27,7 @@
 from typing import Dict, Any, List, Optional, Tuple, Set
 from universal_model_names import ModelNameManager
 from global_utils.robustness import compute_robustness_score
+from global_utils.router_usage import ROUTER_USAGE_FIELD, router_usage_cost
 from parallel_evaluation import ParallelEvaluationManager
 
 # Add parent directory to path for imports
@@ -315,6 +316,38 @@ def get_livecodebench_ground_truth(global_index: str) -> Optional[Dict[str, Any]
         return None
 
 
+def compute_row_cost(
+    prediction: Dict[str, Any],
+    evaluator: ModelEvaluator,
+    universal_model_name: str,
+) -> Tuple[float, float]:
+    """Return (answer cost, router-call cost) in USD for one prediction row.
+
+    The answer is billed at the model the provider actually served
+    (generated_result.model_used) when it has a known price, so retired/redirected
+    slugs are billed at the model that truly answered rather than the requested
+    alias. For example the retired `grok-4-1-fast-reasoning` slug is redirected by
+    xAI to grok-4.3, which is both stronger and ~5-6x more expensive; billing it at
+    the alias's old price understates cost. Otherwise it falls back to the router's
+    selected model (universal name, to respect universal_model_names.py mappings).
+    See #166.
+
+    Router calls (the row's `router_usage`, see global_utils/router_usage.py) are
+    billed at their own models' prices.
+    """
+    generated_result = prediction.get("generated_result") or {}
+    token_usage = generated_result.get("token_usage", {})
+    cost_model_name = universal_model_name
+    actual_model = generated_result.get("model_used")
+    if actual_model and evaluator.has_price(actual_model):
+        cost_model_name = actual_model
+    answer_cost = evaluator.calculate_inference_cost(cost_model_name, token_usage)
+    router_cost = router_usage_cost(
+        prediction.get(ROUTER_USAGE_FIELD), evaluator.calculate_inference_cost
+    )
+    return answer_cost, router_cost
+
+
 def evaluate_single_prediction(
     prediction: Dict[str, Any],
     ground_truth_map: Dict[str, Dict[str, Any]],
@@ -429,28 +462,15 @@ def evaluate_single_prediction(
                 generated_answer, ground_truth, scorer_func, dataset_name
             )
 
-        # Calculate inference cost.
-        # Prefer the model the provider actually served (generated_result.model_used)
-        # when it has a known price, so retired/redirected slugs are billed at the
-        # model that truly answered rather than the requested alias. For example the
-        # retired `grok-4-1-fast-reasoning` slug is redirected by xAI to grok-4.3,
-        # which is both stronger and ~5-6x more expensive; billing it at the alias's
-        # old price understates cost. Fall back to the router's selected model
-        # (universal name, to respect universal_model_names.py mappings) when the
-        # provider did not report an actual model or its price is unknown. See #166.
-        token_usage = generated_result.get("token_usage", {})
-        cost_model_name = universal_model_name
-        actual_model = generated_result.get("model_used")
-        if actual_model and evaluator.has_price(actual_model):
-            cost_model_name = actual_model
-        inference_cost = evaluator.calculate_inference_cost(
-            cost_model_name,
-            token_usage,
+        answer_cost, router_cost = compute_row_cost(
+            prediction, evaluator, universal_model_name
         )
 
         # Update the prediction with evaluation results
         prediction["accuracy"] = score
-        prediction["cost"] = inference_cost
+        prediction["cost"] = answer_cost + router_cost
+        if router_cost:
+            prediction["router_cost"] = router_cost
 
         return True
 

router_inference/check_config_prediction_files.py

@@ -28,6 +28,11 @@
 sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "../")))
 
 from universal_model_names import ModelNameManager
+from global_utils.router_usage import (
+    ROUTER_USAGE_FIELD,
+    router_usage_models,
+    validate_router_usage,
+)
 
 # Expected dataset sizes (robustness derived from dataset length)
 EXPECTED_SIZES = {
@@ -170,6 +175,7 @@ def check_model_costs(
         model_prediction = prediction.get("prediction")
         if model_prediction:
             used_models.add(model_prediction)
+        used_models.update(router_usage_models(prediction.get(ROUTER_USAGE_FIELD)))
 
     # Also include models from config
     config_models = config.get("pipeline_params", {}).get("models", [])
@@ -388,6 +394,13 @@ def check_prediction_fields(
             )
             continue
 
+        # Router calls the router made itself (optional; see global_utils/router_usage.py)
+        if ROUTER_USAGE_FIELD in prediction:
+            for problem in validate_router_usage(prediction[ROUTER_USAGE_FIELD]):
+                errors.append(
+                    f"Entry {i} (global_index: {pred_global_index}): {problem}"
+                )
+
         # Check prompt - try both "prompt" and "prompt_formatted" fields
         pred_prompt = prediction.get("prompt") or prediction.get("prompt_formatted")
         dataset_entry = dataset_map[pred_global_index]

scripts/pricing/compute_cost.py

@@ -0,0 +1,81 @@
+# SPDX-FileCopyrightText: Copyright contributors to the RouterArena project
+# SPDX-License-Identifier: Apache-2.0
+
+"""Compute a prediction file's cost the way the evaluator does, router calls included.
+
+Every scored row is billed for its answer (token_usage at the answering model's price) plus
+its router calls (``router_usage``, see global_utils/router_usage.py), using the price table
+in model_cost/. Models without a price profile are charged the table's highest price and
+listed at the end. No answer is graded, so this runs offline in seconds.
+
+Usage:
+    uv run python scripts/pricing/compute_cost.py your-router [--split full] [--per-row out.csv]
+"""
+
+from __future__ import annotations
+
+import argparse
+import csv
+import os
+import sys
+from pathlib import Path
+from typing import List, Optional
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+sys.path.insert(0, str(REPO_ROOT))
+sys.path.insert(0, str(REPO_ROOT / "llm_evaluation"))
+
+from evaluate_models import ModelEvaluator  # noqa: E402
+from run import compute_row_cost, load_predictions_file  # noqa: E402
+from universal_model_names import ModelNameManager  # noqa: E402
+
+
+def main(argv: Optional[List[str]] = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    parser.add_argument("router_name")
+    parser.add_argument("--split", default="full", choices=["sub_10", "full"])
+    parser.add_argument("--per-row", help="write one CSV line per scored row")
+    args = parser.parse_args(argv)
+
+    os.chdir(REPO_ROOT)
+    evaluator = ModelEvaluator(cached_results_dir="./cached_results")
+    rows = [
+        p
+        for p in load_predictions_file(args.router_name, split=args.split)
+        if not p.get("for_optimality")
+    ]
+
+    answer_total = router_total = 0.0
+    per_row = []
+    for prediction in rows:
+        model = prediction.get("prediction", "")
+        try:
+            universal = ModelNameManager.get_universal_name(model)
+        except Exception:
+            universal = model
+        answer, router = compute_row_cost(prediction, evaluator, universal)
+        answer_total += answer
+        router_total += router
+        per_row.append((prediction.get("global index"), model, answer, router))
+
+    total = answer_total + router_total
+    n = max(1, len(rows))
+    print(f"Router: {args.router_name} ({len(rows)} scored rows, split {args.split})")
+    print(f"  answers:      ${answer_total:.6f}")
+    print(f"  router calls: ${router_total:.6f}")
+    print(f"  total:        ${total:.6f}  (${1000 * total / n:.4f} per 1K queries)")
+    if evaluator.unpriced_models:
+        print("Charged the maximum price (no price profile):")
+        for name, count in sorted(evaluator.unpriced_models.items()):
+            print(f"  - {name}: {count} call(s)")
+
+    if args.per_row:
+        with open(args.per_row, "w", newline="", encoding="utf-8") as f:
+            writer = csv.writer(f)
+            writer.writerow(["global_index", "model", "answer_cost", "router_cost"])
+            writer.writerows(per_row)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

tests/test_router_usage.py

@@ -0,0 +1,93 @@
+# SPDX-FileCopyrightText: Copyright contributors to the RouterArena project
+# SPDX-License-Identifier: Apache-2.0
+
+"""Tests for router-call costs (global_utils/router_usage.py and the evaluator)."""
+
+import sys
+from pathlib import Path
+
+import pytest
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(REPO_ROOT))
+sys.path.insert(0, str(REPO_ROOT / "llm_evaluation"))
+
+from global_utils.router_usage import (  # noqa: E402
+    router_usage_cost,
+    router_usage_models,
+    validate_router_usage,
+)
+from evaluate_models import ModelEvaluator  # noqa: E402
+from run import compute_row_cost  # noqa: E402
+
+CALL = {"model": "gpt-5-nano", "input_tokens": 1000, "output_tokens": 500}
+
+
+def fake_price(model, usage):
+    return usage["input_tokens"] * 1e-6 + usage["output_tokens"] * 2e-6
+
+
+@pytest.fixture(scope="module")
+def evaluator():
+    return ModelEvaluator(cached_results_dir=str(REPO_ROOT / "cached_results"))
+
+
+def test_valid_record_has_no_errors():
+    assert validate_router_usage([CALL, {**CALL, "purpose": "routing"}]) == []
+    assert validate_router_usage([]) == []
+
+
+@pytest.mark.parametrize(
+    "value, problem",
+    [
+        ({"model": "x"}, "must be a list"),
+        (["x"], "must be an object"),
+        ([{**CALL, "model": ""}], "model must be a non-empty string"),
+        ([{**CALL, "input_tokens": -1}], "input_tokens must be an integer"),
+        ([{**CALL, "output_tokens": 1.5}], "output_tokens must be an integer"),
+        ([{**CALL, "output_tokens": True}], "output_tokens must be an integer"),
+        ([{**CALL, "purpose": 3}], "purpose must be a string"),
+    ],
+)
+def test_invalid_records_are_reported(value, problem):
+    assert any(problem in e for e in validate_router_usage(value))
+
+
+def test_cost_sums_every_call_and_missing_field_is_free():
+    assert router_usage_cost(None, fake_price) == 0.0
+    assert router_usage_cost([CALL, CALL], fake_price) == pytest.approx(2 * 0.002)
+    assert router_usage_models([CALL, "x"]) == ["gpt-5-nano"]
+
+
+def test_malformed_record_is_never_priced_as_free():
+    with pytest.raises(ValueError):
+        router_usage_cost([{"model": "gpt-5-nano"}], fake_price)
+
+
+def test_row_cost_adds_router_calls_at_their_own_price(evaluator):
+    row = {
+        "prediction": "gpt-5-mini",
+        "generated_result": {
+            "token_usage": {"input_tokens": 1_000_000, "output_tokens": 1_000_000}
+        },
+        "router_usage": [
+            {"model": "gpt-5-nano", "input_tokens": 1_000_000, "output_tokens": 0}
+        ],
+    }
+    answer, router = compute_row_cost(row, evaluator, "gpt-5-mini")
+    assert answer == pytest.approx(0.25 + 2.0)  # gpt-5-mini: $0.25 / $2.00 per 1M
+    assert router == pytest.approx(0.05)  # gpt-5-nano input: $0.05 per 1M
+
+
+def test_unpriced_router_model_pays_the_maximum_price(evaluator):
+    row = {
+        "prediction": "gpt-5-mini",
+        "generated_result": {"token_usage": {"input_tokens": 0, "output_tokens": 0}},
+        "router_usage": [
+            {"model": "no-such-model", "input_tokens": 1_000_000, "output_tokens": 0}
+        ],
+    }
+    _, router = compute_row_cost(row, evaluator, "gpt-5-mini")
+    assert router == pytest.approx(
+        evaluator.max_price_info()["input_token_price_per_million"]
+    )