From f0cbf4918a99dd88218ce6052769780739e0d4fe Mon Sep 17 00:00:00 2001 From: Yifan Lu <111810457+yl231@users.noreply.github.com> Date: Thu, 29 Oct 2026 17:51:27 -0500 Subject: [PATCH] Charge routers for the LLM calls they make themselves A prediction row may now list router_usage: each LLM call the router made to decide (routing model, classifier, voters, critic, earlier cascade steps) with its model, input tokens and output tokens (reasoning included). The evaluator bills these calls at their models' prices on top of the answer, the checker validates the field and includes its models in the price check, and scripts/pricing/compute_cost.py reports a file's cost the same way, offline. Per-row cost logic moves into compute_row_cost so the evaluator and the tool share it. Co-Authored-By: Claude Opus 5.5 --- README.md | 4 + global_utils/router_usage.py | 88 ++++++++++++++++++ llm_evaluation/run.py | 56 +++++++---- .../check_config_prediction_files.py | 13 +++ scripts/pricing/compute_cost.py | 81 ++++++++++++++++ tests/test_router_usage.py | 93 +++++++++++++++++++ 6 files changed, 317 insertions(+), 18 deletions(-) create mode 100644 global_utils/router_usage.py create mode 100644 scripts/pricing/compute_cost.py create mode 100644 tests/test_router_usage.py diff --git a/README.md b/README.md index 4840fd78..29b1c350 100644 --- a/README.md +++ b/README.md @@ -212,6 +212,10 @@ To get your router on the leaderboard, you can open a Pull Request with your rou - `router_inference/config/.json` - Your router configuration - `router_inference/predictions/.json` - Your prediction file with `generated_result` fields populated - `router_inference/predictions/-robustness.json` - Your prediction file for robustness evaluation, no `generated_result` fields needed + + > [!NOTE] + > **Routers that call LLMs themselves.** If your router calls an LLM to make its decision (a routing model, a classifier, voters, a critic, or the earlier steps of a cascade), record each call in the row's `router_usage` list as `{"model": ..., "input_tokens": ..., "output_tokens": ...}`, with an optional `purpose`. Count reasoning tokens in `output_tokens`. The evaluator adds these calls to the row's cost at each model's price. Check your total with `uv run python scripts/pricing/compute_cost.py `. + 2. **Open a Pull Request to `main` branch and call `/evaluate` in the PR comment** - When the PR is ready for evaluation, call `/evaluate` in the PR comment to trigger the evaluation workflow. See an example [here](https://github.com/RouteWorks/RouterArena/pull/71#issuecomment-3904936480). - The automated workflow will: diff --git a/global_utils/router_usage.py b/global_utils/router_usage.py new file mode 100644 index 00000000..fd6e39b9 --- /dev/null +++ b/global_utils/router_usage.py @@ -0,0 +1,88 @@ +# SPDX-FileCopyrightText: Copyright contributors to the RouterArena project +# SPDX-License-Identifier: Apache-2.0 + +"""Router calls: LLM calls a router makes itself, besides the call that produces the answer. + +A router that uses an LLM to decide (a routing model, a classifier, voters, a critic, the +earlier steps of a cascade) records each such call in its prediction row: + + "router_usage": [ + {"model": "openai/gpt-4o-mini", "input_tokens": 812, "output_tokens": 37, + "purpose": "routing"} + ] + +``output_tokens`` counts every generated token, reasoning included, as the provider bills +it. ``purpose`` is optional free text. Each call is priced like an answer: the model's +price profile in ``model_cost/model_profiles.yaml``, or the table's highest price if the +model has none. The row's cost is the answer's cost plus the cost of its router calls. +""" + +from __future__ import annotations + +from typing import Any, Callable, Dict, List + +__all__ = [ + "ROUTER_USAGE_FIELD", + "validate_router_usage", + "router_usage_models", + "router_usage_cost", +] + +ROUTER_USAGE_FIELD = "router_usage" + +# (model name, token usage) -> USD; ModelEvaluator.calculate_inference_cost fits. +PriceFn = Callable[[str, Dict[str, int]], float] + + +def validate_router_usage(value: Any) -> List[str]: + """Return a list of problems with a row's ``router_usage`` (empty if it is valid).""" + if not isinstance(value, list): + return [f"{ROUTER_USAGE_FIELD} must be a list of calls"] + errors: List[str] = [] + for i, call in enumerate(value): + where = f"{ROUTER_USAGE_FIELD}[{i}]" + if not isinstance(call, dict): + errors.append(f"{where} must be an object") + continue + if not isinstance(call.get("model"), str) or not call["model"].strip(): + errors.append(f"{where}.model must be a non-empty string") + for field in ("input_tokens", "output_tokens"): + count = call.get(field) + if not isinstance(count, int) or isinstance(count, bool) or count < 0: + errors.append(f"{where}.{field} must be an integer >= 0") + if "purpose" in call and not isinstance(call["purpose"], str): + errors.append(f"{where}.purpose must be a string") + return errors + + +def router_usage_models(value: Any) -> List[str]: + """Model names used by a row's router calls (for price checks).""" + if not isinstance(value, list): + return [] + return [ + c["model"] + for c in value + if isinstance(c, dict) and isinstance(c.get("model"), str) + ] + + +def router_usage_cost(value: Any, price: PriceFn) -> float: + """Total cost of a row's router calls. A missing field costs nothing. + + Raises ValueError for a malformed field, so a bad record is never priced as free.""" + if value is None: + return 0.0 + errors = validate_router_usage(value) + if errors: + raise ValueError("; ".join(errors)) + return sum( + price( + call["model"], + { + "input_tokens": call["input_tokens"], + "output_tokens": call["output_tokens"], + "total_tokens": call["input_tokens"] + call["output_tokens"], + }, + ) + for call in value + ) diff --git a/llm_evaluation/run.py b/llm_evaluation/run.py index c72fca01..18f8c886 100644 --- a/llm_evaluation/run.py +++ b/llm_evaluation/run.py @@ -27,6 +27,7 @@ from typing import Dict, Any, List, Optional, Tuple, Set from universal_model_names import ModelNameManager from global_utils.robustness import compute_robustness_score +from global_utils.router_usage import ROUTER_USAGE_FIELD, router_usage_cost from parallel_evaluation import ParallelEvaluationManager # Add parent directory to path for imports @@ -315,6 +316,38 @@ def get_livecodebench_ground_truth(global_index: str) -> Optional[Dict[str, Any] return None +def compute_row_cost( + prediction: Dict[str, Any], + evaluator: ModelEvaluator, + universal_model_name: str, +) -> Tuple[float, float]: + """Return (answer cost, router-call cost) in USD for one prediction row. + + The answer is billed at the model the provider actually served + (generated_result.model_used) when it has a known price, so retired/redirected + slugs are billed at the model that truly answered rather than the requested + alias. For example the retired `grok-4-1-fast-reasoning` slug is redirected by + xAI to grok-4.3, which is both stronger and ~5-6x more expensive; billing it at + the alias's old price understates cost. Otherwise it falls back to the router's + selected model (universal name, to respect universal_model_names.py mappings). + See #166. + + Router calls (the row's `router_usage`, see global_utils/router_usage.py) are + billed at their own models' prices. + """ + generated_result = prediction.get("generated_result") or {} + token_usage = generated_result.get("token_usage", {}) + cost_model_name = universal_model_name + actual_model = generated_result.get("model_used") + if actual_model and evaluator.has_price(actual_model): + cost_model_name = actual_model + answer_cost = evaluator.calculate_inference_cost(cost_model_name, token_usage) + router_cost = router_usage_cost( + prediction.get(ROUTER_USAGE_FIELD), evaluator.calculate_inference_cost + ) + return answer_cost, router_cost + + def evaluate_single_prediction( prediction: Dict[str, Any], ground_truth_map: Dict[str, Dict[str, Any]], @@ -429,28 +462,15 @@ def evaluate_single_prediction( generated_answer, ground_truth, scorer_func, dataset_name ) - # Calculate inference cost. - # Prefer the model the provider actually served (generated_result.model_used) - # when it has a known price, so retired/redirected slugs are billed at the - # model that truly answered rather than the requested alias. For example the - # retired `grok-4-1-fast-reasoning` slug is redirected by xAI to grok-4.3, - # which is both stronger and ~5-6x more expensive; billing it at the alias's - # old price understates cost. Fall back to the router's selected model - # (universal name, to respect universal_model_names.py mappings) when the - # provider did not report an actual model or its price is unknown. See #166. - token_usage = generated_result.get("token_usage", {}) - cost_model_name = universal_model_name - actual_model = generated_result.get("model_used") - if actual_model and evaluator.has_price(actual_model): - cost_model_name = actual_model - inference_cost = evaluator.calculate_inference_cost( - cost_model_name, - token_usage, + answer_cost, router_cost = compute_row_cost( + prediction, evaluator, universal_model_name ) # Update the prediction with evaluation results prediction["accuracy"] = score - prediction["cost"] = inference_cost + prediction["cost"] = answer_cost + router_cost + if router_cost: + prediction["router_cost"] = router_cost return True diff --git a/router_inference/check_config_prediction_files.py b/router_inference/check_config_prediction_files.py index c1c7857d..1351afaf 100644 --- a/router_inference/check_config_prediction_files.py +++ b/router_inference/check_config_prediction_files.py @@ -28,6 +28,11 @@ sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "../"))) from universal_model_names import ModelNameManager +from global_utils.router_usage import ( + ROUTER_USAGE_FIELD, + router_usage_models, + validate_router_usage, +) # Expected dataset sizes (robustness derived from dataset length) EXPECTED_SIZES = { @@ -170,6 +175,7 @@ def check_model_costs( model_prediction = prediction.get("prediction") if model_prediction: used_models.add(model_prediction) + used_models.update(router_usage_models(prediction.get(ROUTER_USAGE_FIELD))) # Also include models from config config_models = config.get("pipeline_params", {}).get("models", []) @@ -388,6 +394,13 @@ def check_prediction_fields( ) continue + # Router calls the router made itself (optional; see global_utils/router_usage.py) + if ROUTER_USAGE_FIELD in prediction: + for problem in validate_router_usage(prediction[ROUTER_USAGE_FIELD]): + errors.append( + f"Entry {i} (global_index: {pred_global_index}): {problem}" + ) + # Check prompt - try both "prompt" and "prompt_formatted" fields pred_prompt = prediction.get("prompt") or prediction.get("prompt_formatted") dataset_entry = dataset_map[pred_global_index] diff --git a/scripts/pricing/compute_cost.py b/scripts/pricing/compute_cost.py new file mode 100644 index 00000000..474b1fd2 --- /dev/null +++ b/scripts/pricing/compute_cost.py @@ -0,0 +1,81 @@ +# SPDX-FileCopyrightText: Copyright contributors to the RouterArena project +# SPDX-License-Identifier: Apache-2.0 + +"""Compute a prediction file's cost the way the evaluator does, router calls included. + +Every scored row is billed for its answer (token_usage at the answering model's price) plus +its router calls (``router_usage``, see global_utils/router_usage.py), using the price table +in model_cost/. Models without a price profile are charged the table's highest price and +listed at the end. No answer is graded, so this runs offline in seconds. + +Usage: + uv run python scripts/pricing/compute_cost.py your-router [--split full] [--per-row out.csv] +""" + +from __future__ import annotations + +import argparse +import csv +import os +import sys +from pathlib import Path +from typing import List, Optional + +REPO_ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(REPO_ROOT)) +sys.path.insert(0, str(REPO_ROOT / "llm_evaluation")) + +from evaluate_models import ModelEvaluator # noqa: E402 +from run import compute_row_cost, load_predictions_file # noqa: E402 +from universal_model_names import ModelNameManager # noqa: E402 + + +def main(argv: Optional[List[str]] = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("router_name") + parser.add_argument("--split", default="full", choices=["sub_10", "full"]) + parser.add_argument("--per-row", help="write one CSV line per scored row") + args = parser.parse_args(argv) + + os.chdir(REPO_ROOT) + evaluator = ModelEvaluator(cached_results_dir="./cached_results") + rows = [ + p + for p in load_predictions_file(args.router_name, split=args.split) + if not p.get("for_optimality") + ] + + answer_total = router_total = 0.0 + per_row = [] + for prediction in rows: + model = prediction.get("prediction", "") + try: + universal = ModelNameManager.get_universal_name(model) + except Exception: + universal = model + answer, router = compute_row_cost(prediction, evaluator, universal) + answer_total += answer + router_total += router + per_row.append((prediction.get("global index"), model, answer, router)) + + total = answer_total + router_total + n = max(1, len(rows)) + print(f"Router: {args.router_name} ({len(rows)} scored rows, split {args.split})") + print(f" answers: ${answer_total:.6f}") + print(f" router calls: ${router_total:.6f}") + print(f" total: ${total:.6f} (${1000 * total / n:.4f} per 1K queries)") + if evaluator.unpriced_models: + print("Charged the maximum price (no price profile):") + for name, count in sorted(evaluator.unpriced_models.items()): + print(f" - {name}: {count} call(s)") + + if args.per_row: + with open(args.per_row, "w", newline="", encoding="utf-8") as f: + writer = csv.writer(f) + writer.writerow(["global_index", "model", "answer_cost", "router_cost"]) + writer.writerows(per_row) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_router_usage.py b/tests/test_router_usage.py new file mode 100644 index 00000000..fcd05556 --- /dev/null +++ b/tests/test_router_usage.py @@ -0,0 +1,93 @@ +# SPDX-FileCopyrightText: Copyright contributors to the RouterArena project +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for router-call costs (global_utils/router_usage.py and the evaluator).""" + +import sys +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO_ROOT)) +sys.path.insert(0, str(REPO_ROOT / "llm_evaluation")) + +from global_utils.router_usage import ( # noqa: E402 + router_usage_cost, + router_usage_models, + validate_router_usage, +) +from evaluate_models import ModelEvaluator # noqa: E402 +from run import compute_row_cost # noqa: E402 + +CALL = {"model": "gpt-5-nano", "input_tokens": 1000, "output_tokens": 500} + + +def fake_price(model, usage): + return usage["input_tokens"] * 1e-6 + usage["output_tokens"] * 2e-6 + + +@pytest.fixture(scope="module") +def evaluator(): + return ModelEvaluator(cached_results_dir=str(REPO_ROOT / "cached_results")) + + +def test_valid_record_has_no_errors(): + assert validate_router_usage([CALL, {**CALL, "purpose": "routing"}]) == [] + assert validate_router_usage([]) == [] + + +@pytest.mark.parametrize( + "value, problem", + [ + ({"model": "x"}, "must be a list"), + (["x"], "must be an object"), + ([{**CALL, "model": ""}], "model must be a non-empty string"), + ([{**CALL, "input_tokens": -1}], "input_tokens must be an integer"), + ([{**CALL, "output_tokens": 1.5}], "output_tokens must be an integer"), + ([{**CALL, "output_tokens": True}], "output_tokens must be an integer"), + ([{**CALL, "purpose": 3}], "purpose must be a string"), + ], +) +def test_invalid_records_are_reported(value, problem): + assert any(problem in e for e in validate_router_usage(value)) + + +def test_cost_sums_every_call_and_missing_field_is_free(): + assert router_usage_cost(None, fake_price) == 0.0 + assert router_usage_cost([CALL, CALL], fake_price) == pytest.approx(2 * 0.002) + assert router_usage_models([CALL, "x"]) == ["gpt-5-nano"] + + +def test_malformed_record_is_never_priced_as_free(): + with pytest.raises(ValueError): + router_usage_cost([{"model": "gpt-5-nano"}], fake_price) + + +def test_row_cost_adds_router_calls_at_their_own_price(evaluator): + row = { + "prediction": "gpt-5-mini", + "generated_result": { + "token_usage": {"input_tokens": 1_000_000, "output_tokens": 1_000_000} + }, + "router_usage": [ + {"model": "gpt-5-nano", "input_tokens": 1_000_000, "output_tokens": 0} + ], + } + answer, router = compute_row_cost(row, evaluator, "gpt-5-mini") + assert answer == pytest.approx(0.25 + 2.0) # gpt-5-mini: $0.25 / $2.00 per 1M + assert router == pytest.approx(0.05) # gpt-5-nano input: $0.05 per 1M + + +def test_unpriced_router_model_pays_the_maximum_price(evaluator): + row = { + "prediction": "gpt-5-mini", + "generated_result": {"token_usage": {"input_tokens": 0, "output_tokens": 0}}, + "router_usage": [ + {"model": "no-such-model", "input_tokens": 1_000_000, "output_tokens": 0} + ], + } + _, router = compute_row_cost(row, evaluator, "gpt-5-mini") + assert router == pytest.approx( + evaluator.max_price_info()["input_token_price_per_million"] + )