diff --git a/README.md b/README.md index f3e3d86a..d1e31b65 100644 --- a/README.md +++ b/README.md @@ -218,6 +218,10 @@ To get your router on the leaderboard, you can open a Pull Request with your rou - `router_inference/config/.json` - Your router configuration - `router_inference/predictions/.json` - Your prediction file with `generated_result` fields populated - `router_inference/predictions/-robustness.json` - Your prediction file for robustness evaluation, no `generated_result` fields needed + + > [!NOTE] + > **Routers that call LLMs themselves.** If your router calls an LLM to make its decision (a routing model, a classifier, voters, a critic, or the earlier steps of a cascade), record each call in the row's `router_usage` list as `{"model": ..., "input_tokens": ..., "output_tokens": ...}`, with an optional `purpose`. Count reasoning tokens in `output_tokens`. The evaluator adds these calls to the row's cost at each model's price. Check your total with `uv run python scripts/pricing/compute_cost.py `. + 2. **Open a Pull Request to `main` branch and call `/evaluate` in the PR comment** - When the PR is ready for evaluation, call `/evaluate` in the PR comment to trigger the evaluation workflow. See an example [here](https://github.com/RouteWorks/RouterArena/pull/71#issuecomment-3904936480). - The automated workflow will: diff --git a/global_utils/router_usage.py b/global_utils/router_usage.py new file mode 100644 index 00000000..fd6e39b9 --- /dev/null +++ b/global_utils/router_usage.py @@ -0,0 +1,88 @@ +# SPDX-FileCopyrightText: Copyright contributors to the RouterArena project +# SPDX-License-Identifier: Apache-2.0 + +"""Router calls: LLM calls a router makes itself, besides the call that produces the answer. + +A router that uses an LLM to decide (a routing model, a classifier, voters, a critic, the +earlier steps of a cascade) records each such call in its prediction row: + + "router_usage": [ + {"model": "openai/gpt-4o-mini", "input_tokens": 812, "output_tokens": 37, + "purpose": "routing"} + ] + +``output_tokens`` counts every generated token, reasoning included, as the provider bills +it. ``purpose`` is optional free text. Each call is priced like an answer: the model's +price profile in ``model_cost/model_profiles.yaml``, or the table's highest price if the +model has none. The row's cost is the answer's cost plus the cost of its router calls. +""" + +from __future__ import annotations + +from typing import Any, Callable, Dict, List + +__all__ = [ + "ROUTER_USAGE_FIELD", + "validate_router_usage", + "router_usage_models", + "router_usage_cost", +] + +ROUTER_USAGE_FIELD = "router_usage" + +# (model name, token usage) -> USD; ModelEvaluator.calculate_inference_cost fits. +PriceFn = Callable[[str, Dict[str, int]], float] + + +def validate_router_usage(value: Any) -> List[str]: + """Return a list of problems with a row's ``router_usage`` (empty if it is valid).""" + if not isinstance(value, list): + return [f"{ROUTER_USAGE_FIELD} must be a list of calls"] + errors: List[str] = [] + for i, call in enumerate(value): + where = f"{ROUTER_USAGE_FIELD}[{i}]" + if not isinstance(call, dict): + errors.append(f"{where} must be an object") + continue + if not isinstance(call.get("model"), str) or not call["model"].strip(): + errors.append(f"{where}.model must be a non-empty string") + for field in ("input_tokens", "output_tokens"): + count = call.get(field) + if not isinstance(count, int) or isinstance(count, bool) or count < 0: + errors.append(f"{where}.{field} must be an integer >= 0") + if "purpose" in call and not isinstance(call["purpose"], str): + errors.append(f"{where}.purpose must be a string") + return errors + + +def router_usage_models(value: Any) -> List[str]: + """Model names used by a row's router calls (for price checks).""" + if not isinstance(value, list): + return [] + return [ + c["model"] + for c in value + if isinstance(c, dict) and isinstance(c.get("model"), str) + ] + + +def router_usage_cost(value: Any, price: PriceFn) -> float: + """Total cost of a row's router calls. A missing field costs nothing. + + Raises ValueError for a malformed field, so a bad record is never priced as free.""" + if value is None: + return 0.0 + errors = validate_router_usage(value) + if errors: + raise ValueError("; ".join(errors)) + return sum( + price( + call["model"], + { + "input_tokens": call["input_tokens"], + "output_tokens": call["output_tokens"], + "total_tokens": call["input_tokens"] + call["output_tokens"], + }, + ) + for call in value + ) diff --git a/llm_evaluation/run.py b/llm_evaluation/run.py index c72fca01..18f8c886 100644 --- a/llm_evaluation/run.py +++ b/llm_evaluation/run.py @@ -27,6 +27,7 @@ from typing import Dict, Any, List, Optional, Tuple, Set from universal_model_names import ModelNameManager from global_utils.robustness import compute_robustness_score +from global_utils.router_usage import ROUTER_USAGE_FIELD, router_usage_cost from parallel_evaluation import ParallelEvaluationManager # Add parent directory to path for imports @@ -315,6 +316,38 @@ def get_livecodebench_ground_truth(global_index: str) -> Optional[Dict[str, Any] return None +def compute_row_cost( + prediction: Dict[str, Any], + evaluator: ModelEvaluator, + universal_model_name: str, +) -> Tuple[float, float]: + """Return (answer cost, router-call cost) in USD for one prediction row. + + The answer is billed at the model the provider actually served + (generated_result.model_used) when it has a known price, so retired/redirected + slugs are billed at the model that truly answered rather than the requested + alias. For example the retired `grok-4-1-fast-reasoning` slug is redirected by + xAI to grok-4.3, which is both stronger and ~5-6x more expensive; billing it at + the alias's old price understates cost. Otherwise it falls back to the router's + selected model (universal name, to respect universal_model_names.py mappings). + See #166. + + Router calls (the row's `router_usage`, see global_utils/router_usage.py) are + billed at their own models' prices. + """ + generated_result = prediction.get("generated_result") or {} + token_usage = generated_result.get("token_usage", {}) + cost_model_name = universal_model_name + actual_model = generated_result.get("model_used") + if actual_model and evaluator.has_price(actual_model): + cost_model_name = actual_model + answer_cost = evaluator.calculate_inference_cost(cost_model_name, token_usage) + router_cost = router_usage_cost( + prediction.get(ROUTER_USAGE_FIELD), evaluator.calculate_inference_cost + ) + return answer_cost, router_cost + + def evaluate_single_prediction( prediction: Dict[str, Any], ground_truth_map: Dict[str, Dict[str, Any]], @@ -429,28 +462,15 @@ def evaluate_single_prediction( generated_answer, ground_truth, scorer_func, dataset_name ) - # Calculate inference cost. - # Prefer the model the provider actually served (generated_result.model_used) - # when it has a known price, so retired/redirected slugs are billed at the - # model that truly answered rather than the requested alias. For example the - # retired `grok-4-1-fast-reasoning` slug is redirected by xAI to grok-4.3, - # which is both stronger and ~5-6x more expensive; billing it at the alias's - # old price understates cost. Fall back to the router's selected model - # (universal name, to respect universal_model_names.py mappings) when the - # provider did not report an actual model or its price is unknown. See #166. - token_usage = generated_result.get("token_usage", {}) - cost_model_name = universal_model_name - actual_model = generated_result.get("model_used") - if actual_model and evaluator.has_price(actual_model): - cost_model_name = actual_model - inference_cost = evaluator.calculate_inference_cost( - cost_model_name, - token_usage, + answer_cost, router_cost = compute_row_cost( + prediction, evaluator, universal_model_name ) # Update the prediction with evaluation results prediction["accuracy"] = score - prediction["cost"] = inference_cost + prediction["cost"] = answer_cost + router_cost + if router_cost: + prediction["router_cost"] = router_cost return True diff --git a/router_inference/check_config_prediction_files.py b/router_inference/check_config_prediction_files.py index c1c7857d..1351afaf 100644 --- a/router_inference/check_config_prediction_files.py +++ b/router_inference/check_config_prediction_files.py @@ -28,6 +28,11 @@ sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "../"))) from universal_model_names import ModelNameManager +from global_utils.router_usage import ( + ROUTER_USAGE_FIELD, + router_usage_models, + validate_router_usage, +) # Expected dataset sizes (robustness derived from dataset length) EXPECTED_SIZES = { @@ -170,6 +175,7 @@ def check_model_costs( model_prediction = prediction.get("prediction") if model_prediction: used_models.add(model_prediction) + used_models.update(router_usage_models(prediction.get(ROUTER_USAGE_FIELD))) # Also include models from config config_models = config.get("pipeline_params", {}).get("models", []) @@ -388,6 +394,13 @@ def check_prediction_fields( ) continue + # Router calls the router made itself (optional; see global_utils/router_usage.py) + if ROUTER_USAGE_FIELD in prediction: + for problem in validate_router_usage(prediction[ROUTER_USAGE_FIELD]): + errors.append( + f"Entry {i} (global_index: {pred_global_index}): {problem}" + ) + # Check prompt - try both "prompt" and "prompt_formatted" fields pred_prompt = prediction.get("prompt") or prediction.get("prompt_formatted") dataset_entry = dataset_map[pred_global_index] diff --git a/scripts/pricing/compute_cost.py b/scripts/pricing/compute_cost.py new file mode 100644 index 00000000..474b1fd2 --- /dev/null +++ b/scripts/pricing/compute_cost.py @@ -0,0 +1,81 @@ +# SPDX-FileCopyrightText: Copyright contributors to the RouterArena project +# SPDX-License-Identifier: Apache-2.0 + +"""Compute a prediction file's cost the way the evaluator does, router calls included. + +Every scored row is billed for its answer (token_usage at the answering model's price) plus +its router calls (``router_usage``, see global_utils/router_usage.py), using the price table +in model_cost/. Models without a price profile are charged the table's highest price and +listed at the end. No answer is graded, so this runs offline in seconds. + +Usage: + uv run python scripts/pricing/compute_cost.py your-router [--split full] [--per-row out.csv] +""" + +from __future__ import annotations + +import argparse +import csv +import os +import sys +from pathlib import Path +from typing import List, Optional + +REPO_ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(REPO_ROOT)) +sys.path.insert(0, str(REPO_ROOT / "llm_evaluation")) + +from evaluate_models import ModelEvaluator # noqa: E402 +from run import compute_row_cost, load_predictions_file # noqa: E402 +from universal_model_names import ModelNameManager # noqa: E402 + + +def main(argv: Optional[List[str]] = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("router_name") + parser.add_argument("--split", default="full", choices=["sub_10", "full"]) + parser.add_argument("--per-row", help="write one CSV line per scored row") + args = parser.parse_args(argv) + + os.chdir(REPO_ROOT) + evaluator = ModelEvaluator(cached_results_dir="./cached_results") + rows = [ + p + for p in load_predictions_file(args.router_name, split=args.split) + if not p.get("for_optimality") + ] + + answer_total = router_total = 0.0 + per_row = [] + for prediction in rows: + model = prediction.get("prediction", "") + try: + universal = ModelNameManager.get_universal_name(model) + except Exception: + universal = model + answer, router = compute_row_cost(prediction, evaluator, universal) + answer_total += answer + router_total += router + per_row.append((prediction.get("global index"), model, answer, router)) + + total = answer_total + router_total + n = max(1, len(rows)) + print(f"Router: {args.router_name} ({len(rows)} scored rows, split {args.split})") + print(f" answers: ${answer_total:.6f}") + print(f" router calls: ${router_total:.6f}") + print(f" total: ${total:.6f} (${1000 * total / n:.4f} per 1K queries)") + if evaluator.unpriced_models: + print("Charged the maximum price (no price profile):") + for name, count in sorted(evaluator.unpriced_models.items()): + print(f" - {name}: {count} call(s)") + + if args.per_row: + with open(args.per_row, "w", newline="", encoding="utf-8") as f: + writer = csv.writer(f) + writer.writerow(["global_index", "model", "answer_cost", "router_cost"]) + writer.writerows(per_row) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_router_usage.py b/tests/test_router_usage.py new file mode 100644 index 00000000..fcd05556 --- /dev/null +++ b/tests/test_router_usage.py @@ -0,0 +1,93 @@ +# SPDX-FileCopyrightText: Copyright contributors to the RouterArena project +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for router-call costs (global_utils/router_usage.py and the evaluator).""" + +import sys +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO_ROOT)) +sys.path.insert(0, str(REPO_ROOT / "llm_evaluation")) + +from global_utils.router_usage import ( # noqa: E402 + router_usage_cost, + router_usage_models, + validate_router_usage, +) +from evaluate_models import ModelEvaluator # noqa: E402 +from run import compute_row_cost # noqa: E402 + +CALL = {"model": "gpt-5-nano", "input_tokens": 1000, "output_tokens": 500} + + +def fake_price(model, usage): + return usage["input_tokens"] * 1e-6 + usage["output_tokens"] * 2e-6 + + +@pytest.fixture(scope="module") +def evaluator(): + return ModelEvaluator(cached_results_dir=str(REPO_ROOT / "cached_results")) + + +def test_valid_record_has_no_errors(): + assert validate_router_usage([CALL, {**CALL, "purpose": "routing"}]) == [] + assert validate_router_usage([]) == [] + + +@pytest.mark.parametrize( + "value, problem", + [ + ({"model": "x"}, "must be a list"), + (["x"], "must be an object"), + ([{**CALL, "model": ""}], "model must be a non-empty string"), + ([{**CALL, "input_tokens": -1}], "input_tokens must be an integer"), + ([{**CALL, "output_tokens": 1.5}], "output_tokens must be an integer"), + ([{**CALL, "output_tokens": True}], "output_tokens must be an integer"), + ([{**CALL, "purpose": 3}], "purpose must be a string"), + ], +) +def test_invalid_records_are_reported(value, problem): + assert any(problem in e for e in validate_router_usage(value)) + + +def test_cost_sums_every_call_and_missing_field_is_free(): + assert router_usage_cost(None, fake_price) == 0.0 + assert router_usage_cost([CALL, CALL], fake_price) == pytest.approx(2 * 0.002) + assert router_usage_models([CALL, "x"]) == ["gpt-5-nano"] + + +def test_malformed_record_is_never_priced_as_free(): + with pytest.raises(ValueError): + router_usage_cost([{"model": "gpt-5-nano"}], fake_price) + + +def test_row_cost_adds_router_calls_at_their_own_price(evaluator): + row = { + "prediction": "gpt-5-mini", + "generated_result": { + "token_usage": {"input_tokens": 1_000_000, "output_tokens": 1_000_000} + }, + "router_usage": [ + {"model": "gpt-5-nano", "input_tokens": 1_000_000, "output_tokens": 0} + ], + } + answer, router = compute_row_cost(row, evaluator, "gpt-5-mini") + assert answer == pytest.approx(0.25 + 2.0) # gpt-5-mini: $0.25 / $2.00 per 1M + assert router == pytest.approx(0.05) # gpt-5-nano input: $0.05 per 1M + + +def test_unpriced_router_model_pays_the_maximum_price(evaluator): + row = { + "prediction": "gpt-5-mini", + "generated_result": {"token_usage": {"input_tokens": 0, "output_tokens": 0}}, + "router_usage": [ + {"model": "no-such-model", "input_tokens": 1_000_000, "output_tokens": 0} + ], + } + _, router = compute_row_cost(row, evaluator, "gpt-5-mini") + assert router == pytest.approx( + evaluator.max_price_info()["input_token_price_per_million"] + )