From 107a209cb5782da8571038f59f7b0d890165b1d9 Mon Sep 17 00:00:00 2001 From: Yifan Lu Date: Sat, 31 Oct 2026 17:14:46 -0500 Subject: [PATCH] Add a re-pricing tool: re-cost routers at the current price table without re-grading scripts/pricing/reprice.py re-costs every row the evaluator costed, from the token counts already in the prediction file, with the evaluator's own compute_row_cost (answer plus router calls, served model when priced), then recomputes the metrics with the evaluator's own router_metrics. No answer is graded and no submitted code runs. reprice.py ROUTER [ROUTER ...] report only (e.g. a new submission) reprice.py --all every router in the manifest reprice.py --all --write rewrite costs (file formatting kept) and README rows, re-ranking the table run.py: the metric computation moves out of compute_router_metrics into router_metrics(), which returns the metrics.json dict; compute_router_metrics keeps logging and writing metrics.json as before. Co-Authored-By: Claude Opus 5.5 --- llm_evaluation/run.py | 151 ++++++++++-------- scripts/pricing/reprice.py | 245 +++++++++++++++++++++++++++++ scripts/pricing/sync_openrouter.py | 2 +- tests/test_reprice.py | 161 +++++++++++++++++++ 4 files changed, 496 insertions(+), 63 deletions(-) create mode 100644 scripts/pricing/reprice.py create mode 100644 tests/test_reprice.py diff --git a/llm_evaluation/run.py b/llm_evaluation/run.py index 18f8c886..f757ec5b 100644 --- a/llm_evaluation/run.py +++ b/llm_evaluation/run.py @@ -1006,9 +1006,11 @@ def run_robustness_only(router_name: str, robustness_path: Optional[str]) -> Non logger.info("Robustness metrics saved to %s", metrics_path) -def compute_router_metrics(predictions: List[Dict[str, Any]], router_name: str) -> None: +def router_metrics( + predictions: List[Dict[str, Any]], router_name: str +) -> Tuple[Dict[str, Any], Dict[str, Any]]: """ - Compute router-level metrics (accuracy, cost, RouterArena score, etc.) and display them. + Compute router-level metrics (accuracy, cost, RouterArena score, optimality). Separates regular and optimality entries: - Regular entries (for_optimality=False): Used for RouterArena score @@ -1017,6 +1019,10 @@ def compute_router_metrics(predictions: List[Dict[str, Any]], router_name: str) Args: predictions: List of prediction dictionaries with evaluation results router_name: Name of the router + + Returns: + (metrics, details): ``metrics`` is what gets saved to metrics.json; + ``details`` holds counts and the optimality result used only for logging. """ # Separate regular and optimality predictions regular_predictions = [p for p in predictions if not p.get("for_optimality", False)] @@ -1090,94 +1096,115 @@ def compute_router_metrics(predictions: List[Dict[str, Any]], router_name: str) # Compute RouterArena score using average cost per 1000 queries and average accuracy arena_score = compute_arena_score(avg_cost_per_1000, avg_accuracy) + metrics_dict = { + "arena_score": arena_score, + "accuracy": avg_accuracy, + "total_cost": total_cost, + "avg_cost_per_query": total_cost / num_queries if num_queries > 0 else 0.0, + "avg_cost_per_1000": avg_cost_per_1000, + "num_queries": num_queries, + "abnormal_count": abnormal_count, + } + details: Dict[str, Any] = { + "num_accuracies": len(accuracies), + "valid_cost_count": valid_cost_count, + "num_optimality_entries": len(optimality_predictions), + "optimality_scores": None, + "optimality_error": None, + } + + # Compute optimality scores if we have optimality entries + if optimality_predictions: + try: + optimality_scores = compute_optimality_from_predictions( + predictions, router_name + ) + except Exception as e: + details["optimality_error"] = e + optimality_scores = None + details["optimality_scores"] = optimality_scores + if optimality_scores: + metrics_dict["optimality"] = { + "opt_sel": optimality_scores["opt_sel"], + "opt_cost": optimality_scores["opt_cost"], + "opt_acc": optimality_scores["opt_acc"], + "queries_with_optimal_data": optimality_scores[ + "queries_with_optimal_data" + ], + "num_sub10_queries": optimality_scores["num_sub10_queries"], + } + + return metrics_dict, details + + +def compute_router_metrics(predictions: List[Dict[str, Any]], router_name: str) -> None: + """ + Compute router-level metrics (see ``router_metrics``), display them and save + them to metrics.json. + + Args: + predictions: List of prediction dictionaries with evaluation results + router_name: Name of the router + """ + metrics_dict, details = router_metrics(predictions, router_name) + num_queries = metrics_dict["num_queries"] + total_cost = metrics_dict["total_cost"] + # Print RouterArena score results logger.info("\n" + "=" * 80) logger.info(f"Router: {router_name}") logger.info("=" * 80) logger.info(f"Total Queries (Regular): {num_queries}") - if optimality_predictions: + if details["num_optimality_entries"]: logger.info( - f"Optimality Entries: {len(optimality_predictions)} (excluded from RouterArena score)" + f"Optimality Entries: {details['num_optimality_entries']} (excluded from RouterArena score)" ) - logger.info(f"Queries with Accuracy: {len(accuracies)}") - logger.info(f"Queries with Valid Cost: {valid_cost_count}") + logger.info(f"Queries with Accuracy: {details['num_accuracies']}") + logger.info(f"Queries with Valid Cost: {details['valid_cost_count']}") logger.info( - f"Abnormal Entries (no valid generation, scored as 0): {abnormal_count}" + f"Abnormal Entries (no valid generation, scored as 0): {metrics_dict['abnormal_count']}" ) - logger.info(f"Average Accuracy: {avg_accuracy:.4f}") + logger.info(f"Average Accuracy: {metrics_dict['accuracy']:.4f}") logger.info(f"Total Cost: ${total_cost:.6f}") if num_queries > 0: logger.info(f"Average Cost per Query: ${total_cost / num_queries:.6f}") else: logger.info("Average Cost per Query: $0.00") - logger.info(f"Average Cost per 1K Queries: ${avg_cost_per_1000:.4f}") - logger.info(f"RouterArena Score: {arena_score:.4f}") + logger.info( + f"Average Cost per 1K Queries: ${metrics_dict['avg_cost_per_1000']:.4f}" + ) + logger.info(f"RouterArena Score: {metrics_dict['arena_score']:.4f}") - # Compute optimality scores if we have optimality entries (compute once, reuse for logging and metrics) - optimality_scores = None - if optimality_predictions: + if details["num_optimality_entries"]: logger.info("\n" + "-" * 80) logger.info("Computing Optimality Scores...") logger.info("-" * 80) - try: - optimality_scores = compute_optimality_from_predictions( - predictions, router_name + optimality_scores = details["optimality_scores"] + if details["optimality_error"] is not None: + logger.warning( + f"Could not compute optimality scores: {details['optimality_error']}" ) - if optimality_scores: - logger.info( - f"Opt.Sel (Optimal Selection): {optimality_scores['opt_sel']:.4f}\n" - f"Opt.Cost (Optimal Cost Ratio): {optimality_scores['opt_cost']:.4f}\n" - f"Opt.Acc (Optimal Accuracy Ratio): {optimality_scores['opt_acc']:.4f}\n" - f"Queries Evaluated for Optimality: {optimality_scores['queries_with_optimal_data']}/{optimality_scores['num_sub10_queries']}" - ) - - # Print optimality scores in JSON format for easy parsing by automation - logger.info( - "\nOptimality Scores: " - + json.dumps( - { - "opt_sel": optimality_scores["opt_sel"], - "opt_cost": optimality_scores["opt_cost"], - "opt_acc": optimality_scores["opt_acc"], - "queries_with_optimal_data": optimality_scores[ - "queries_with_optimal_data" - ], - "num_sub10_queries": optimality_scores["num_sub10_queries"], - } - ) - ) - except Exception as e: - logger.warning(f"Could not compute optimality scores: {e}") logger.warning( "This is expected if optimality entries were not generated during prediction file creation." ) + elif optimality_scores: + logger.info( + f"Opt.Sel (Optimal Selection): {optimality_scores['opt_sel']:.4f}\n" + f"Opt.Cost (Optimal Cost Ratio): {optimality_scores['opt_cost']:.4f}\n" + f"Opt.Acc (Optimal Accuracy Ratio): {optimality_scores['opt_acc']:.4f}\n" + f"Queries Evaluated for Optimality: {optimality_scores['queries_with_optimal_data']}/{optimality_scores['num_sub10_queries']}" + ) + + # Print optimality scores in JSON format for easy parsing by automation + logger.info( + "\nOptimality Scores: " + json.dumps(metrics_dict["optimality"]) + ) logger.info( "\nPLEASE NOTE: The sub_10 dataset is a subset of the full dataset and is used for testing purposes. It is generally easier than the full dataset." ) logger.info("=" * 80 + "\n") - # Save metrics to metrics.json for automation/workflows - metrics_dict = { - "arena_score": arena_score, - "accuracy": avg_accuracy, - "total_cost": total_cost, - "avg_cost_per_query": total_cost / num_queries if num_queries > 0 else 0.0, - "avg_cost_per_1000": avg_cost_per_1000, - "num_queries": num_queries, - "abnormal_count": abnormal_count, - } - - # Add optimality scores if available (reuse previously computed result) - if optimality_scores: - metrics_dict["optimality"] = { - "opt_sel": optimality_scores["opt_sel"], - "opt_cost": optimality_scores["opt_cost"], - "opt_acc": optimality_scores["opt_acc"], - "queries_with_optimal_data": optimality_scores["queries_with_optimal_data"], - "num_sub10_queries": optimality_scores["num_sub10_queries"], - } - # Save to metrics.json metrics_path = "./metrics.json" with open(metrics_path, "w") as f: diff --git a/scripts/pricing/reprice.py b/scripts/pricing/reprice.py new file mode 100644 index 00000000..02c23ec0 --- /dev/null +++ b/scripts/pricing/reprice.py @@ -0,0 +1,245 @@ +# SPDX-FileCopyrightText: Copyright contributors to the RouterArena project +# SPDX-License-Identifier: Apache-2.0 + +"""Re-price routers at the current price table, without re-grading. + +Every row the evaluator costed is re-costed from the token counts already in the +prediction file, at the prices in model_cost/model_cost.json, exactly as the evaluator +bills it (answer plus router calls, see ``compute_row_cost`` in llm_evaluation/run.py). +Metrics are then recomputed with the evaluator's own ``router_metrics``. No answer is +graded and no submitted code runs, so this is quick and safe to run offline. + +Use it after a price sync, to bring the leaderboard to current prices, or on a new +submission whose file was graded with an older price table. + +Usage: + python scripts/pricing/reprice.py ROUTER [ROUTER ...] # report only + python scripts/pricing/reprice.py --all # every router in the manifest + python scripts/pricing/reprice.py --all --write # also rewrite costs and the README + python scripts/pricing/reprice.py --all --report out.md # save the before/after table + +With ``--write``, each prediction file keeps its JSON formatting and only cost values +change, and the README rows of the re-priced routers are updated and the table +re-ranked. README rows without a prediction file (external baselines) keep their numbers. +""" + +from __future__ import annotations + +import argparse +import json +import logging +import os +import re +import sys +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple, Union + +import yaml + +REPO_ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(REPO_ROOT)) +sys.path.insert(0, str(REPO_ROOT / "llm_evaluation")) + +from evaluate_models import ModelEvaluator # noqa: E402 +from run import compute_row_cost, router_metrics # noqa: E402 +from universal_model_names import ModelNameManager # noqa: E402 + +PREDICTIONS_DIR = REPO_ROOT / "router_inference" / "predictions" +MANIFEST_PATH = REPO_ROOT / "leaderboard_manifest.yaml" +README_PATH = REPO_ROOT / "README.md" + + +def reprice_rows(predictions: List[Dict[str, Any]], evaluator: ModelEvaluator) -> int: + """Re-cost, in place, every row that has a numeric cost (the rows the evaluator + costed). Returns the number of rows re-costed.""" + count = 0 + for prediction in predictions: + cost = prediction.get("cost") + if not isinstance(cost, (int, float)) or isinstance(cost, bool): + continue + model = prediction.get("prediction", "") + try: + universal = ModelNameManager.get_universal_name(model) + except Exception: + universal = model + answer_cost, router_cost = compute_row_cost(prediction, evaluator, universal) + prediction["cost"] = answer_cost + router_cost + if router_cost: + prediction["router_cost"] = router_cost + else: + prediction.pop("router_cost", None) + count += 1 + return count + + +def dump_like(original: str, data: Any) -> str: + """Serialize ``data`` in the JSON style of ``original`` (indent, separators, ASCII + escaping, trailing newline), so a diff shows only the values that changed.""" + parsed = json.loads(original) + indents: List[Union[None, int, str]] = [None, 1, 2, 4, "\t"] + for indent in indents: + for ensure_ascii in (False, True): + for separators in ((", ", ": "), (",", ":"), (",", ": ")): + if indent is not None and separators == (", ", ": "): + continue # with an indent, json writes "," between items + text = json.dumps( + parsed, + indent=indent, + ensure_ascii=ensure_ascii, + separators=separators, + ) + if text == original.rstrip("\n"): + rendered = json.dumps( + data, + indent=indent, + ensure_ascii=ensure_ascii, + separators=separators, + ) + return rendered + original[len(text) :] + return json.dumps(data, indent=2, ensure_ascii=False) + "\n" + + +def leaderboard_routers() -> List[Tuple[str, str]]: + """(prediction file name, README name) for every router in the manifest.""" + manifest = yaml.safe_load(MANIFEST_PATH.read_text(encoding="utf-8")) + return [ + (r["prediction"], r["readme_name"]) + for r in manifest["routers"] + if r.get("prediction") and r.get("readme_name") + ] + + +def _norm(name: str) -> str: + return re.sub(r"[\s\-‑\\]", "", name).lower() + + +def _row_name(line: str) -> str: + cell = line.split(" | ")[1] + match = re.match(r"\[([^\]]*)\]", cell) + return match.group(1) if match else cell + + +def _pct(value: float) -> str: + return f"{100 * value:.2f}" + + +def update_readme( + text: str, metrics: Dict[str, Dict[str, Any]] +) -> Tuple[str, List[str]]: + """Update the leaderboard rows named in ``metrics`` (README name -> metrics.json + dict) and re-rank the table by Arena score. Returns (new text, names updated).""" + lines = text.split("\n") + header = next( + i for i, line in enumerate(lines) if line.startswith("| Rank | Router |") + ) + start = header + 2 # skip the header and separator rows + end = start + while end < len(lines) and lines[end].startswith("|"): + end += 1 + by_name = {_norm(name): name for name in metrics} + rows, updated = [], [] + for line in lines[start:end]: + name = by_name.get(_norm(_row_name(line))) + if name is not None: + m = metrics[name] + cells = line.split(" | ") + cells[3] = _pct(m["arena_score"]) + cells[4] = _pct(m["accuracy"]) + cells[5] = f"${m['avg_cost_per_1000']:.2f}" + opt = m.get("optimality") + if opt: + cells[6:9] = [ + _pct(opt["opt_sel"]), + _pct(opt["opt_cost"]), + _pct(opt["opt_acc"]), + ] + line = " | ".join(cells) + updated.append(name) + rows.append(line) + rows.sort(key=lambda line: float(line.split(" | ")[3]), reverse=True) # stable + medals = {1: "🥇", 2: "🥈", 3: "🥉"} + rows = [ + f"| {medals.get(n, n)} | " + line.split(" | ", 1)[1] + for n, line in enumerate(rows, 1) + ] + lines[start:end] = rows + return "\n".join(lines), updated + + +def report_table(results: List[Dict[str, Any]]) -> str: + out = [ + "| Router | Arena | Cost / 1K queries | Accuracy |", + "| --- | --- | --- | --- |", + ] + for r in sorted(results, key=lambda r: -r["after"]["arena_score"]): + b, a = r["before"], r["after"] + out.append( + f"| {r['router']} | {_pct(b['arena_score'])} → {_pct(a['arena_score'])} | " + f"${b['avg_cost_per_1000']:.4f} → ${a['avg_cost_per_1000']:.4f} | " + f"{_pct(a['accuracy'])} |" + ) + return "\n".join(out) + "\n" + + +def main(argv: Optional[List[str]] = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("routers", nargs="*", help="prediction file names") + parser.add_argument( + "--all", action="store_true", help="every router in the manifest" + ) + parser.add_argument( + "--write", + action="store_true", + help="rewrite the prediction files' costs and the README rows", + ) + parser.add_argument("--report", help="also write the before/after table here") + args = parser.parse_args(argv) + + readme_of = dict(leaderboard_routers()) + routers = list(readme_of) if args.all else args.routers + if not routers: + parser.error("name at least one router, or pass --all") + + os.chdir(REPO_ROOT) # the optimality metrics read ./dataset/router_data_10.json + logging.getLogger().setLevel(logging.WARNING) + evaluator = ModelEvaluator(cached_results_dir="./cached_results") + + results, readme_metrics = [], {} + for router in routers: + path = PREDICTIONS_DIR / f"{router}.json" + original = path.read_text(encoding="utf-8") + predictions = json.loads(original) + before, _ = router_metrics(predictions, router) + recosted = reprice_rows(predictions, evaluator) + after, _ = router_metrics(predictions, router) + results.append( + {"router": router, "before": before, "after": after, "rows": recosted} + ) + if args.write: + path.write_text(dump_like(original, predictions), encoding="utf-8") + if router in readme_of: + readme_metrics[readme_of[router]] = after + + table = report_table(results) + print(table) + if evaluator.unpriced_models: + print("Charged the maximum price (no price profile):") + for name, count in sorted(evaluator.unpriced_models.items()): + print(f" - {name}: {count} row(s)") + if args.report: + Path(args.report).write_text(table, encoding="utf-8") + + if args.write and readme_metrics: + text, updated = update_readme( + README_PATH.read_text(encoding="utf-8"), readme_metrics + ) + README_PATH.write_text(text, encoding="utf-8") + print(f"Updated {len(updated)} README row(s) and re-ranked the table.") + missing = set(readme_metrics) - set(updated) + if missing: + print("No README row found for: " + ", ".join(sorted(missing))) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/pricing/sync_openrouter.py b/scripts/pricing/sync_openrouter.py index 50634982..34abfb98 100644 --- a/scripts/pricing/sync_openrouter.py +++ b/scripts/pricing/sync_openrouter.py @@ -16,7 +16,7 @@ (``--always-snapshot`` writes one even when nothing changed). Syncing changes the price table only. Leaderboard costs change when a maintainer re-prices -the routers, as a separate step. +the routers, as a separate step (scripts/pricing/reprice.py). Usage: python scripts/pricing/sync_openrouter.py # fetch, update, write diff --git a/tests/test_reprice.py b/tests/test_reprice.py new file mode 100644 index 00000000..3872a917 --- /dev/null +++ b/tests/test_reprice.py @@ -0,0 +1,161 @@ +# SPDX-FileCopyrightText: Copyright contributors to the RouterArena project +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the re-pricing tool (scripts/pricing/reprice.py).""" + +import json +import sys +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO_ROOT)) +sys.path.insert(0, str(REPO_ROOT / "llm_evaluation")) +sys.path.insert(0, str(REPO_ROOT / "scripts" / "pricing")) + +import reprice # noqa: E402 +from evaluate_models import ModelEvaluator # noqa: E402 +from run import compute_router_metrics, router_metrics # noqa: E402 + +PRICES = { + "cheap-model": { + "input_token_price_per_million": 1.0, + "output_token_price_per_million": 2.0, + }, + "pricey-model": { + "input_token_price_per_million": 10.0, + "output_token_price_per_million": 20.0, + }, +} + + +@pytest.fixture +def evaluator(): + ev = ModelEvaluator(cached_results_dir=str(REPO_ROOT / "cached_results")) + ev.cost_config = dict(PRICES) + return ev + + +def row(model, accuracy, cost, input_tokens=1_000_000, output_tokens=1_000_000): + return { + "global index": f"q-{model}-{accuracy}", + "prediction": model, + "accuracy": accuracy, + "cost": cost, + "generated_result": { + "success": True, + "generated_answer": "x", + "token_usage": { + "input_tokens": input_tokens, + "output_tokens": output_tokens, + "total_tokens": input_tokens + output_tokens, + }, + }, + } + + +def test_rows_are_recosted_at_the_current_prices(evaluator): + rows = [row("cheap-model", 1.0, 99.0), row("pricey-model", 0.0, 99.0)] + assert reprice.reprice_rows(rows, evaluator) == 2 + assert rows[0]["cost"] == pytest.approx(3.0) + assert rows[1]["cost"] == pytest.approx(30.0) + assert rows[0]["accuracy"] == 1.0 and rows[1]["accuracy"] == 0.0 + + +def test_rows_the_evaluator_did_not_cost_are_left_alone(evaluator): + rows = [row("cheap-model", 1.0, None)] + del rows[0]["cost"] + assert reprice.reprice_rows(rows, evaluator) == 0 + assert "cost" not in rows[0] + + +def test_served_model_is_billed_when_it_has_a_price(evaluator): + rows = [row("cheap-model", 1.0, 0.5)] + rows[0]["generated_result"]["model_used"] = "pricey-model" + reprice.reprice_rows(rows, evaluator) + assert rows[0]["cost"] == pytest.approx(30.0) + + +def test_router_calls_are_charged(evaluator): + rows = [row("cheap-model", 1.0, 3.0)] + rows[0]["router_usage"] = [ + {"model": "pricey-model", "input_tokens": 100_000, "output_tokens": 0} + ] + reprice.reprice_rows(rows, evaluator) + assert rows[0]["router_cost"] == pytest.approx(1.0) + assert rows[0]["cost"] == pytest.approx(4.0) + + +@pytest.mark.parametrize( + "style", + [ + dict(indent=2, ensure_ascii=False), + dict(indent=4, ensure_ascii=True), + dict(indent=None, separators=(", ", ": ")), + dict(indent=None, separators=(",", ":")), + ], +) +@pytest.mark.parametrize("newline", ["", "\n"]) +def test_dump_keeps_the_file_style(style, newline): + data = [{"a": "é", "cost": 1.5}, {"a": "b", "cost": 2.0}] + original = json.dumps(data, **style) + newline + data[0]["cost"] = 0.25 + assert reprice.dump_like(original, data) == json.dumps(data, **style) + newline + + +README = """intro +| Rank | Router | Affiliation | Arena | Accuracy | Cost/1K | Opt Sel | Opt Cost | Opt Acc | Latency | Robustness | +|------|---|---|---|---|---|---|---|---|---|---| +| 🥇 | [Alpha]() | a | 77.00 | 80.00 | $0.50 | 10.00 | 20.00 | 90.00 | — | 70.00 | +| 🥈 | [Beta Router](https://x) | b | 76.00 | 79.00 | $0.40 | — | — | — | — | 60.00 | +| 🥉 | [Gamma]() | c | 75.00 | 78.00 | $0.30 | — | — | — | — | 50.00 | + +after""" + + +def test_readme_rows_update_and_rerank(): + metrics = { + "Alpha": { + "arena_score": 0.7412, + "accuracy": 0.80, + "avg_cost_per_1000": 2.5, + "optimality": {"opt_sel": 0.5, "opt_cost": 0.25, "opt_acc": 0.9}, + }, + "Beta Router": { + "arena_score": 0.7700, + "accuracy": 0.79, + "avg_cost_per_1000": 0.3, + }, + } + text, updated = reprice.update_readme(README, metrics) + assert sorted(updated) == ["Alpha", "Beta Router"] + rows = [ + line + for line in text.split("\n") + if line.startswith("| ") and "Rank" not in line + ] + assert ( + rows[0].startswith("| 🥇 | [Beta Router]") + and "| 77.00 | 79.00 | $0.30 | — |" in rows[0] + ) + assert ( + rows[1].startswith("| 🥈 | [Gamma]()") + and "| 75.00 | 78.00 | $0.30 |" in rows[1] + ) + assert rows[2].startswith("| 🥉 | [Alpha]()") + assert "| 74.12 | 80.00 | $2.50 | 50.00 | 25.00 | 90.00 | — | 70.00 |" in rows[2] + assert text.startswith("intro\n") and text.endswith("\nafter") + + +def test_router_metrics_match_metrics_json(tmp_path, monkeypatch): + rows = [row("cheap-model", 1.0, 3.0), row("pricey-model", 0.0, 30.0)] + rows.append(row("cheap-model", 1.0, 3.0)) + rows[-1]["generated_result"]["success"] = False # abnormal: scored 0, no cost + monkeypatch.chdir(tmp_path) + compute_router_metrics(rows, "test-router") + saved = json.loads((tmp_path / "metrics.json").read_text()) + metrics, details = router_metrics(rows, "test-router") + assert saved == metrics + assert metrics["abnormal_count"] == 1 and details["valid_cost_count"] == 2 + assert metrics["avg_cost_per_1000"] == pytest.approx(33.0 / 3 * 1000)