From ec8079798e0b27d1360868c9e88df3fc45c57130 Mon Sep 17 00:00:00 2001 From: Yifan Lu Date: Sat, 31 Oct 2026 16:42:41 -0500 Subject: [PATCH] Suspend R2-Router and AgentForge from the leaderboard pending review Mirrors RouteWorks/RouterArena#260. Deletions only. Co-Authored-By: Claude Opus 5.5 --- .../flip_labels/flip_labels_agentforge.json | 1682 --------- .../flip_labels/flip_labels_r2-router.json | 1682 --------- src/data/routerMetrics/category_scores.json | 3340 ++++------------- src/data/routerMetrics/leaderboard.json | 22 - src/data/routers.json | 36 - 5 files changed, 738 insertions(+), 6024 deletions(-) delete mode 100644 src/data/flip_labels/flip_labels_agentforge.json delete mode 100644 src/data/flip_labels/flip_labels_r2-router.json diff --git a/src/data/flip_labels/flip_labels_agentforge.json b/src/data/flip_labels/flip_labels_agentforge.json deleted file mode 100644 index 2bc24b5..0000000 --- a/src/data/flip_labels/flip_labels_agentforge.json +++ /dev/null @@ -1,1682 +0,0 @@ -[ - { - "global index": "AIME_112", - "flip": 0 - }, - { - "global index": "AIME_58", - "flip": 0 - }, - { - "global index": "ArcMMLU_12", - "flip": 0 - }, - { - "global index": "ArcMMLU_123", - "flip": 0 - }, - { - "global index": "ArcMMLU_16", - "flip": 1 - }, - { - "global index": "ArcMMLU_182", - "flip": 0 - }, - { - "global index": "ArcMMLU_230", - "flip": 0 - }, - { - "global index": "ArcMMLU_293", - "flip": 1 - }, - { - "global index": "ArcMMLU_349", - "flip": 0 - }, - { - "global index": "ArcMMLU_378", - "flip": 0 - }, - { - "global index": "ArcMMLU_443", - "flip": 1 - }, - { - "global index": "ArcMMLU_496", - "flip": 0 - }, - { - "global index": "ArcMMLU_631", - "flip": 1 - }, - { - "global index": "ArcMMLU_646", - "flip": 1 - }, - { - "global index": "ArcMMLU_659", - "flip": 0 - }, - { - "global index": "ArcMMLU_676", - "flip": 1 - }, - { - "global index": "ArcMMLU_685", - "flip": 1 - }, - { - "global index": "ArcMMLU_689", - "flip": 1 - }, - { - "global index": "ArcMMLU_702", - "flip": 1 - }, - { - "global index": "ArcMMLU_713", - "flip": 1 - }, - { - "global index": "ArcMMLU_98", - "flip": 0 - }, - { - "global index": "AsDiv_1165", - "flip": 0 - }, - { - "global index": "AsDiv_1347", - "flip": 1 - }, - { - "global index": "AsDiv_472", - "flip": 1 - }, - { - "global index": "AsDiv_733", - "flip": 1 - }, - { - "global index": "ChessInstruct_0", - "flip": 0 - }, - { - "global index": "ChessInstruct_107", - "flip": 1 - }, - { - "global index": "ChessInstruct_144", - "flip": 1 - }, - { - "global index": "ChessInstruct_42", - "flip": 1 - }, - { - "global index": "ChessInstruct_58", - "flip": 1 - }, - { - "global index": "ChessInstruct_71", - "flip": 1 - }, - { - "global index": "ChessInstruct_84", - "flip": 1 - }, - { - "global index": "Ethics_commonsense_28", - "flip": 0 - }, - { - "global index": "Ethics_commonsense_51", - "flip": 0 - }, - { - "global index": "Ethics_commonsense_6", - "flip": 1 - }, - { - "global index": "Ethics_commonsense_62", - "flip": 0 - }, - { - "global index": "Ethics_commonsense_70", - "flip": 1 - }, - { - "global index": "Ethics_commonsense_85", - "flip": 0 - }, - { - "global index": "Ethics_commonsense_90", - "flip": 1 - }, - { - "global index": "Ethics_deontology_0", - "flip": 1 - }, - { - "global index": "Ethics_deontology_2", - "flip": 1 - }, - { - "global index": "Ethics_deontology_31", - "flip": 1 - }, - { - "global index": "Ethics_deontology_32", - "flip": 1 - }, - { - "global index": "Ethics_deontology_56", - "flip": 1 - }, - { - "global index": "Ethics_justice_1", - "flip": 1 - }, - { - "global index": "Ethics_justice_45", - "flip": 0 - }, - { - "global index": "Ethics_justice_76", - "flip": 1 - }, - { - "global index": "Ethics_justice_84", - "flip": 0 - }, - { - "global index": "Ethics_virtue_14", - "flip": 1 - }, - { - "global index": "Ethics_virtue_26", - "flip": 1 - }, - { - "global index": "Ethics_virtue_30", - "flip": 1 - }, - { - "global index": "Ethics_virtue_48", - "flip": 0 - }, - { - "global index": "Ethics_virtue_51", - "flip": 0 - }, - { - "global index": "FinQA_149", - "flip": 1 - }, - { - "global index": "FinQA_208", - "flip": 0 - }, - { - "global index": "FinQA_56", - "flip": 0 - }, - { - "global index": "FinQA_60", - "flip": 1 - }, - { - "global index": "GSM8K_43", - "flip": 0 - }, - { - "global index": "GeoBench_1002", - "flip": 0 - }, - { - "global index": "GeoBench_1094", - "flip": 1 - }, - { - "global index": "GeoBench_1102", - "flip": 0 - }, - { - "global index": "GeoBench_1113", - "flip": 1 - }, - { - "global index": "GeoBench_124", - "flip": 1 - }, - { - "global index": "GeoBench_1243", - "flip": 0 - }, - { - "global index": "GeoBench_30", - "flip": 1 - }, - { - "global index": "GeoBench_502", - "flip": 0 - }, - { - "global index": "GeoBench_526", - "flip": 0 - }, - { - "global index": "GeoBench_591", - "flip": 1 - }, - { - "global index": "GeoBench_766", - "flip": 0 - }, - { - "global index": "GeoBench_87", - "flip": 0 - }, - { - "global index": "GeoBench_915", - "flip": 0 - }, - { - "global index": "GeoBench_944", - "flip": 0 - }, - { - "global index": "GeoBench_968", - "flip": 1 - }, - { - "global index": "GeoGraphyData_100k_27", - "flip": 0 - }, - { - "global index": "GeoGraphyData_100k_42", - "flip": 0 - }, - { - "global index": "LiveCodeBench_105", - "flip": 1 - }, - { - "global index": "LiveCodeBench_114", - "flip": 1 - }, - { - "global index": "LiveCodeBench_118", - "flip": 1 - }, - { - "global index": "LiveCodeBench_131", - "flip": 1 - }, - { - "global index": "LiveCodeBench_136", - "flip": 1 - }, - { - "global index": "LiveCodeBench_181", - "flip": 1 - }, - { - "global index": "LiveCodeBench_237", - "flip": 0 - }, - { - "global index": "LiveCodeBench_271", - "flip": 1 - }, - { - "global index": "LiveCodeBench_350", - "flip": 1 - }, - { - "global index": "LiveCodeBench_386", - "flip": 0 - }, - { - "global index": "LiveCodeBench_405", - "flip": 0 - }, - { - "global index": "LiveCodeBench_43", - "flip": 1 - }, - { - "global index": "LiveCodeBench_431", - "flip": 1 - }, - { - "global index": "LiveCodeBench_437", - "flip": 0 - }, - { - "global index": "LiveCodeBench_476", - "flip": 0 - }, - { - "global index": "LiveCodeBench_485", - "flip": 1 - }, - { - "global index": "LiveCodeBench_49", - "flip": 1 - }, - { - "global index": "LiveCodeBench_491", - "flip": 1 - }, - { - "global index": "LiveCodeBench_499", - "flip": 1 - }, - { - "global index": "MATH_108", - "flip": 1 - }, - { - "global index": "MATH_442", - "flip": 0 - }, - { - "global index": "MATH_53", - "flip": 0 - }, - { - "global index": "MMLUPro_biology_2808", - "flip": 1 - }, - { - "global index": "MMLUPro_biology_2912", - "flip": 1 - }, - { - "global index": "MMLUPro_biology_2980", - "flip": 0 - }, - { - "global index": "MMLUPro_biology_2985", - "flip": 0 - }, - { - "global index": "MMLUPro_biology_3188", - "flip": 0 - }, - { - "global index": "MMLUPro_biology_3215", - "flip": 1 - }, - { - "global index": "MMLUPro_biology_3225", - "flip": 0 - }, - { - "global index": "MMLUPro_business_226", - "flip": 1 - }, - { - "global index": "MMLUPro_business_294", - "flip": 1 - }, - { - "global index": "MMLUPro_business_378", - "flip": 0 - }, - { - "global index": "MMLUPro_business_430", - "flip": 1 - }, - { - "global index": "MMLUPro_business_503", - "flip": 1 - }, - { - "global index": "MMLUPro_business_507", - "flip": 0 - }, - { - "global index": "MMLUPro_business_6", - "flip": 1 - }, - { - "global index": "MMLUPro_business_784", - "flip": 1 - }, - { - "global index": "MMLUPro_chemistry_3796", - "flip": 1 - }, - { - "global index": "MMLUPro_chemistry_3837", - "flip": 1 - }, - { - "global index": "MMLUPro_chemistry_3974", - "flip": 1 - }, - { - "global index": "MMLUPro_chemistry_4067", - "flip": 0 - }, - { - "global index": "MMLUPro_chemistry_4407", - "flip": 0 - }, - { - "global index": "MMLUPro_computer science_9086", - "flip": 0 - }, - { - "global index": "MMLUPro_computer science_9110", - "flip": 0 - }, - { - "global index": "MMLUPro_computer science_9136", - "flip": 1 - }, - { - "global index": "MMLUPro_computer science_9138", - "flip": 1 - }, - { - "global index": "MMLUPro_computer science_9149", - "flip": 1 - }, - { - "global index": "MMLUPro_computer science_9200", - "flip": 1 - }, - { - "global index": "MMLUPro_computer science_9212", - "flip": 1 - }, - { - "global index": "MMLUPro_computer science_9239", - "flip": 1 - }, - { - "global index": "MMLUPro_computer science_9264", - "flip": 1 - }, - { - "global index": "MMLUPro_computer science_9285", - "flip": 0 - }, - { - "global index": "MMLUPro_computer science_9289", - "flip": 1 - }, - { - "global index": "MMLUPro_computer science_9414", - "flip": 0 - }, - { - "global index": "MMLUPro_computer science_9415", - "flip": 1 - }, - { - "global index": "MMLUPro_computer science_9430", - "flip": 1 - }, - { - "global index": "MMLUPro_computer science_9452", - "flip": 0 - }, - { - "global index": "MMLUPro_computer science_9471", - "flip": 0 - }, - { - "global index": "MMLUPro_computer science_9475", - "flip": 0 - }, - { - "global index": "MMLUPro_economics_5769", - "flip": 1 - }, - { - "global index": "MMLUPro_economics_5907", - "flip": 0 - }, - { - "global index": "MMLUPro_economics_5931", - "flip": 1 - }, - { - "global index": "MMLUPro_economics_5965", - "flip": 1 - }, - { - "global index": "MMLUPro_economics_6114", - "flip": 1 - }, - { - "global index": "MMLUPro_economics_6122", - "flip": 1 - }, - { - "global index": "MMLUPro_economics_6135", - "flip": 1 - }, - { - "global index": "MMLUPro_economics_6325", - "flip": 1 - }, - { - "global index": "MMLUPro_economics_6353", - "flip": 1 - }, - { - "global index": "MMLUPro_engineering_10076", - "flip": 1 - }, - { - "global index": "MMLUPro_engineering_10125", - "flip": 1 - }, - { - "global index": "MMLUPro_engineering_10179", - "flip": 1 - }, - { - "global index": "MMLUPro_engineering_10195", - "flip": 0 - }, - { - "global index": "MMLUPro_engineering_10199", - "flip": 1 - }, - { - "global index": "MMLUPro_engineering_10298", - "flip": 1 - }, - { - "global index": "MMLUPro_engineering_10342", - "flip": 1 - }, - { - "global index": "MMLUPro_engineering_10395", - "flip": 0 - }, - { - "global index": "MMLUPro_engineering_10428", - "flip": 0 - }, - { - "global index": "MMLUPro_engineering_10432", - "flip": 0 - }, - { - "global index": "MMLUPro_engineering_10473", - "flip": 0 - }, - { - "global index": "MMLUPro_engineering_10537", - "flip": 0 - }, - { - "global index": "MMLUPro_engineering_10701", - "flip": 1 - }, - { - "global index": "MMLUPro_engineering_10823", - "flip": 0 - }, - { - "global index": "MMLUPro_engineering_10864", - "flip": 1 - }, - { - "global index": "MMLUPro_health_4885", - "flip": 1 - }, - { - "global index": "MMLUPro_health_4973", - "flip": 0 - }, - { - "global index": "MMLUPro_health_5093", - "flip": 1 - }, - { - "global index": "MMLUPro_health_5144", - "flip": 1 - }, - { - "global index": "MMLUPro_health_5214", - "flip": 1 - }, - { - "global index": "MMLUPro_health_5215", - "flip": 1 - }, - { - "global index": "MMLUPro_health_5261", - "flip": 1 - }, - { - "global index": "MMLUPro_health_5473", - "flip": 1 - }, - { - "global index": "MMLUPro_health_5514", - "flip": 1 - }, - { - "global index": "MMLUPro_history_4486", - "flip": 0 - }, - { - "global index": "MMLUPro_history_4490", - "flip": 1 - }, - { - "global index": "MMLUPro_history_4497", - "flip": 0 - }, - { - "global index": "MMLUPro_history_4509", - "flip": 1 - }, - { - "global index": "MMLUPro_history_4517", - "flip": 1 - }, - { - "global index": "MMLUPro_history_4523", - "flip": 1 - }, - { - "global index": "MMLUPro_history_4605", - "flip": 1 - }, - { - "global index": "MMLUPro_history_4629", - "flip": 0 - }, - { - "global index": "MMLUPro_history_4638", - "flip": 1 - }, - { - "global index": "MMLUPro_history_4717", - "flip": 1 - }, - { - "global index": "MMLUPro_history_4749", - "flip": 0 - }, - { - "global index": "MMLUPro_history_4752", - "flip": 1 - }, - { - "global index": "MMLUPro_history_4774", - "flip": 1 - }, - { - "global index": "MMLUPro_history_4810", - "flip": 0 - }, - { - "global index": "MMLUPro_history_4833", - "flip": 1 - }, - { - "global index": "MMLUPro_history_4836", - "flip": 0 - }, - { - "global index": "MMLUPro_history_4841", - "flip": 0 - }, - { - "global index": "MMLUPro_law_1007", - "flip": 0 - }, - { - "global index": "MMLUPro_law_1031", - "flip": 0 - }, - { - "global index": "MMLUPro_law_1386", - "flip": 1 - }, - { - "global index": "MMLUPro_law_1462", - "flip": 1 - }, - { - "global index": "MMLUPro_law_1484", - "flip": 1 - }, - { - "global index": "MMLUPro_law_1518", - "flip": 0 - }, - { - "global index": "MMLUPro_law_1818", - "flip": 0 - }, - { - "global index": "MMLUPro_law_806", - "flip": 1 - }, - { - "global index": "MMLUPro_law_899", - "flip": 0 - }, - { - "global index": "MMLUPro_math_6429", - "flip": 1 - }, - { - "global index": "MMLUPro_math_6526", - "flip": 1 - }, - { - "global index": "MMLUPro_math_6623", - "flip": 0 - }, - { - "global index": "MMLUPro_math_6848", - "flip": 1 - }, - { - "global index": "MMLUPro_math_7101", - "flip": 1 - }, - { - "global index": "MMLUPro_math_7249", - "flip": 1 - }, - { - "global index": "MMLUPro_math_7284", - "flip": 1 - }, - { - "global index": "MMLUPro_math_7451", - "flip": 0 - }, - { - "global index": "MMLUPro_math_7577", - "flip": 0 - }, - { - "global index": "MMLUPro_philosophy_9510", - "flip": 1 - }, - { - "global index": "MMLUPro_philosophy_9536", - "flip": 0 - }, - { - "global index": "MMLUPro_philosophy_9663", - "flip": 0 - }, - { - "global index": "MMLUPro_philosophy_9672", - "flip": 1 - }, - { - "global index": "MMLUPro_philosophy_9943", - "flip": 1 - }, - { - "global index": "MMLUPro_physics_7773", - "flip": 1 - }, - { - "global index": "MMLUPro_physics_7887", - "flip": 1 - }, - { - "global index": "MMLUPro_physics_7893", - "flip": 1 - }, - { - "global index": "MMLUPro_physics_8888", - "flip": 1 - }, - { - "global index": "MMLUPro_physics_9017", - "flip": 1 - }, - { - "global index": "MMLUPro_psychology_2005", - "flip": 1 - }, - { - "global index": "MMLUPro_psychology_2186", - "flip": 0 - }, - { - "global index": "MMLUPro_psychology_2329", - "flip": 1 - }, - { - "global index": "MMLUPro_psychology_2406", - "flip": 1 - }, - { - "global index": "MMLUPro_psychology_2420", - "flip": 1 - }, - { - "global index": "MMLUPro_psychology_2450", - "flip": 1 - }, - { - "global index": "MMLUPro_psychology_2457", - "flip": 0 - }, - { - "global index": "MMLUPro_psychology_2524", - "flip": 1 - }, - { - "global index": "MMLU_formal_logic_121", - "flip": 1 - }, - { - "global index": "MMLU_formal_logic_16", - "flip": 1 - }, - { - "global index": "MMLU_formal_logic_32", - "flip": 1 - }, - { - "global index": "MMLU_formal_logic_63", - "flip": 1 - }, - { - "global index": "MMLU_formal_logic_7", - "flip": 1 - }, - { - "global index": "MMLU_formal_logic_70", - "flip": 1 - }, - { - "global index": "MMLU_formal_logic_85", - "flip": 0 - }, - { - "global index": "MMLU_management_4", - "flip": 0 - }, - { - "global index": "MMLU_management_41", - "flip": 1 - }, - { - "global index": "MMLU_management_77", - "flip": 0 - }, - { - "global index": "MMLU_management_91", - "flip": 1 - }, - { - "global index": "MMLU_management_93", - "flip": 1 - }, - { - "global index": "MathQA_158", - "flip": 1 - }, - { - "global index": "MathQA_1742", - "flip": 1 - }, - { - "global index": "MathQA_202", - "flip": 1 - }, - { - "global index": "MathQA_2092", - "flip": 1 - }, - { - "global index": "MathQA_2102", - "flip": 0 - }, - { - "global index": "MathQA_2851", - "flip": 1 - }, - { - "global index": "MathQA_827", - "flip": 0 - }, - { - "global index": "MathQA_84", - "flip": 1 - }, - { - "global index": "MedMCQA_1005", - "flip": 1 - }, - { - "global index": "MedMCQA_1054", - "flip": 0 - }, - { - "global index": "MedMCQA_1298", - "flip": 1 - }, - { - "global index": "MedMCQA_1309", - "flip": 1 - }, - { - "global index": "MedMCQA_1362", - "flip": 1 - }, - { - "global index": "MedMCQA_145", - "flip": 1 - }, - { - "global index": "MedMCQA_2010", - "flip": 0 - }, - { - "global index": "MedMCQA_2323", - "flip": 1 - }, - { - "global index": "MedMCQA_2366", - "flip": 0 - }, - { - "global index": "MedMCQA_2581", - "flip": 1 - }, - { - "global index": "MedMCQA_511", - "flip": 1 - }, - { - "global index": "MedMCQA_59", - "flip": 1 - }, - { - "global index": "MedMCQA_643", - "flip": 1 - }, - { - "global index": "MedMCQA_853", - "flip": 1 - }, - { - "global index": "MusicTheoryBench_126", - "flip": 1 - }, - { - "global index": "MusicTheoryBench_14", - "flip": 0 - }, - { - "global index": "MusicTheoryBench_147", - "flip": 1 - }, - { - "global index": "MusicTheoryBench_152", - "flip": 1 - }, - { - "global index": "MusicTheoryBench_188", - "flip": 0 - }, - { - "global index": "MusicTheoryBench_189", - "flip": 0 - }, - { - "global index": "MusicTheoryBench_240", - "flip": 1 - }, - { - "global index": "MusicTheoryBench_33", - "flip": 0 - }, - { - "global index": "MusicTheoryBench_337", - "flip": 1 - }, - { - "global index": "MusicTheoryBench_340", - "flip": 0 - }, - { - "global index": "MusicTheoryBench_70", - "flip": 0 - }, - { - "global index": "NarrativeQA_131", - "flip": 1 - }, - { - "global index": "NarrativeQA_1683", - "flip": 1 - }, - { - "global index": "NarrativeQA_2474", - "flip": 0 - }, - { - "global index": "NarrativeQA_2820", - "flip": 1 - }, - { - "global index": "NarrativeQA_3282", - "flip": 1 - }, - { - "global index": "NarrativeQA_4102", - "flip": 0 - }, - { - "global index": "NarrativeQA_4128", - "flip": 1 - }, - { - "global index": "NarrativeQA_4347", - "flip": 1 - }, - { - "global index": "NarrativeQA_4540", - "flip": 1 - }, - { - "global index": "NarrativeQA_5022", - "flip": 1 - }, - { - "global index": "NarrativeQA_5259", - "flip": 1 - }, - { - "global index": "NarrativeQA_533", - "flip": 1 - }, - { - "global index": "NarrativeQA_5894", - "flip": 1 - }, - { - "global index": "NarrativeQA_6829", - "flip": 1 - }, - { - "global index": "NarrativeQA_7678", - "flip": 1 - }, - { - "global index": "NarrativeQA_7964", - "flip": 1 - }, - { - "global index": "NarrativeQA_8215", - "flip": 1 - }, - { - "global index": "NarrativeQA_8598", - "flip": 0 - }, - { - "global index": "NarrativeQA_927", - "flip": 1 - }, - { - "global index": "OpenTDB_Animals_2545", - "flip": 0 - }, - { - "global index": "OpenTDB_Animals_262", - "flip": 1 - }, - { - "global index": "OpenTDB_Art_1429", - "flip": 1 - }, - { - "global index": "OpenTDB_Celebrities_1078", - "flip": 0 - }, - { - "global index": "OpenTDB_Celebrities_1456", - "flip": 0 - }, - { - "global index": "OpenTDB_Celebrities_3577", - "flip": 1 - }, - { - "global index": "OpenTDB_Celebrities_3968", - "flip": 1 - }, - { - "global index": "OpenTDB_General Knowledge_1178", - "flip": 1 - }, - { - "global index": "OpenTDB_General Knowledge_1807", - "flip": 0 - }, - { - "global index": "OpenTDB_General Knowledge_1957", - "flip": 1 - }, - { - "global index": "OpenTDB_General Knowledge_2181", - "flip": 1 - }, - { - "global index": "OpenTDB_General Knowledge_3404", - "flip": 0 - }, - { - "global index": "OpenTDB_General Knowledge_3727", - "flip": 1 - }, - { - "global index": "OpenTDB_General Knowledge_4019", - "flip": 1 - }, - { - "global index": "OpenTDB_Geography_2099", - "flip": 0 - }, - { - "global index": "OpenTDB_Geography_2346", - "flip": 1 - }, - { - "global index": "OpenTDB_Geography_3880", - "flip": 0 - }, - { - "global index": "OpenTDB_Geography_792", - "flip": 0 - }, - { - "global index": "OpenTDB_History_1162", - "flip": 0 - }, - { - "global index": "OpenTDB_History_2026", - "flip": 0 - }, - { - "global index": "OpenTDB_History_3712", - "flip": 0 - }, - { - "global index": "OpenTDB_History_3902", - "flip": 0 - }, - { - "global index": "OpenTDB_Science & Nature_1175", - "flip": 1 - }, - { - "global index": "OpenTDB_Science & Nature_1560", - "flip": 1 - }, - { - "global index": "OpenTDB_Science & Nature_3716", - "flip": 0 - }, - { - "global index": "OpenTDB_Science & Nature_476", - "flip": 1 - }, - { - "global index": "OpenTDB_Sports_2289", - "flip": 1 - }, - { - "global index": "OpenTDB_Vehicles_1173", - "flip": 0 - }, - { - "global index": "OpenTDB_Vehicles_1419", - "flip": 0 - }, - { - "global index": "OpenTDB_Vehicles_2519", - "flip": 1 - }, - { - "global index": "OpenTDB_Vehicles_3234", - "flip": 0 - }, - { - "global index": "PubMedQA_0", - "flip": 0 - }, - { - "global index": "PubMedQA_154", - "flip": 1 - }, - { - "global index": "PubMedQA_18", - "flip": 0 - }, - { - "global index": "PubMedQA_238", - "flip": 0 - }, - { - "global index": "PubMedQA_250", - "flip": 1 - }, - { - "global index": "PubMedQA_337", - "flip": 0 - }, - { - "global index": "PubMedQA_362", - "flip": 0 - }, - { - "global index": "PubMedQA_437", - "flip": 0 - }, - { - "global index": "PubMedQA_510", - "flip": 0 - }, - { - "global index": "PubMedQA_520", - "flip": 0 - }, - { - "global index": "PubMedQA_575", - "flip": 1 - }, - { - "global index": "PubMedQA_582", - "flip": 0 - }, - { - "global index": "PubMedQA_588", - "flip": 0 - }, - { - "global index": "PubMedQA_610", - "flip": 1 - }, - { - "global index": "PubMedQA_63", - "flip": 0 - }, - { - "global index": "PubMedQA_643", - "flip": 0 - }, - { - "global index": "PubMedQA_687", - "flip": 1 - }, - { - "global index": "PubMedQA_722", - "flip": 1 - }, - { - "global index": "PubMedQA_73", - "flip": 0 - }, - { - "global index": "PubMedQA_755", - "flip": 1 - }, - { - "global index": "PubMedQA_8", - "flip": 1 - }, - { - "global index": "PubMedQA_81", - "flip": 0 - }, - { - "global index": "PubMedQA_854", - "flip": 0 - }, - { - "global index": "PubMedQA_905", - "flip": 1 - }, - { - "global index": "QANTA_Fine Arts_1212", - "flip": 1 - }, - { - "global index": "QANTA_Fine Arts_1702", - "flip": 0 - }, - { - "global index": "QANTA_Fine Arts_828", - "flip": 1 - }, - { - "global index": "QANTA_Fine Arts_865", - "flip": 0 - }, - { - "global index": "QANTA_Geography_1023", - "flip": 0 - }, - { - "global index": "QANTA_Geography_1555", - "flip": 0 - }, - { - "global index": "QANTA_Geography_304", - "flip": 0 - }, - { - "global index": "QANTA_History_1084", - "flip": 0 - }, - { - "global index": "QANTA_History_1154", - "flip": 1 - }, - { - "global index": "QANTA_History_433", - "flip": 0 - }, - { - "global index": "QANTA_History_473", - "flip": 0 - }, - { - "global index": "QANTA_History_926", - "flip": 1 - }, - { - "global index": "QANTA_Literature_1045", - "flip": 0 - }, - { - "global index": "QANTA_Literature_1073", - "flip": 1 - }, - { - "global index": "QANTA_Literature_1239", - "flip": 0 - }, - { - "global index": "QANTA_Literature_1326", - "flip": 1 - }, - { - "global index": "QANTA_Literature_1727", - "flip": 1 - }, - { - "global index": "QANTA_Literature_1843", - "flip": 1 - }, - { - "global index": "QANTA_Literature_386", - "flip": 0 - }, - { - "global index": "QANTA_Literature_408", - "flip": 1 - }, - { - "global index": "QANTA_Literature_475", - "flip": 0 - }, - { - "global index": "QANTA_Literature_833", - "flip": 1 - }, - { - "global index": "QANTA_Philosophy_1270", - "flip": 1 - }, - { - "global index": "QANTA_Philosophy_499", - "flip": 0 - }, - { - "global index": "QANTA_Philosophy_91", - "flip": 1 - }, - { - "global index": "QANTA_Science_1360", - "flip": 0 - }, - { - "global index": "QANTA_Science_1473", - "flip": 0 - }, - { - "global index": "QANTA_Science_308", - "flip": 0 - }, - { - "global index": "QANTA_Science_619", - "flip": 1 - }, - { - "global index": "QANTA_Social Science_1847", - "flip": 0 - }, - { - "global index": "QANTA_Social Science_2", - "flip": 1 - }, - { - "global index": "QANTA_Social Science_77", - "flip": 0 - }, - { - "global index": "SocialiQA_13810", - "flip": 0 - }, - { - "global index": "SocialiQA_22095", - "flip": 1 - }, - { - "global index": "SocialiQA_26846", - "flip": 1 - }, - { - "global index": "SocialiQA_7839", - "flip": 1 - }, - { - "global index": "SuperGLUE-CausalReasoning_4526", - "flip": 1 - }, - { - "global index": "SuperGLUE-ClozeTest_12894", - "flip": 0 - }, - { - "global index": "SuperGLUE-ClozeTest_17965", - "flip": 0 - }, - { - "global index": "SuperGLUE-ClozeTest_18766", - "flip": 1 - }, - { - "global index": "SuperGLUE-Entailment_19410", - "flip": 1 - }, - { - "global index": "SuperGLUE-Entailment_19567", - "flip": 1 - }, - { - "global index": "SuperGLUE-Entailment_522", - "flip": 1 - }, - { - "global index": "SuperGLUE-Entailment_767", - "flip": 0 - }, - { - "global index": "SuperGLUE-QA_1408", - "flip": 1 - }, - { - "global index": "SuperGLUE-QA_3137", - "flip": 0 - }, - { - "global index": "SuperGLUE-QA_4046", - "flip": 1 - }, - { - "global index": "SuperGLUE-QA_4102", - "flip": 0 - }, - { - "global index": "SuperGLUE-QA_4160", - "flip": 0 - }, - { - "global index": "SuperGLUE-RC_7725", - "flip": 0 - }, - { - "global index": "SuperGLUE-RC_7738", - "flip": 0 - }, - { - "global index": "SuperGLUE-RC_8531", - "flip": 0 - }, - { - "global index": "SuperGLUE-Wic_19695", - "flip": 0 - }, - { - "global index": "SuperGLUE-Wic_19738", - "flip": 1 - }, - { - "global index": "SuperGLUE-Wic_20079", - "flip": 1 - }, - { - "global index": "SuperGLUE-Wic_20189", - "flip": 0 - }, - { - "global index": "SuperGLUE-Wic_20253", - "flip": 1 - }, - { - "global index": "SuperGLUE-Wsc_20368", - "flip": 1 - }, - { - "global index": "SuperGLUE-Wsc_20370", - "flip": 0 - }, - { - "global index": "WMT19-cs-en_156", - "flip": 0 - }, - { - "global index": "WMT19-cs-en_246", - "flip": 1 - }, - { - "global index": "WMT19-cs-en_568", - "flip": 0 - }, - { - "global index": "WMT19-de-en_46", - "flip": 0 - }, - { - "global index": "WMT19-de-en_715", - "flip": 1 - }, - { - "global index": "WMT19-de-en_883", - "flip": 1 - }, - { - "global index": "WMT19-fi-en_222", - "flip": 1 - }, - { - "global index": "WMT19-fi-en_610", - "flip": 1 - }, - { - "global index": "WMT19-gu-en_116", - "flip": 0 - }, - { - "global index": "WMT19-gu-en_123", - "flip": 0 - }, - { - "global index": "WMT19-gu-en_191", - "flip": 1 - }, - { - "global index": "WMT19-gu-en_491", - "flip": 1 - }, - { - "global index": "WMT19-gu-en_968", - "flip": 1 - }, - { - "global index": "WMT19-kk-en_528", - "flip": 1 - }, - { - "global index": "WMT19-kk-en_826", - "flip": 1 - }, - { - "global index": "WMT19-lt-en_135", - "flip": 1 - }, - { - "global index": "WMT19-lt-en_269", - "flip": 0 - }, - { - "global index": "WMT19-lt-en_636", - "flip": 1 - }, - { - "global index": "WMT19-ru-en_222", - "flip": 1 - }, - { - "global index": "WMT19-zh-en_218", - "flip": 1 - }, - { - "global index": "WMT19-zh-en_252", - "flip": 1 - }, - { - "global index": "WMT19-zh-en_59", - "flip": 1 - } -] diff --git a/src/data/flip_labels/flip_labels_r2-router.json b/src/data/flip_labels/flip_labels_r2-router.json deleted file mode 100644 index 41fee9c..0000000 --- a/src/data/flip_labels/flip_labels_r2-router.json +++ /dev/null @@ -1,1682 +0,0 @@ -[ - { - "global index": "AIME_112", - "flip": 0 - }, - { - "global index": "AIME_58", - "flip": 0 - }, - { - "global index": "ArcMMLU_12", - "flip": 1 - }, - { - "global index": "ArcMMLU_123", - "flip": 1 - }, - { - "global index": "ArcMMLU_16", - "flip": 1 - }, - { - "global index": "ArcMMLU_182", - "flip": 1 - }, - { - "global index": "ArcMMLU_230", - "flip": 1 - }, - { - "global index": "ArcMMLU_293", - "flip": 1 - }, - { - "global index": "ArcMMLU_349", - "flip": 0 - }, - { - "global index": "ArcMMLU_378", - "flip": 0 - }, - { - "global index": "ArcMMLU_443", - "flip": 1 - }, - { - "global index": "ArcMMLU_496", - "flip": 1 - }, - { - "global index": "ArcMMLU_631", - "flip": 1 - }, - { - "global index": "ArcMMLU_646", - "flip": 1 - }, - { - "global index": "ArcMMLU_659", - "flip": 0 - }, - { - "global index": "ArcMMLU_676", - "flip": 1 - }, - { - "global index": "ArcMMLU_685", - "flip": 0 - }, - { - "global index": "ArcMMLU_689", - "flip": 1 - }, - { - "global index": "ArcMMLU_702", - "flip": 0 - }, - { - "global index": "ArcMMLU_713", - "flip": 1 - }, - { - "global index": "ArcMMLU_98", - "flip": 0 - }, - { - "global index": "AsDiv_1165", - "flip": 0 - }, - { - "global index": "AsDiv_1347", - "flip": 0 - }, - { - "global index": "AsDiv_472", - "flip": 0 - }, - { - "global index": "AsDiv_733", - "flip": 0 - }, - { - "global index": "ChessInstruct_0", - "flip": 0 - }, - { - "global index": "ChessInstruct_107", - "flip": 1 - }, - { - "global index": "ChessInstruct_144", - "flip": 1 - }, - { - "global index": "ChessInstruct_42", - "flip": 1 - }, - { - "global index": "ChessInstruct_58", - "flip": 0 - }, - { - "global index": "ChessInstruct_71", - "flip": 0 - }, - { - "global index": "ChessInstruct_84", - "flip": 0 - }, - { - "global index": "Ethics_commonsense_28", - "flip": 1 - }, - { - "global index": "Ethics_commonsense_51", - "flip": 0 - }, - { - "global index": "Ethics_commonsense_6", - "flip": 1 - }, - { - "global index": "Ethics_commonsense_62", - "flip": 0 - }, - { - "global index": "Ethics_commonsense_70", - "flip": 1 - }, - { - "global index": "Ethics_commonsense_85", - "flip": 1 - }, - { - "global index": "Ethics_commonsense_90", - "flip": 0 - }, - { - "global index": "Ethics_deontology_0", - "flip": 1 - }, - { - "global index": "Ethics_deontology_2", - "flip": 0 - }, - { - "global index": "Ethics_deontology_31", - "flip": 1 - }, - { - "global index": "Ethics_deontology_32", - "flip": 1 - }, - { - "global index": "Ethics_deontology_56", - "flip": 1 - }, - { - "global index": "Ethics_justice_1", - "flip": 1 - }, - { - "global index": "Ethics_justice_45", - "flip": 0 - }, - { - "global index": "Ethics_justice_76", - "flip": 0 - }, - { - "global index": "Ethics_justice_84", - "flip": 1 - }, - { - "global index": "Ethics_virtue_14", - "flip": 1 - }, - { - "global index": "Ethics_virtue_26", - "flip": 1 - }, - { - "global index": "Ethics_virtue_30", - "flip": 1 - }, - { - "global index": "Ethics_virtue_48", - "flip": 1 - }, - { - "global index": "Ethics_virtue_51", - "flip": 1 - }, - { - "global index": "FinQA_149", - "flip": 0 - }, - { - "global index": "FinQA_208", - "flip": 0 - }, - { - "global index": "FinQA_56", - "flip": 0 - }, - { - "global index": "FinQA_60", - "flip": 0 - }, - { - "global index": "GSM8K_43", - "flip": 0 - }, - { - "global index": "GeoBench_1002", - "flip": 1 - }, - { - "global index": "GeoBench_1094", - "flip": 1 - }, - { - "global index": "GeoBench_1102", - "flip": 0 - }, - { - "global index": "GeoBench_1113", - "flip": 0 - }, - { - "global index": "GeoBench_124", - "flip": 0 - }, - { - "global index": "GeoBench_1243", - "flip": 0 - }, - { - "global index": "GeoBench_30", - "flip": 0 - }, - { - "global index": "GeoBench_502", - "flip": 0 - }, - { - "global index": "GeoBench_526", - "flip": 1 - }, - { - "global index": "GeoBench_591", - "flip": 0 - }, - { - "global index": "GeoBench_766", - "flip": 1 - }, - { - "global index": "GeoBench_87", - "flip": 1 - }, - { - "global index": "GeoBench_915", - "flip": 1 - }, - { - "global index": "GeoBench_944", - "flip": 1 - }, - { - "global index": "GeoBench_968", - "flip": 1 - }, - { - "global index": "GeoGraphyData_100k_27", - "flip": 1 - }, - { - "global index": "GeoGraphyData_100k_42", - "flip": 1 - }, - { - "global index": "LiveCodeBench_105", - "flip": 1 - }, - { - "global index": "LiveCodeBench_114", - "flip": 1 - }, - { - "global index": "LiveCodeBench_118", - "flip": 0 - }, - { - "global index": "LiveCodeBench_131", - "flip": 0 - }, - { - "global index": "LiveCodeBench_136", - "flip": 1 - }, - { - "global index": "LiveCodeBench_181", - "flip": 1 - }, - { - "global index": "LiveCodeBench_237", - "flip": 1 - }, - { - "global index": "LiveCodeBench_271", - "flip": 1 - }, - { - "global index": "LiveCodeBench_350", - "flip": 1 - }, - { - "global index": "LiveCodeBench_386", - "flip": 0 - }, - { - "global index": "LiveCodeBench_405", - "flip": 1 - }, - { - "global index": "LiveCodeBench_43", - "flip": 0 - }, - { - "global index": "LiveCodeBench_431", - "flip": 1 - }, - { - "global index": "LiveCodeBench_437", - "flip": 0 - }, - { - "global index": "LiveCodeBench_476", - "flip": 1 - }, - { - "global index": "LiveCodeBench_485", - "flip": 1 - }, - { - "global index": "LiveCodeBench_49", - "flip": 0 - }, - { - "global index": "LiveCodeBench_491", - "flip": 0 - }, - { - "global index": "LiveCodeBench_499", - "flip": 1 - }, - { - "global index": "MATH_108", - "flip": 0 - }, - { - "global index": "MATH_442", - "flip": 0 - }, - { - "global index": "MATH_53", - "flip": 1 - }, - { - "global index": "MMLUPro_biology_2808", - "flip": 0 - }, - { - "global index": "MMLUPro_biology_2912", - "flip": 0 - }, - { - "global index": "MMLUPro_biology_2980", - "flip": 0 - }, - { - "global index": "MMLUPro_biology_2985", - "flip": 0 - }, - { - "global index": "MMLUPro_biology_3188", - "flip": 1 - }, - { - "global index": "MMLUPro_biology_3215", - "flip": 1 - }, - { - "global index": "MMLUPro_biology_3225", - "flip": 0 - }, - { - "global index": "MMLUPro_business_226", - "flip": 0 - }, - { - "global index": "MMLUPro_business_294", - "flip": 0 - }, - { - "global index": "MMLUPro_business_378", - "flip": 1 - }, - { - "global index": "MMLUPro_business_430", - "flip": 0 - }, - { - "global index": "MMLUPro_business_503", - "flip": 0 - }, - { - "global index": "MMLUPro_business_507", - "flip": 0 - }, - { - "global index": "MMLUPro_business_6", - "flip": 1 - }, - { - "global index": "MMLUPro_business_784", - "flip": 0 - }, - { - "global index": "MMLUPro_chemistry_3796", - "flip": 0 - }, - { - "global index": "MMLUPro_chemistry_3837", - "flip": 1 - }, - { - "global index": "MMLUPro_chemistry_3974", - "flip": 1 - }, - { - "global index": "MMLUPro_chemistry_4067", - "flip": 0 - }, - { - "global index": "MMLUPro_chemistry_4407", - "flip": 0 - }, - { - "global index": "MMLUPro_computer science_9086", - "flip": 0 - }, - { - "global index": "MMLUPro_computer science_9110", - "flip": 0 - }, - { - "global index": "MMLUPro_computer science_9136", - "flip": 0 - }, - { - "global index": "MMLUPro_computer science_9138", - "flip": 1 - }, - { - "global index": "MMLUPro_computer science_9149", - "flip": 0 - }, - { - "global index": "MMLUPro_computer science_9200", - "flip": 1 - }, - { - "global index": "MMLUPro_computer science_9212", - "flip": 0 - }, - { - "global index": "MMLUPro_computer science_9239", - "flip": 1 - }, - { - "global index": "MMLUPro_computer science_9264", - "flip": 0 - }, - { - "global index": "MMLUPro_computer science_9285", - "flip": 0 - }, - { - "global index": "MMLUPro_computer science_9289", - "flip": 1 - }, - { - "global index": "MMLUPro_computer science_9414", - "flip": 0 - }, - { - "global index": "MMLUPro_computer science_9415", - "flip": 1 - }, - { - "global index": "MMLUPro_computer science_9430", - "flip": 0 - }, - { - "global index": "MMLUPro_computer science_9452", - "flip": 0 - }, - { - "global index": "MMLUPro_computer science_9471", - "flip": 1 - }, - { - "global index": "MMLUPro_computer science_9475", - "flip": 0 - }, - { - "global index": "MMLUPro_economics_5769", - "flip": 0 - }, - { - "global index": "MMLUPro_economics_5907", - "flip": 0 - }, - { - "global index": "MMLUPro_economics_5931", - "flip": 0 - }, - { - "global index": "MMLUPro_economics_5965", - "flip": 1 - }, - { - "global index": "MMLUPro_economics_6114", - "flip": 1 - }, - { - "global index": "MMLUPro_economics_6122", - "flip": 1 - }, - { - "global index": "MMLUPro_economics_6135", - "flip": 1 - }, - { - "global index": "MMLUPro_economics_6325", - "flip": 0 - }, - { - "global index": "MMLUPro_economics_6353", - "flip": 0 - }, - { - "global index": "MMLUPro_engineering_10076", - "flip": 1 - }, - { - "global index": "MMLUPro_engineering_10125", - "flip": 0 - }, - { - "global index": "MMLUPro_engineering_10179", - "flip": 1 - }, - { - "global index": "MMLUPro_engineering_10195", - "flip": 1 - }, - { - "global index": "MMLUPro_engineering_10199", - "flip": 0 - }, - { - "global index": "MMLUPro_engineering_10298", - "flip": 0 - }, - { - "global index": "MMLUPro_engineering_10342", - "flip": 1 - }, - { - "global index": "MMLUPro_engineering_10395", - "flip": 0 - }, - { - "global index": "MMLUPro_engineering_10428", - "flip": 0 - }, - { - "global index": "MMLUPro_engineering_10432", - "flip": 1 - }, - { - "global index": "MMLUPro_engineering_10473", - "flip": 0 - }, - { - "global index": "MMLUPro_engineering_10537", - "flip": 0 - }, - { - "global index": "MMLUPro_engineering_10701", - "flip": 0 - }, - { - "global index": "MMLUPro_engineering_10823", - "flip": 1 - }, - { - "global index": "MMLUPro_engineering_10864", - "flip": 1 - }, - { - "global index": "MMLUPro_health_4885", - "flip": 0 - }, - { - "global index": "MMLUPro_health_4973", - "flip": 1 - }, - { - "global index": "MMLUPro_health_5093", - "flip": 1 - }, - { - "global index": "MMLUPro_health_5144", - "flip": 0 - }, - { - "global index": "MMLUPro_health_5214", - "flip": 0 - }, - { - "global index": "MMLUPro_health_5215", - "flip": 0 - }, - { - "global index": "MMLUPro_health_5261", - "flip": 1 - }, - { - "global index": "MMLUPro_health_5473", - "flip": 1 - }, - { - "global index": "MMLUPro_health_5514", - "flip": 0 - }, - { - "global index": "MMLUPro_history_4486", - "flip": 1 - }, - { - "global index": "MMLUPro_history_4490", - "flip": 0 - }, - { - "global index": "MMLUPro_history_4497", - "flip": 1 - }, - { - "global index": "MMLUPro_history_4509", - "flip": 0 - }, - { - "global index": "MMLUPro_history_4517", - "flip": 0 - }, - { - "global index": "MMLUPro_history_4523", - "flip": 1 - }, - { - "global index": "MMLUPro_history_4605", - "flip": 1 - }, - { - "global index": "MMLUPro_history_4629", - "flip": 0 - }, - { - "global index": "MMLUPro_history_4638", - "flip": 1 - }, - { - "global index": "MMLUPro_history_4717", - "flip": 1 - }, - { - "global index": "MMLUPro_history_4749", - "flip": 1 - }, - { - "global index": "MMLUPro_history_4752", - "flip": 0 - }, - { - "global index": "MMLUPro_history_4774", - "flip": 0 - }, - { - "global index": "MMLUPro_history_4810", - "flip": 1 - }, - { - "global index": "MMLUPro_history_4833", - "flip": 0 - }, - { - "global index": "MMLUPro_history_4836", - "flip": 1 - }, - { - "global index": "MMLUPro_history_4841", - "flip": 1 - }, - { - "global index": "MMLUPro_law_1007", - "flip": 0 - }, - { - "global index": "MMLUPro_law_1031", - "flip": 1 - }, - { - "global index": "MMLUPro_law_1386", - "flip": 0 - }, - { - "global index": "MMLUPro_law_1462", - "flip": 0 - }, - { - "global index": "MMLUPro_law_1484", - "flip": 1 - }, - { - "global index": "MMLUPro_law_1518", - "flip": 0 - }, - { - "global index": "MMLUPro_law_1818", - "flip": 1 - }, - { - "global index": "MMLUPro_law_806", - "flip": 0 - }, - { - "global index": "MMLUPro_law_899", - "flip": 1 - }, - { - "global index": "MMLUPro_math_6429", - "flip": 0 - }, - { - "global index": "MMLUPro_math_6526", - "flip": 1 - }, - { - "global index": "MMLUPro_math_6623", - "flip": 1 - }, - { - "global index": "MMLUPro_math_6848", - "flip": 1 - }, - { - "global index": "MMLUPro_math_7101", - "flip": 1 - }, - { - "global index": "MMLUPro_math_7249", - "flip": 0 - }, - { - "global index": "MMLUPro_math_7284", - "flip": 0 - }, - { - "global index": "MMLUPro_math_7451", - "flip": 0 - }, - { - "global index": "MMLUPro_math_7577", - "flip": 0 - }, - { - "global index": "MMLUPro_philosophy_9510", - "flip": 1 - }, - { - "global index": "MMLUPro_philosophy_9536", - "flip": 1 - }, - { - "global index": "MMLUPro_philosophy_9663", - "flip": 0 - }, - { - "global index": "MMLUPro_philosophy_9672", - "flip": 1 - }, - { - "global index": "MMLUPro_philosophy_9943", - "flip": 1 - }, - { - "global index": "MMLUPro_physics_7773", - "flip": 1 - }, - { - "global index": "MMLUPro_physics_7887", - "flip": 0 - }, - { - "global index": "MMLUPro_physics_7893", - "flip": 0 - }, - { - "global index": "MMLUPro_physics_8888", - "flip": 0 - }, - { - "global index": "MMLUPro_physics_9017", - "flip": 0 - }, - { - "global index": "MMLUPro_psychology_2005", - "flip": 1 - }, - { - "global index": "MMLUPro_psychology_2186", - "flip": 1 - }, - { - "global index": "MMLUPro_psychology_2329", - "flip": 1 - }, - { - "global index": "MMLUPro_psychology_2406", - "flip": 1 - }, - { - "global index": "MMLUPro_psychology_2420", - "flip": 0 - }, - { - "global index": "MMLUPro_psychology_2450", - "flip": 0 - }, - { - "global index": "MMLUPro_psychology_2457", - "flip": 0 - }, - { - "global index": "MMLUPro_psychology_2524", - "flip": 1 - }, - { - "global index": "MMLU_formal_logic_121", - "flip": 1 - }, - { - "global index": "MMLU_formal_logic_16", - "flip": 0 - }, - { - "global index": "MMLU_formal_logic_32", - "flip": 0 - }, - { - "global index": "MMLU_formal_logic_63", - "flip": 1 - }, - { - "global index": "MMLU_formal_logic_7", - "flip": 0 - }, - { - "global index": "MMLU_formal_logic_70", - "flip": 1 - }, - { - "global index": "MMLU_formal_logic_85", - "flip": 1 - }, - { - "global index": "MMLU_management_4", - "flip": 0 - }, - { - "global index": "MMLU_management_41", - "flip": 0 - }, - { - "global index": "MMLU_management_77", - "flip": 0 - }, - { - "global index": "MMLU_management_91", - "flip": 1 - }, - { - "global index": "MMLU_management_93", - "flip": 1 - }, - { - "global index": "MathQA_158", - "flip": 0 - }, - { - "global index": "MathQA_1742", - "flip": 0 - }, - { - "global index": "MathQA_202", - "flip": 0 - }, - { - "global index": "MathQA_2092", - "flip": 1 - }, - { - "global index": "MathQA_2102", - "flip": 1 - }, - { - "global index": "MathQA_2851", - "flip": 1 - }, - { - "global index": "MathQA_827", - "flip": 0 - }, - { - "global index": "MathQA_84", - "flip": 0 - }, - { - "global index": "MedMCQA_1005", - "flip": 1 - }, - { - "global index": "MedMCQA_1054", - "flip": 0 - }, - { - "global index": "MedMCQA_1298", - "flip": 1 - }, - { - "global index": "MedMCQA_1309", - "flip": 0 - }, - { - "global index": "MedMCQA_1362", - "flip": 0 - }, - { - "global index": "MedMCQA_145", - "flip": 0 - }, - { - "global index": "MedMCQA_2010", - "flip": 1 - }, - { - "global index": "MedMCQA_2323", - "flip": 0 - }, - { - "global index": "MedMCQA_2366", - "flip": 0 - }, - { - "global index": "MedMCQA_2581", - "flip": 0 - }, - { - "global index": "MedMCQA_511", - "flip": 1 - }, - { - "global index": "MedMCQA_59", - "flip": 0 - }, - { - "global index": "MedMCQA_643", - "flip": 1 - }, - { - "global index": "MedMCQA_853", - "flip": 1 - }, - { - "global index": "MusicTheoryBench_126", - "flip": 1 - }, - { - "global index": "MusicTheoryBench_14", - "flip": 1 - }, - { - "global index": "MusicTheoryBench_147", - "flip": 0 - }, - { - "global index": "MusicTheoryBench_152", - "flip": 0 - }, - { - "global index": "MusicTheoryBench_188", - "flip": 0 - }, - { - "global index": "MusicTheoryBench_189", - "flip": 1 - }, - { - "global index": "MusicTheoryBench_240", - "flip": 1 - }, - { - "global index": "MusicTheoryBench_33", - "flip": 1 - }, - { - "global index": "MusicTheoryBench_337", - "flip": 0 - }, - { - "global index": "MusicTheoryBench_340", - "flip": 1 - }, - { - "global index": "MusicTheoryBench_70", - "flip": 0 - }, - { - "global index": "NarrativeQA_131", - "flip": 0 - }, - { - "global index": "NarrativeQA_1683", - "flip": 0 - }, - { - "global index": "NarrativeQA_2474", - "flip": 0 - }, - { - "global index": "NarrativeQA_2820", - "flip": 1 - }, - { - "global index": "NarrativeQA_3282", - "flip": 1 - }, - { - "global index": "NarrativeQA_4102", - "flip": 0 - }, - { - "global index": "NarrativeQA_4128", - "flip": 0 - }, - { - "global index": "NarrativeQA_4347", - "flip": 0 - }, - { - "global index": "NarrativeQA_4540", - "flip": 0 - }, - { - "global index": "NarrativeQA_5022", - "flip": 0 - }, - { - "global index": "NarrativeQA_5259", - "flip": 0 - }, - { - "global index": "NarrativeQA_533", - "flip": 0 - }, - { - "global index": "NarrativeQA_5894", - "flip": 0 - }, - { - "global index": "NarrativeQA_6829", - "flip": 1 - }, - { - "global index": "NarrativeQA_7678", - "flip": 0 - }, - { - "global index": "NarrativeQA_7964", - "flip": 1 - }, - { - "global index": "NarrativeQA_8215", - "flip": 0 - }, - { - "global index": "NarrativeQA_8598", - "flip": 0 - }, - { - "global index": "NarrativeQA_927", - "flip": 1 - }, - { - "global index": "OpenTDB_Animals_2545", - "flip": 1 - }, - { - "global index": "OpenTDB_Animals_262", - "flip": 1 - }, - { - "global index": "OpenTDB_Art_1429", - "flip": 1 - }, - { - "global index": "OpenTDB_Celebrities_1078", - "flip": 1 - }, - { - "global index": "OpenTDB_Celebrities_1456", - "flip": 0 - }, - { - "global index": "OpenTDB_Celebrities_3577", - "flip": 0 - }, - { - "global index": "OpenTDB_Celebrities_3968", - "flip": 0 - }, - { - "global index": "OpenTDB_General Knowledge_1178", - "flip": 0 - }, - { - "global index": "OpenTDB_General Knowledge_1807", - "flip": 1 - }, - { - "global index": "OpenTDB_General Knowledge_1957", - "flip": 1 - }, - { - "global index": "OpenTDB_General Knowledge_2181", - "flip": 0 - }, - { - "global index": "OpenTDB_General Knowledge_3404", - "flip": 1 - }, - { - "global index": "OpenTDB_General Knowledge_3727", - "flip": 0 - }, - { - "global index": "OpenTDB_General Knowledge_4019", - "flip": 0 - }, - { - "global index": "OpenTDB_Geography_2099", - "flip": 0 - }, - { - "global index": "OpenTDB_Geography_2346", - "flip": 1 - }, - { - "global index": "OpenTDB_Geography_3880", - "flip": 0 - }, - { - "global index": "OpenTDB_Geography_792", - "flip": 0 - }, - { - "global index": "OpenTDB_History_1162", - "flip": 1 - }, - { - "global index": "OpenTDB_History_2026", - "flip": 0 - }, - { - "global index": "OpenTDB_History_3712", - "flip": 0 - }, - { - "global index": "OpenTDB_History_3902", - "flip": 1 - }, - { - "global index": "OpenTDB_Science & Nature_1175", - "flip": 1 - }, - { - "global index": "OpenTDB_Science & Nature_1560", - "flip": 1 - }, - { - "global index": "OpenTDB_Science & Nature_3716", - "flip": 0 - }, - { - "global index": "OpenTDB_Science & Nature_476", - "flip": 0 - }, - { - "global index": "OpenTDB_Sports_2289", - "flip": 0 - }, - { - "global index": "OpenTDB_Vehicles_1173", - "flip": 0 - }, - { - "global index": "OpenTDB_Vehicles_1419", - "flip": 1 - }, - { - "global index": "OpenTDB_Vehicles_2519", - "flip": 1 - }, - { - "global index": "OpenTDB_Vehicles_3234", - "flip": 1 - }, - { - "global index": "PubMedQA_0", - "flip": 1 - }, - { - "global index": "PubMedQA_154", - "flip": 1 - }, - { - "global index": "PubMedQA_18", - "flip": 1 - }, - { - "global index": "PubMedQA_238", - "flip": 0 - }, - { - "global index": "PubMedQA_250", - "flip": 1 - }, - { - "global index": "PubMedQA_337", - "flip": 0 - }, - { - "global index": "PubMedQA_362", - "flip": 0 - }, - { - "global index": "PubMedQA_437", - "flip": 0 - }, - { - "global index": "PubMedQA_510", - "flip": 0 - }, - { - "global index": "PubMedQA_520", - "flip": 1 - }, - { - "global index": "PubMedQA_575", - "flip": 0 - }, - { - "global index": "PubMedQA_582", - "flip": 0 - }, - { - "global index": "PubMedQA_588", - "flip": 1 - }, - { - "global index": "PubMedQA_610", - "flip": 0 - }, - { - "global index": "PubMedQA_63", - "flip": 0 - }, - { - "global index": "PubMedQA_643", - "flip": 0 - }, - { - "global index": "PubMedQA_687", - "flip": 0 - }, - { - "global index": "PubMedQA_722", - "flip": 0 - }, - { - "global index": "PubMedQA_73", - "flip": 1 - }, - { - "global index": "PubMedQA_755", - "flip": 0 - }, - { - "global index": "PubMedQA_8", - "flip": 0 - }, - { - "global index": "PubMedQA_81", - "flip": 1 - }, - { - "global index": "PubMedQA_854", - "flip": 0 - }, - { - "global index": "PubMedQA_905", - "flip": 0 - }, - { - "global index": "QANTA_Fine Arts_1212", - "flip": 1 - }, - { - "global index": "QANTA_Fine Arts_1702", - "flip": 1 - }, - { - "global index": "QANTA_Fine Arts_828", - "flip": 0 - }, - { - "global index": "QANTA_Fine Arts_865", - "flip": 0 - }, - { - "global index": "QANTA_Geography_1023", - "flip": 1 - }, - { - "global index": "QANTA_Geography_1555", - "flip": 1 - }, - { - "global index": "QANTA_Geography_304", - "flip": 1 - }, - { - "global index": "QANTA_History_1084", - "flip": 1 - }, - { - "global index": "QANTA_History_1154", - "flip": 0 - }, - { - "global index": "QANTA_History_433", - "flip": 1 - }, - { - "global index": "QANTA_History_473", - "flip": 1 - }, - { - "global index": "QANTA_History_926", - "flip": 1 - }, - { - "global index": "QANTA_Literature_1045", - "flip": 1 - }, - { - "global index": "QANTA_Literature_1073", - "flip": 1 - }, - { - "global index": "QANTA_Literature_1239", - "flip": 1 - }, - { - "global index": "QANTA_Literature_1326", - "flip": 1 - }, - { - "global index": "QANTA_Literature_1727", - "flip": 0 - }, - { - "global index": "QANTA_Literature_1843", - "flip": 1 - }, - { - "global index": "QANTA_Literature_386", - "flip": 1 - }, - { - "global index": "QANTA_Literature_408", - "flip": 1 - }, - { - "global index": "QANTA_Literature_475", - "flip": 1 - }, - { - "global index": "QANTA_Literature_833", - "flip": 1 - }, - { - "global index": "QANTA_Philosophy_1270", - "flip": 1 - }, - { - "global index": "QANTA_Philosophy_499", - "flip": 1 - }, - { - "global index": "QANTA_Philosophy_91", - "flip": 1 - }, - { - "global index": "QANTA_Science_1360", - "flip": 1 - }, - { - "global index": "QANTA_Science_1473", - "flip": 1 - }, - { - "global index": "QANTA_Science_308", - "flip": 1 - }, - { - "global index": "QANTA_Science_619", - "flip": 1 - }, - { - "global index": "QANTA_Social Science_1847", - "flip": 1 - }, - { - "global index": "QANTA_Social Science_2", - "flip": 0 - }, - { - "global index": "QANTA_Social Science_77", - "flip": 0 - }, - { - "global index": "SocialiQA_13810", - "flip": 1 - }, - { - "global index": "SocialiQA_22095", - "flip": 1 - }, - { - "global index": "SocialiQA_26846", - "flip": 1 - }, - { - "global index": "SocialiQA_7839", - "flip": 1 - }, - { - "global index": "SuperGLUE-CausalReasoning_4526", - "flip": 1 - }, - { - "global index": "SuperGLUE-ClozeTest_12894", - "flip": 1 - }, - { - "global index": "SuperGLUE-ClozeTest_17965", - "flip": 1 - }, - { - "global index": "SuperGLUE-ClozeTest_18766", - "flip": 1 - }, - { - "global index": "SuperGLUE-Entailment_19410", - "flip": 1 - }, - { - "global index": "SuperGLUE-Entailment_19567", - "flip": 1 - }, - { - "global index": "SuperGLUE-Entailment_522", - "flip": 1 - }, - { - "global index": "SuperGLUE-Entailment_767", - "flip": 1 - }, - { - "global index": "SuperGLUE-QA_1408", - "flip": 1 - }, - { - "global index": "SuperGLUE-QA_3137", - "flip": 1 - }, - { - "global index": "SuperGLUE-QA_4046", - "flip": 1 - }, - { - "global index": "SuperGLUE-QA_4102", - "flip": 1 - }, - { - "global index": "SuperGLUE-QA_4160", - "flip": 1 - }, - { - "global index": "SuperGLUE-RC_7725", - "flip": 1 - }, - { - "global index": "SuperGLUE-RC_7738", - "flip": 1 - }, - { - "global index": "SuperGLUE-RC_8531", - "flip": 1 - }, - { - "global index": "SuperGLUE-Wic_19695", - "flip": 1 - }, - { - "global index": "SuperGLUE-Wic_19738", - "flip": 1 - }, - { - "global index": "SuperGLUE-Wic_20079", - "flip": 1 - }, - { - "global index": "SuperGLUE-Wic_20189", - "flip": 1 - }, - { - "global index": "SuperGLUE-Wic_20253", - "flip": 1 - }, - { - "global index": "SuperGLUE-Wsc_20368", - "flip": 1 - }, - { - "global index": "SuperGLUE-Wsc_20370", - "flip": 1 - }, - { - "global index": "WMT19-cs-en_156", - "flip": 0 - }, - { - "global index": "WMT19-cs-en_246", - "flip": 1 - }, - { - "global index": "WMT19-cs-en_568", - "flip": 1 - }, - { - "global index": "WMT19-de-en_46", - "flip": 0 - }, - { - "global index": "WMT19-de-en_715", - "flip": 1 - }, - { - "global index": "WMT19-de-en_883", - "flip": 1 - }, - { - "global index": "WMT19-fi-en_222", - "flip": 1 - }, - { - "global index": "WMT19-fi-en_610", - "flip": 1 - }, - { - "global index": "WMT19-gu-en_116", - "flip": 0 - }, - { - "global index": "WMT19-gu-en_123", - "flip": 1 - }, - { - "global index": "WMT19-gu-en_191", - "flip": 1 - }, - { - "global index": "WMT19-gu-en_491", - "flip": 1 - }, - { - "global index": "WMT19-gu-en_968", - "flip": 1 - }, - { - "global index": "WMT19-kk-en_528", - "flip": 1 - }, - { - "global index": "WMT19-kk-en_826", - "flip": 0 - }, - { - "global index": "WMT19-lt-en_135", - "flip": 1 - }, - { - "global index": "WMT19-lt-en_269", - "flip": 0 - }, - { - "global index": "WMT19-lt-en_636", - "flip": 1 - }, - { - "global index": "WMT19-ru-en_222", - "flip": 1 - }, - { - "global index": "WMT19-zh-en_218", - "flip": 1 - }, - { - "global index": "WMT19-zh-en_252", - "flip": 1 - }, - { - "global index": "WMT19-zh-en_59", - "flip": 1 - } -] diff --git a/src/data/routerMetrics/category_scores.json b/src/data/routerMetrics/category_scores.json index 138bf85..f8dbafe 100644 --- a/src/data/routerMetrics/category_scores.json +++ b/src/data/routerMetrics/category_scores.json @@ -12067,123 +12067,123 @@ } } }, - "r2-router": { + "auto_router": { "metrics": { "easy": { - "accuracy": 95.8, - "cost": 0.0, - "robustness": 47.0 + "accuracy": 95.2, + "cost": 0.0001, + "robustness": 48.0 }, "medium": { - "accuracy": 67.3, - "cost": 0.0, - "robustness": 50.4 + "accuracy": 65.8, + "cost": 0.0001, + "robustness": 45.1 }, "hard": { - "accuracy": 26.1, - "cost": 0.0001, - "robustness": 38.5 + "accuracy": 25.9, + "cost": 0.0002, + "robustness": 56.9 }, "all": { - "accuracy": 71.2, - "cost": 0.0, - "robustness": 45.7 + "accuracy": 70.4, + "cost": 0.0001, + "robustness": 49.5 } }, "categories": { "Arts & recreation": { "metrics": { "easy": { - "accuracy": 94.6, + "accuracy": 93.0, "cost": 0.0, - "robustness": 50.0 + "robustness": 62.5 }, "medium": { - "accuracy": 53.7, + "accuracy": 49.4, "cost": 0.0001, - "robustness": 72.7 + "robustness": 63.6 }, "hard": { - "accuracy": 6.1, + "accuracy": 11.6, "cost": 0.0001, - "robustness": 0.0 + "robustness": 80.0 }, "all": { - "accuracy": 61.1, + "accuracy": 60.0, "cost": 0.0001, - "robustness": 50.0 + "robustness": 66.7 } }, "subcategories": { "Arts": { "metrics": { "easy": { - "accuracy": 94.9, - "cost": 0.0, + "accuracy": 96.6, + "cost": 0.0001, "robustness": 0.0 }, "medium": { - "accuracy": 84.6, + "accuracy": 74.4, "cost": 0.0, - "robustness": 100.0 + "robustness": 50.0 }, "hard": { "accuracy": 5.4, - "cost": 0.0, - "robustness": 0.0 + "cost": 0.0001, + "robustness": 100.0 }, "all": { - "accuracy": 59.7, - "cost": 0.0, - "robustness": 40.0 + "accuracy": 57.8, + "cost": 0.0001, + "robustness": 60.0 } } }, "Music": { "metrics": { "easy": { - "accuracy": 98.0, + "accuracy": 93.9, "cost": 0.0, - "robustness": 40.0 + "robustness": 80.0 }, "medium": { - "accuracy": 43.1, + "accuracy": 33.6, "cost": 0.0001, - "robustness": 60.0 + "robustness": 40.0 }, "hard": { - "accuracy": 8.0, + "accuracy": 24.0, "cost": 0.0001, "robustness": 0.0 }, "all": { - "accuracy": 62.1, + "accuracy": 57.5, "cost": 0.0001, - "robustness": 45.5 + "robustness": 54.5 } } }, "Sports, games and entertainment": { "metrics": { "easy": { - "accuracy": 92.1, + "accuracy": 90.7, "cost": 0.0, - "robustness": 100.0 + "robustness": 50.0 }, "medium": { - "accuracy": 54.0, + "accuracy": 58.0, "cost": 0.0001, - "robustness": 75.0 + "robustness": 100.0 }, "hard": { - "accuracy": 6.1, + "accuracy": 12.1, "cost": 0.0001, - "robustness": 0.0 + "robustness": 100.0 }, "all": { - "accuracy": 61.1, + "accuracy": 63.1, "cost": 0.0001, - "robustness": 62.5 + "robustness": 87.5 } } } @@ -12192,24 +12192,24 @@ "Computer science, information, and general works": { "metrics": { "easy": { - "accuracy": 97.3, - "cost": 0.0, + "accuracy": 97.8, + "cost": 0.0001, "robustness": 42.3 }, "medium": { - "accuracy": 80.7, - "cost": 0.0, - "robustness": 42.9 + "accuracy": 78.8, + "cost": 0.0002, + "robustness": 57.1 }, "hard": { - "accuracy": 31.4, - "cost": 0.0001, - "robustness": 53.3 + "accuracy": 34.7, + "cost": 0.0005, + "robustness": 60.0 }, "all": { - "accuracy": 80.3, - "cost": 0.0, - "robustness": 45.2 + "accuracy": 80.5, + "cost": 0.0002, + "robustness": 51.6 } }, "subcategories": { @@ -12217,47 +12217,47 @@ "metrics": { "easy": { "accuracy": 98.1, - "cost": 0.0, + "cost": 0.0001, "robustness": 46.2 }, "medium": { - "accuracy": 84.9, - "cost": 0.0001, - "robustness": 52.9 + "accuracy": 84.1, + "cost": 0.0002, + "robustness": 64.7 }, "hard": { - "accuracy": 36.4, - "cost": 0.0001, - "robustness": 53.8 + "accuracy": 39.7, + "cost": 0.0006, + "robustness": 61.5 }, "all": { - "accuracy": 80.3, - "cost": 0.0001, - "robustness": 51.2 + "accuracy": 80.7, + "cost": 0.0002, + "robustness": 58.1 } } }, "Library and information sciences": { "metrics": { "easy": { - "accuracy": 96.0, - "cost": 0.0, + "accuracy": 97.5, + "cost": 0.0001, "robustness": 38.5 }, "medium": { - "accuracy": 62.4, - "cost": 0.0, - "robustness": 0.0 + "accuracy": 55.3, + "cost": 0.0001, + "robustness": 25.0 }, "hard": { - "accuracy": 2.8, - "cost": 0.0, + "accuracy": 5.6, + "cost": 0.0001, "robustness": 50.0 }, "all": { - "accuracy": 80.3, - "cost": 0.0, - "robustness": 31.6 + "accuracy": 80.1, + "cost": 0.0001, + "robustness": 36.8 } } } @@ -12266,23 +12266,23 @@ "History": { "metrics": { "easy": { - "accuracy": 96.8, - "cost": 0.0, - "robustness": 52.2 + "accuracy": 96.0, + "cost": 0.0001, + "robustness": 30.4 }, "medium": { - "accuracy": 62.1, - "cost": 0.0, - "robustness": 16.7 + "accuracy": 63.3, + "cost": 0.0001, + "robustness": 33.3 }, "hard": { - "accuracy": 10.1, - "cost": 0.0, - "robustness": 30.0 + "accuracy": 9.4, + "cost": 0.0001, + "robustness": 70.0 }, "all": { - "accuracy": 69.6, - "cost": 0.0, + "accuracy": 69.3, + "cost": 0.0001, "robustness": 41.0 } }, @@ -12290,12 +12290,12 @@ "Biography and genealogy": { "metrics": { "easy": { - "accuracy": 92.9, + "accuracy": 97.6, "cost": 0.0, - "robustness": 75.0 + "robustness": 25.0 }, "medium": { - "accuracy": 77.8, + "accuracy": 100.0, "cost": 0.0, "robustness": 0 }, @@ -12305,31 +12305,31 @@ "robustness": 0 }, "all": { - "accuracy": 88.5, + "accuracy": 96.2, "cost": 0.0, - "robustness": 75.0 + "robustness": 25.0 } } }, "Geography": { "metrics": { "easy": { - "accuracy": 100.0, + "accuracy": 97.2, "cost": 0.0, "robustness": 37.5 }, "medium": { - "accuracy": 51.7, + "accuracy": 48.3, "cost": 0.0, "robustness": 0.0 }, "hard": { - "accuracy": 8.3, + "accuracy": 16.7, "cost": 0.0, "robustness": 0 }, "all": { - "accuracy": 83.3, + "accuracy": 81.3, "cost": 0.0, "robustness": 33.3 } @@ -12338,24 +12338,24 @@ "History": { "metrics": { "easy": { - "accuracy": 96.0, - "cost": 0.0, - "robustness": 54.5 + "accuracy": 95.1, + "cost": 0.0001, + "robustness": 27.3 }, "medium": { - "accuracy": 63.3, - "cost": 0.0, - "robustness": 20.0 + "accuracy": 64.0, + "cost": 0.0001, + "robustness": 40.0 }, "hard": { - "accuracy": 10.3, - "cost": 0.0, - "robustness": 30.0 + "accuracy": 8.8, + "cost": 0.0001, + "robustness": 70.0 }, "all": { - "accuracy": 63.5, - "cost": 0.0, - "robustness": 38.5 + "accuracy": 62.9, + "cost": 0.0001, + "robustness": 46.2 } } } @@ -12364,48 +12364,48 @@ "Language": { "metrics": { "easy": { - "accuracy": 90.4, - "cost": 0.0, - "robustness": 0.0 + "accuracy": 88.5, + "cost": 0.0001, + "robustness": 63.6 }, "medium": { - "accuracy": 61.7, + "accuracy": 44.9, "cost": 0.0001, - "robustness": 0.0 + "robustness": 44.4 }, "hard": { - "accuracy": 58.0, - "cost": 0.0, - "robustness": 20.0 + "accuracy": 48.4, + "cost": 0.0002, + "robustness": 80.0 }, "all": { - "accuracy": 68.6, - "cost": 0.0, - "robustness": 11.1 + "accuracy": 59.6, + "cost": 0.0001, + "robustness": 68.9 } }, "subcategories": { "Language": { "metrics": { "easy": { - "accuracy": 90.4, - "cost": 0.0, - "robustness": 0.0 + "accuracy": 88.5, + "cost": 0.0001, + "robustness": 63.6 }, "medium": { - "accuracy": 61.7, + "accuracy": 44.9, "cost": 0.0001, - "robustness": 0.0 + "robustness": 44.4 }, "hard": { - "accuracy": 58.0, - "cost": 0.0, - "robustness": 20.0 + "accuracy": 48.4, + "cost": 0.0002, + "robustness": 80.0 }, "all": { - "accuracy": 68.6, - "cost": 0.0, - "robustness": 11.1 + "accuracy": 59.6, + "cost": 0.0001, + "robustness": 68.9 } } } @@ -12415,23 +12415,23 @@ "metrics": { "easy": { "accuracy": 100.0, - "cost": 0.0, - "robustness": 66.7 + "cost": 0.0001, + "robustness": 33.3 }, "medium": { - "accuracy": 56.2, - "cost": 0.0, - "robustness": 50.0 - }, - "hard": { - "accuracy": 30.1, + "accuracy": 68.7, "cost": 0.0001, "robustness": 50.0 }, + "hard": { + "accuracy": 28.2, + "cost": 0.0002, + "robustness": 45.8 + }, "all": { - "accuracy": 44.0, - "cost": 0.0001, - "robustness": 51.7 + "accuracy": 43.7, + "cost": 0.0002, + "robustness": 44.8 } }, "subcategories": { @@ -12439,23 +12439,23 @@ "metrics": { "easy": { "accuracy": 100.0, - "cost": 0.0, - "robustness": 66.7 + "cost": 0.0001, + "robustness": 33.3 }, "medium": { - "accuracy": 56.2, - "cost": 0.0, + "accuracy": 68.7, + "cost": 0.0001, "robustness": 50.0 }, "hard": { - "accuracy": 30.1, - "cost": 0.0001, - "robustness": 50.0 + "accuracy": 28.2, + "cost": 0.0002, + "robustness": 45.8 }, "all": { - "accuracy": 44.0, - "cost": 0.0001, - "robustness": 51.7 + "accuracy": 43.7, + "cost": 0.0002, + "robustness": 44.8 } } } @@ -12464,96 +12464,96 @@ "Philosophy and psychology": { "metrics": { "easy": { - "accuracy": 91.9, - "cost": 0.0, - "robustness": 22.2 + "accuracy": 93.1, + "cost": 0.0001, + "robustness": 51.9 }, "medium": { - "accuracy": 62.6, - "cost": 0.0, - "robustness": 40.0 + "accuracy": 62.1, + "cost": 0.0001, + "robustness": 30.0 }, "hard": { - "accuracy": 13.6, - "cost": 0.0, - "robustness": 42.9 + "accuracy": 12.3, + "cost": 0.0001, + "robustness": 28.6 }, "all": { - "accuracy": 74.0, - "cost": 0.0, - "robustness": 29.5 + "accuracy": 74.4, + "cost": 0.0001, + "robustness": 43.2 } }, "subcategories": { "Ethics": { "metrics": { "easy": { - "accuracy": 85.9, + "accuracy": 89.2, "cost": 0.0, - "robustness": 21.4 + "robustness": 50.0 }, "medium": { - "accuracy": 47.1, - "cost": 0.0, - "robustness": 20.0 + "accuracy": 32.9, + "cost": 0.0001, + "robustness": 0.0 }, "hard": { "accuracy": 0.0, - "cost": 0.0, - "robustness": 100.0 + "cost": 0.0001, + "robustness": 0.0 }, "all": { - "accuracy": 70.3, - "cost": 0.0, - "robustness": 28.6 + "accuracy": 68.2, + "cost": 0.0001, + "robustness": 33.3 } } }, "Philosophical logic": { "metrics": { "easy": { - "accuracy": 96.9, - "cost": 0.0, + "accuracy": 100.0, + "cost": 0.0001, "robustness": 25.0 }, "medium": { - "accuracy": 84.2, - "cost": 0.0, - "robustness": 66.7 + "accuracy": 89.5, + "cost": 0.0001, + "robustness": 100.0 }, "hard": { - "accuracy": 60.0, + "accuracy": 20.0, "cost": 0.0001, "robustness": 0 }, "all": { - "accuracy": 90.7, - "cost": 0.0, - "robustness": 42.9 + "accuracy": 92.6, + "cost": 0.0001, + "robustness": 57.1 } } }, "Philosophy": { "metrics": { "easy": { - "accuracy": 94.6, + "accuracy": 86.5, "cost": 0.0, - "robustness": 0.0 + "robustness": 33.3 }, "medium": { - "accuracy": 52.5, - "cost": 0.0, - "robustness": 50.0 + "accuracy": 70.0, + "cost": 0.0001, + "robustness": 0.0 }, "hard": { - "accuracy": 13.3, - "cost": 0.0, - "robustness": 0.0 + "accuracy": 17.8, + "cost": 0.0001, + "robustness": 66.7 }, "all": { - "accuracy": 50.8, - "cost": 0.0, - "robustness": 12.5 + "accuracy": 55.7, + "cost": 0.0001, + "robustness": 37.5 } } }, @@ -12561,23 +12561,23 @@ "metrics": { "easy": { "accuracy": 97.5, - "cost": 0.0, - "robustness": 33.3 + "cost": 0.0001, + "robustness": 83.3 }, "medium": { - "accuracy": 81.2, - "cost": 0.0, + "accuracy": 85.4, + "cost": 0.0001, "robustness": 0 }, "hard": { - "accuracy": 11.1, + "accuracy": 5.6, "cost": 0.0, - "robustness": 50.0 + "robustness": 0.0 }, "all": { - "accuracy": 85.0, - "cost": 0.0, - "robustness": 37.5 + "accuracy": 85.6, + "cost": 0.0001, + "robustness": 62.5 } } } @@ -12586,33 +12586,33 @@ "Science": { "metrics": { "easy": { - "accuracy": 98.6, - "cost": 0.0, - "robustness": 61.5 + "accuracy": 97.2, + "cost": 0.0001, + "robustness": 43.6 }, "medium": { - "accuracy": 76.2, - "cost": 0.0, - "robustness": 57.1 + "accuracy": 75.5, + "cost": 0.0001, + "robustness": 47.6 }, "hard": { - "accuracy": 11.3, - "cost": 0.0, - "robustness": 44.4 + "accuracy": 19.1, + "cost": 0.0003, + "robustness": 55.6 }, "all": { "accuracy": 79.6, - "cost": 0.0, - "robustness": 58.0 + "cost": 0.0001, + "robustness": 46.4 } }, "subcategories": { "Animals (Zoology)": { "metrics": { "easy": { - "accuracy": 100.0, + "accuracy": 95.0, "cost": 0.0, - "robustness": 0.0 + "robustness": 50.0 }, "medium": { "accuracy": 80.0, @@ -12625,32 +12625,32 @@ "robustness": 0 }, "all": { - "accuracy": 96.0, + "accuracy": 92.0, "cost": 0.0, - "robustness": 0.0 + "robustness": 50.0 } } }, "Biology": { "metrics": { "easy": { - "accuracy": 99.1, - "cost": 0.0, + "accuracy": 98.3, + "cost": 0.0001, "robustness": 71.4 }, "medium": { - "accuracy": 64.7, + "accuracy": 58.8, "cost": 0.0, "robustness": 0 }, "hard": { "accuracy": 11.1, - "cost": 0.0, + "cost": 0.0002, "robustness": 0 }, "all": { - "accuracy": 89.5, - "cost": 0.0, + "accuracy": 88.1, + "cost": 0.0001, "robustness": 71.4 } } @@ -12659,94 +12659,94 @@ "metrics": { "easy": { "accuracy": 100.0, - "cost": 0.0, - "robustness": 66.7 + "cost": 0.0001, + "robustness": 33.3 }, "medium": { - "accuracy": 79.4, - "cost": 0.0, - "robustness": 50.0 + "accuracy": 82.4, + "cost": 0.0001, + "robustness": 0.0 }, "hard": { - "accuracy": 7.7, - "cost": 0.0, + "accuracy": 15.4, + "cost": 0.0002, "robustness": 0 }, "all": { - "accuracy": 78.9, - "cost": 0.0, - "robustness": 60.0 + "accuracy": 81.1, + "cost": 0.0001, + "robustness": 20.0 } } }, "Earth sciences and geology": { "metrics": { "easy": { - "accuracy": 98.1, - "cost": 0.0, - "robustness": 50.0 + "accuracy": 97.3, + "cost": 0.0001, + "robustness": 37.5 }, "medium": { - "accuracy": 72.9, - "cost": 0.0, - "robustness": 50.0 + "accuracy": 67.8, + "cost": 0.0001, + "robustness": 16.7 }, "hard": { - "accuracy": 7.7, - "cost": 0.0, - "robustness": 0.0 + "accuracy": 0.0, + "cost": 0.0002, + "robustness": 100.0 }, "all": { - "accuracy": 90.0, - "cost": 0.0, - "robustness": 46.7 + "accuracy": 88.2, + "cost": 0.0001, + "robustness": 33.3 } } }, "Mathematics": { "metrics": { "easy": { - "accuracy": 98.1, - "cost": 0.0, - "robustness": 72.7 + "accuracy": 96.2, + "cost": 0.0001, + "robustness": 18.2 }, "medium": { - "accuracy": 79.2, - "cost": 0.0, + "accuracy": 80.0, + "cost": 0.0001, "robustness": 63.6 }, "hard": { - "accuracy": 12.4, - "cost": 0.0, - "robustness": 80.0 + "accuracy": 30.3, + "cost": 0.0006, + "robustness": 20.0 }, "all": { - "accuracy": 77.5, - "cost": 0.0, - "robustness": 70.4 + "accuracy": 79.7, + "cost": 0.0002, + "robustness": 37.0 } } }, "Physics": { "metrics": { "easy": { - "accuracy": 100.0, + "accuracy": 97.7, "cost": 0.0, - "robustness": 100.0 + "robustness": 66.7 }, "medium": { - "accuracy": 83.3, - "cost": 0.0, - "robustness": 50.0 + "accuracy": 73.8, + "cost": 0.0001, + "robustness": 100.0 }, "hard": { - "accuracy": 27.3, + "accuracy": 36.4, "cost": 0.0, "robustness": 0 }, "all": { - "accuracy": 84.5, - "cost": 0.0, + "accuracy": 80.4, + "cost": 0.0001, "robustness": 80.0 } } @@ -12754,24 +12754,24 @@ "Science": { "metrics": { "easy": { - "accuracy": 100.0, + "accuracy": 96.7, "cost": 0.0, - "robustness": 40.0 + "robustness": 60.0 }, "medium": { - "accuracy": 44.0, + "accuracy": 52.0, "cost": 0.0, "robustness": 0 }, "hard": { - "accuracy": 8.5, + "accuracy": 5.1, "cost": 0.0, - "robustness": 0.0 + "robustness": 100.0 }, "all": { - "accuracy": 40.4, + "accuracy": 39.5, "cost": 0.0, - "robustness": 25.0 + "robustness": 75.0 } } } @@ -12780,48 +12780,48 @@ "Social Science": { "metrics": { "easy": { - "accuracy": 95.3, - "cost": 0.0, - "robustness": 55.6 + "accuracy": 94.3, + "cost": 0.0001, + "robustness": 50.0 }, "medium": { - "accuracy": 60.5, - "cost": 0.0, - "robustness": 83.3 + "accuracy": 63.6, + "cost": 0.0001, + "robustness": 25.0 }, "hard": { - "accuracy": 8.7, - "cost": 0.0, - "robustness": 28.6 + "accuracy": 7.4, + "cost": 0.0002, + "robustness": 42.9 }, "all": { - "accuracy": 64.3, - "cost": 0.0, - "robustness": 59.5 + "accuracy": 64.7, + "cost": 0.0001, + "robustness": 40.5 } }, "subcategories": { "Economics": { "metrics": { "easy": { - "accuracy": 97.8, - "cost": 0.0, - "robustness": 66.7 + "accuracy": 97.3, + "cost": 0.0001, + "robustness": 41.7 }, "medium": { - "accuracy": 66.3, - "cost": 0.0, - "robustness": 83.3 + "accuracy": 73.6, + "cost": 0.0002, + "robustness": 33.3 }, "hard": { - "accuracy": 12.5, - "cost": 0.0001, - "robustness": 66.7 + "accuracy": 8.8, + "cost": 0.0003, + "robustness": 33.3 }, "all": { - "accuracy": 69.7, - "cost": 0.0, - "robustness": 71.4 + "accuracy": 71.6, + "cost": 0.0001, + "robustness": 38.1 } } }, @@ -12829,71 +12829,71 @@ "metrics": { "easy": { "accuracy": 87.7, - "cost": 0.0, - "robustness": 66.7 + "cost": 0.0002, + "robustness": 33.3 }, "medium": { - "accuracy": 54.0, - "cost": 0.0, - "robustness": 75.0 + "accuracy": 47.6, + "cost": 0.0002, + "robustness": 0.0 }, "hard": { "accuracy": 6.7, - "cost": 0.0, - "robustness": 0.0 + "cost": 0.0002, + "robustness": 50.0 }, "all": { - "accuracy": 58.9, - "cost": 0.0, - "robustness": 55.6 + "accuracy": 56.3, + "cost": 0.0002, + "robustness": 22.2 } } }, "Social problems": { "metrics": { "easy": { - "accuracy": 95.7, + "accuracy": 93.6, "cost": 0.0, - "robustness": 0.0 + "robustness": 100.0 }, "medium": { - "accuracy": 0.0, + "accuracy": 33.3, "cost": 0.0, "robustness": 0 }, "hard": { - "accuracy": 20.0, + "accuracy": 0.0, "cost": 0.0, - "robustness": 0.0 + "robustness": 100.0 }, "all": { - "accuracy": 75.4, + "accuracy": 77.0, "cost": 0.0, - "robustness": 0.0 + "robustness": 100.0 } } }, "Social sciences, sociology, and anthropology": { "metrics": { "easy": { - "accuracy": 100.0, + "accuracy": 66.7, "cost": 0.0, "robustness": 0 }, "medium": { - "accuracy": 61.1, + "accuracy": 44.4, "cost": 0.0, - "robustness": 100.0 + "robustness": 50.0 }, "hard": { - "accuracy": 0.0, + "accuracy": 5.9, "cost": 0.0, "robustness": 0.0 }, "all": { - "accuracy": 25.5, + "accuracy": 21.8, "cost": 0.0, - "robustness": 66.7 + "robustness": 33.3 } } } @@ -12902,48 +12902,48 @@ "Technology": { "metrics": { "easy": { - "accuracy": 94.7, - "cost": 0.0, + "accuracy": 94.0, + "cost": 0.0001, "robustness": 55.8 }, "medium": { - "accuracy": 64.3, - "cost": 0.0, - "robustness": 57.1 + "accuracy": 63.3, + "cost": 0.0001, + "robustness": 42.9 }, "hard": { - "accuracy": 13.4, - "cost": 0.0, - "robustness": 71.4 + "accuracy": 19.7, + "cost": 0.0002, + "robustness": 14.3 }, "all": { - "accuracy": 76.4, - "cost": 0.0, - "robustness": 57.7 + "accuracy": 76.5, + "cost": 0.0001, + "robustness": 47.9 } }, "subcategories": { "Engineering": { "metrics": { "easy": { - "accuracy": 95.2, - "cost": 0.0, - "robustness": 20.0 + "accuracy": 96.0, + "cost": 0.0001, + "robustness": 60.0 }, "medium": { - "accuracy": 79.0, - "cost": 0.0, - "robustness": 53.8 + "accuracy": 80.2, + "cost": 0.0002, + "robustness": 46.2 }, "hard": { - "accuracy": 31.0, - "cost": 0.0, + "accuracy": 42.9, + "cost": 0.0003, "robustness": 100.0 }, "all": { - "accuracy": 79.1, - "cost": 0.0, - "robustness": 47.4 + "accuracy": 81.5, + "cost": 0.0002, + "robustness": 52.6 } } }, @@ -12951,11 +12951,11 @@ "metrics": { "easy": { "accuracy": 98.3, - "cost": 0.0, + "cost": 0.0001, "robustness": 75.0 }, "medium": { - "accuracy": 81.8, + "accuracy": 63.6, "cost": 0.0, "robustness": 0.0 }, @@ -12965,8 +12965,8 @@ "robustness": 0 }, "all": { - "accuracy": 93.2, - "cost": 0.0, + "accuracy": 90.4, + "cost": 0.0001, "robustness": 60.0 } } @@ -12974,24 +12974,24 @@ "Medicine and health": { "metrics": { "easy": { - "accuracy": 94.2, - "cost": 0.0, - "robustness": 58.8 + "accuracy": 93.2, + "cost": 0.0001, + "robustness": 52.9 }, "medium": { - "accuracy": 53.3, - "cost": 0.0, - "robustness": 71.4 + "accuracy": 51.7, + "cost": 0.0001, + "robustness": 42.9 }, "hard": { - "accuracy": 7.1, - "cost": 0.0, - "robustness": 66.7 + "accuracy": 11.5, + "cost": 0.0001, + "robustness": 0.0 }, "all": { - "accuracy": 74.3, - "cost": 0.0, - "robustness": 61.7 + "accuracy": 73.8, + "cost": 0.0001, + "robustness": 44.7 } } } @@ -12999,74 +12999,74 @@ } } }, - "auto_router": { + "sqwish-router": { "metrics": { "easy": { - "accuracy": 95.2, - "cost": 0.0001, - "robustness": 48.0 + "accuracy": 97.3, + "cost": 0.0002, + "robustness": 51.0 }, "medium": { - "accuracy": 65.8, - "cost": 0.0001, - "robustness": 45.1 + "accuracy": 79.3, + "cost": 0.0008, + "robustness": 44.2 }, "hard": { - "accuracy": 25.9, - "cost": 0.0002, - "robustness": 56.9 + "accuracy": 45.2, + "cost": 0.0015, + "robustness": 60.6 }, "all": { - "accuracy": 70.4, - "cost": 0.0001, - "robustness": 49.5 + "accuracy": 79.9, + "cost": 0.0007, + "robustness": 51.7 } }, "categories": { "Arts & recreation": { "metrics": { "easy": { - "accuracy": 93.0, - "cost": 0.0, - "robustness": 62.5 + "accuracy": 97.0, + "cost": 0.0006, + "robustness": 50.0 }, "medium": { - "accuracy": 49.4, - "cost": 0.0001, - "robustness": 63.6 + "accuracy": 76.5, + "cost": 0.0019, + "robustness": 9.1 }, "hard": { - "accuracy": 11.6, - "cost": 0.0001, - "robustness": 80.0 + "accuracy": 33.3, + "cost": 0.0026, + "robustness": 60.0 }, "all": { - "accuracy": 60.0, - "cost": 0.0001, - "robustness": 66.7 + "accuracy": 76.1, + "cost": 0.0015, + "robustness": 33.3 } }, "subcategories": { "Arts": { "metrics": { "easy": { - "accuracy": 96.6, - "cost": 0.0001, - "robustness": 0.0 + "accuracy": 98.3, + "cost": 0.0004, + "robustness": 100.0 }, "medium": { - "accuracy": 74.4, - "cost": 0.0, - "robustness": 50.0 + "accuracy": 94.9, + "cost": 0.0014, + "robustness": 0.0 }, "hard": { - "accuracy": 5.4, - "cost": 0.0001, + "accuracy": 33.9, + "cost": 0.0034, "robustness": 100.0 }, "all": { - "accuracy": 57.8, - "cost": 0.0001, + "accuracy": 74.0, + "cost": 0.0018, "robustness": 60.0 } } @@ -13074,48 +13074,48 @@ "Music": { "metrics": { "easy": { - "accuracy": 93.9, - "cost": 0.0, - "robustness": 80.0 + "accuracy": 97.0, + "cost": 0.0004, + "robustness": 60.0 }, "medium": { - "accuracy": 33.6, - "cost": 0.0001, - "robustness": 40.0 - }, - "hard": { - "accuracy": 24.0, - "cost": 0.0001, + "accuracy": 58.6, + "cost": 0.0019, + "robustness": 20.0 + }, + "hard": { + "accuracy": 40.0, + "cost": 0.0029, "robustness": 0.0 }, "all": { - "accuracy": 57.5, - "cost": 0.0001, - "robustness": 54.5 + "accuracy": 72.5, + "cost": 0.0014, + "robustness": 36.4 } } }, "Sports, games and entertainment": { "metrics": { "easy": { - "accuracy": 90.7, - "cost": 0.0, - "robustness": 50.0 + "accuracy": 96.4, + "cost": 0.0008, + "robustness": 0.0 }, "medium": { - "accuracy": 58.0, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 90.0, + "cost": 0.0022, + "robustness": 0.0 }, "hard": { - "accuracy": 12.1, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 30.3, + "cost": 0.0018, + "robustness": 50.0 }, "all": { - "accuracy": 63.1, - "cost": 0.0001, - "robustness": 87.5 + "accuracy": 80.1, + "cost": 0.0015, + "robustness": 12.5 } } } @@ -13124,72 +13124,72 @@ "Computer science, information, and general works": { "metrics": { "easy": { - "accuracy": 97.8, - "cost": 0.0001, - "robustness": 42.3 + "accuracy": 98.0, + "cost": 0.0002, + "robustness": 53.8 }, "medium": { - "accuracy": 78.8, - "cost": 0.0002, + "accuracy": 87.2, + "cost": 0.0005, "robustness": 57.1 }, "hard": { - "accuracy": 34.7, - "cost": 0.0005, + "accuracy": 59.2, + "cost": 0.0012, "robustness": 60.0 }, "all": { - "accuracy": 80.5, - "cost": 0.0002, - "robustness": 51.6 + "accuracy": 87.6, + "cost": 0.0005, + "robustness": 56.5 } }, "subcategories": { "Computer science, knowledge, and systems": { "metrics": { "easy": { - "accuracy": 98.1, - "cost": 0.0001, - "robustness": 46.2 + "accuracy": 98.6, + "cost": 0.0002, + "robustness": 53.8 }, "medium": { - "accuracy": 84.1, - "cost": 0.0002, - "robustness": 64.7 + "accuracy": 91.2, + "cost": 0.0006, + "robustness": 52.9 }, "hard": { - "accuracy": 39.7, - "cost": 0.0006, + "accuracy": 66.5, + "cost": 0.0014, "robustness": 61.5 }, "all": { - "accuracy": 80.7, - "cost": 0.0002, - "robustness": 58.1 + "accuracy": 89.1, + "cost": 0.0006, + "robustness": 55.8 } } }, "Library and information sciences": { "metrics": { "easy": { - "accuracy": 97.5, - "cost": 0.0001, - "robustness": 38.5 + "accuracy": 97.1, + "cost": 0.0002, + "robustness": 53.8 }, "medium": { - "accuracy": 55.3, - "cost": 0.0001, - "robustness": 25.0 + "accuracy": 69.4, + "cost": 0.0003, + "robustness": 75.0 }, "hard": { - "accuracy": 5.6, - "cost": 0.0001, + "accuracy": 16.7, + "cost": 0.0002, "robustness": 50.0 }, "all": { - "accuracy": 80.1, - "cost": 0.0001, - "robustness": 36.8 + "accuracy": 83.8, + "cost": 0.0002, + "robustness": 57.9 } } } @@ -13198,48 +13198,48 @@ "History": { "metrics": { "easy": { - "accuracy": 96.0, - "cost": 0.0001, - "robustness": 30.4 + "accuracy": 97.6, + "cost": 0.0003, + "robustness": 39.1 }, "medium": { - "accuracy": 63.3, - "cost": 0.0001, + "accuracy": 81.4, + "cost": 0.001, "robustness": 33.3 }, "hard": { - "accuracy": 9.4, - "cost": 0.0001, - "robustness": 70.0 + "accuracy": 29.5, + "cost": 0.0021, + "robustness": 40.0 }, "all": { - "accuracy": 69.3, - "cost": 0.0001, - "robustness": 41.0 + "accuracy": 79.0, + "cost": 0.0009, + "robustness": 38.5 } }, "subcategories": { "Biography and genealogy": { "metrics": { "easy": { - "accuracy": 97.6, - "cost": 0.0, - "robustness": 25.0 + "accuracy": 100.0, + "cost": 0.0003, + "robustness": 0.0 }, "medium": { "accuracy": 100.0, - "cost": 0.0, + "cost": 0.0002, "robustness": 0 }, "hard": { "accuracy": 0.0, - "cost": 0.0, + "cost": 0.0001, "robustness": 0 }, "all": { - "accuracy": 96.2, - "cost": 0.0, - "robustness": 25.0 + "accuracy": 98.1, + "cost": 0.0003, + "robustness": 0.0 } } }, @@ -13247,1911 +13247,47 @@ "metrics": { "easy": { "accuracy": 97.2, - "cost": 0.0, + "cost": 0.0003, "robustness": 37.5 }, "medium": { - "accuracy": 48.3, - "cost": 0.0, + "accuracy": 65.5, + "cost": 0.0014, "robustness": 0.0 }, "hard": { - "accuracy": 16.7, - "cost": 0.0, + "accuracy": 58.3, + "cost": 0.0027, "robustness": 0 }, "all": { - "accuracy": 81.3, - "cost": 0.0, + "accuracy": 88.0, + "cost": 0.0007, "robustness": 33.3 } } }, "History": { - "metrics": { - "easy": { - "accuracy": 95.1, - "cost": 0.0001, - "robustness": 27.3 - }, - "medium": { - "accuracy": 64.0, - "cost": 0.0001, - "robustness": 40.0 - }, - "hard": { - "accuracy": 8.8, - "cost": 0.0001, - "robustness": 70.0 - }, - "all": { - "accuracy": 62.9, - "cost": 0.0001, - "robustness": 46.2 - } - } - } - } - }, - "Language": { - "metrics": { - "easy": { - "accuracy": 88.5, - "cost": 0.0001, - "robustness": 63.6 - }, - "medium": { - "accuracy": 44.9, - "cost": 0.0001, - "robustness": 44.4 - }, - "hard": { - "accuracy": 48.4, - "cost": 0.0002, - "robustness": 80.0 - }, - "all": { - "accuracy": 59.6, - "cost": 0.0001, - "robustness": 68.9 - } - }, - "subcategories": { - "Language": { - "metrics": { - "easy": { - "accuracy": 88.5, - "cost": 0.0001, - "robustness": 63.6 - }, - "medium": { - "accuracy": 44.9, - "cost": 0.0001, - "robustness": 44.4 - }, - "hard": { - "accuracy": 48.4, - "cost": 0.0002, - "robustness": 80.0 - }, - "all": { - "accuracy": 59.6, - "cost": 0.0001, - "robustness": 68.9 - } - } - } - } - }, - "Literature": { - "metrics": { - "easy": { - "accuracy": 100.0, - "cost": 0.0001, - "robustness": 33.3 - }, - "medium": { - "accuracy": 68.7, - "cost": 0.0001, - "robustness": 50.0 - }, - "hard": { - "accuracy": 28.2, - "cost": 0.0002, - "robustness": 45.8 - }, - "all": { - "accuracy": 43.7, - "cost": 0.0002, - "robustness": 44.8 - } - }, - "subcategories": { - "Literature, rhetoric and criticism": { - "metrics": { - "easy": { - "accuracy": 100.0, - "cost": 0.0001, - "robustness": 33.3 - }, - "medium": { - "accuracy": 68.7, - "cost": 0.0001, - "robustness": 50.0 - }, - "hard": { - "accuracy": 28.2, - "cost": 0.0002, - "robustness": 45.8 - }, - "all": { - "accuracy": 43.7, - "cost": 0.0002, - "robustness": 44.8 - } - } - } - } - }, - "Philosophy and psychology": { - "metrics": { - "easy": { - "accuracy": 93.1, - "cost": 0.0001, - "robustness": 51.9 - }, - "medium": { - "accuracy": 62.1, - "cost": 0.0001, - "robustness": 30.0 - }, - "hard": { - "accuracy": 12.3, - "cost": 0.0001, - "robustness": 28.6 - }, - "all": { - "accuracy": 74.4, - "cost": 0.0001, - "robustness": 43.2 - } - }, - "subcategories": { - "Ethics": { - "metrics": { - "easy": { - "accuracy": 89.2, - "cost": 0.0, - "robustness": 50.0 - }, - "medium": { - "accuracy": 32.9, - "cost": 0.0001, - "robustness": 0.0 - }, - "hard": { - "accuracy": 0.0, - "cost": 0.0001, - "robustness": 0.0 - }, - "all": { - "accuracy": 68.2, - "cost": 0.0001, - "robustness": 33.3 - } - } - }, - "Philosophical logic": { - "metrics": { - "easy": { - "accuracy": 100.0, - "cost": 0.0001, - "robustness": 25.0 - }, - "medium": { - "accuracy": 89.5, - "cost": 0.0001, - "robustness": 100.0 - }, - "hard": { - "accuracy": 20.0, - "cost": 0.0001, - "robustness": 0 - }, - "all": { - "accuracy": 92.6, - "cost": 0.0001, - "robustness": 57.1 - } - } - }, - "Philosophy": { - "metrics": { - "easy": { - "accuracy": 86.5, - "cost": 0.0, - "robustness": 33.3 - }, - "medium": { - "accuracy": 70.0, - "cost": 0.0001, - "robustness": 0.0 - }, - "hard": { - "accuracy": 17.8, - "cost": 0.0001, - "robustness": 66.7 - }, - "all": { - "accuracy": 55.7, - "cost": 0.0001, - "robustness": 37.5 - } - } - }, - "Psychology": { - "metrics": { - "easy": { - "accuracy": 97.5, - "cost": 0.0001, - "robustness": 83.3 - }, - "medium": { - "accuracy": 85.4, - "cost": 0.0001, - "robustness": 0 - }, - "hard": { - "accuracy": 5.6, - "cost": 0.0, - "robustness": 0.0 - }, - "all": { - "accuracy": 85.6, - "cost": 0.0001, - "robustness": 62.5 - } - } - } - } - }, - "Science": { - "metrics": { - "easy": { - "accuracy": 97.2, - "cost": 0.0001, - "robustness": 43.6 - }, - "medium": { - "accuracy": 75.5, - "cost": 0.0001, - "robustness": 47.6 - }, - "hard": { - "accuracy": 19.1, - "cost": 0.0003, - "robustness": 55.6 - }, - "all": { - "accuracy": 79.6, - "cost": 0.0001, - "robustness": 46.4 - } - }, - "subcategories": { - "Animals (Zoology)": { - "metrics": { - "easy": { - "accuracy": 95.0, - "cost": 0.0, - "robustness": 50.0 - }, - "medium": { - "accuracy": 80.0, - "cost": 0.0, - "robustness": 0 - }, - "hard": { - "accuracy": 0, - "cost": 0, - "robustness": 0 - }, - "all": { - "accuracy": 92.0, - "cost": 0.0, - "robustness": 50.0 - } - } - }, - "Biology": { - "metrics": { - "easy": { - "accuracy": 98.3, - "cost": 0.0001, - "robustness": 71.4 - }, - "medium": { - "accuracy": 58.8, - "cost": 0.0, - "robustness": 0 - }, - "hard": { - "accuracy": 11.1, - "cost": 0.0002, - "robustness": 0 - }, - "all": { - "accuracy": 88.1, - "cost": 0.0001, - "robustness": 71.4 - } - } - }, - "Chemistry": { - "metrics": { - "easy": { - "accuracy": 100.0, - "cost": 0.0001, - "robustness": 33.3 - }, - "medium": { - "accuracy": 82.4, - "cost": 0.0001, - "robustness": 0.0 - }, - "hard": { - "accuracy": 15.4, - "cost": 0.0002, - "robustness": 0 - }, - "all": { - "accuracy": 81.1, - "cost": 0.0001, - "robustness": 20.0 - } - } - }, - "Earth sciences and geology": { - "metrics": { - "easy": { - "accuracy": 97.3, - "cost": 0.0001, - "robustness": 37.5 - }, - "medium": { - "accuracy": 67.8, - "cost": 0.0001, - "robustness": 16.7 - }, - "hard": { - "accuracy": 0.0, - "cost": 0.0002, - "robustness": 100.0 - }, - "all": { - "accuracy": 88.2, - "cost": 0.0001, - "robustness": 33.3 - } - } - }, - "Mathematics": { - "metrics": { - "easy": { - "accuracy": 96.2, - "cost": 0.0001, - "robustness": 18.2 - }, - "medium": { - "accuracy": 80.0, - "cost": 0.0001, - "robustness": 63.6 - }, - "hard": { - "accuracy": 30.3, - "cost": 0.0006, - "robustness": 20.0 - }, - "all": { - "accuracy": 79.7, - "cost": 0.0002, - "robustness": 37.0 - } - } - }, - "Physics": { - "metrics": { - "easy": { - "accuracy": 97.7, - "cost": 0.0, - "robustness": 66.7 - }, - "medium": { - "accuracy": 73.8, - "cost": 0.0001, - "robustness": 100.0 - }, - "hard": { - "accuracy": 36.4, - "cost": 0.0, - "robustness": 0 - }, - "all": { - "accuracy": 80.4, - "cost": 0.0001, - "robustness": 80.0 - } - } - }, - "Science": { - "metrics": { - "easy": { - "accuracy": 96.7, - "cost": 0.0, - "robustness": 60.0 - }, - "medium": { - "accuracy": 52.0, - "cost": 0.0, - "robustness": 0 - }, - "hard": { - "accuracy": 5.1, - "cost": 0.0, - "robustness": 100.0 - }, - "all": { - "accuracy": 39.5, - "cost": 0.0, - "robustness": 75.0 - } - } - } - } - }, - "Social Science": { - "metrics": { - "easy": { - "accuracy": 94.3, - "cost": 0.0001, - "robustness": 50.0 - }, - "medium": { - "accuracy": 63.6, - "cost": 0.0001, - "robustness": 25.0 - }, - "hard": { - "accuracy": 7.4, - "cost": 0.0002, - "robustness": 42.9 - }, - "all": { - "accuracy": 64.7, - "cost": 0.0001, - "robustness": 40.5 - } - }, - "subcategories": { - "Economics": { - "metrics": { - "easy": { - "accuracy": 97.3, - "cost": 0.0001, - "robustness": 41.7 - }, - "medium": { - "accuracy": 73.6, - "cost": 0.0002, - "robustness": 33.3 - }, - "hard": { - "accuracy": 8.8, - "cost": 0.0003, - "robustness": 33.3 - }, - "all": { - "accuracy": 71.6, - "cost": 0.0001, - "robustness": 38.1 - } - } - }, - "Law": { - "metrics": { - "easy": { - "accuracy": 87.7, - "cost": 0.0002, - "robustness": 33.3 - }, - "medium": { - "accuracy": 47.6, - "cost": 0.0002, - "robustness": 0.0 - }, - "hard": { - "accuracy": 6.7, - "cost": 0.0002, - "robustness": 50.0 - }, - "all": { - "accuracy": 56.3, - "cost": 0.0002, - "robustness": 22.2 - } - } - }, - "Social problems": { - "metrics": { - "easy": { - "accuracy": 93.6, - "cost": 0.0, - "robustness": 100.0 - }, - "medium": { - "accuracy": 33.3, - "cost": 0.0, - "robustness": 0 - }, - "hard": { - "accuracy": 0.0, - "cost": 0.0, - "robustness": 100.0 - }, - "all": { - "accuracy": 77.0, - "cost": 0.0, - "robustness": 100.0 - } - } - }, - "Social sciences, sociology, and anthropology": { - "metrics": { - "easy": { - "accuracy": 66.7, - "cost": 0.0, - "robustness": 0 - }, - "medium": { - "accuracy": 44.4, - "cost": 0.0, - "robustness": 50.0 - }, - "hard": { - "accuracy": 5.9, - "cost": 0.0, - "robustness": 0.0 - }, - "all": { - "accuracy": 21.8, - "cost": 0.0, - "robustness": 33.3 - } - } - } - } - }, - "Technology": { - "metrics": { - "easy": { - "accuracy": 94.0, - "cost": 0.0001, - "robustness": 55.8 - }, - "medium": { - "accuracy": 63.3, - "cost": 0.0001, - "robustness": 42.9 - }, - "hard": { - "accuracy": 19.7, - "cost": 0.0002, - "robustness": 14.3 - }, - "all": { - "accuracy": 76.5, - "cost": 0.0001, - "robustness": 47.9 - } - }, - "subcategories": { - "Engineering": { - "metrics": { - "easy": { - "accuracy": 96.0, - "cost": 0.0001, - "robustness": 60.0 - }, - "medium": { - "accuracy": 80.2, - "cost": 0.0002, - "robustness": 46.2 - }, - "hard": { - "accuracy": 42.9, - "cost": 0.0003, - "robustness": 100.0 - }, - "all": { - "accuracy": 81.5, - "cost": 0.0002, - "robustness": 52.6 - } - } - }, - "Management and public relations": { - "metrics": { - "easy": { - "accuracy": 98.3, - "cost": 0.0001, - "robustness": 75.0 - }, - "medium": { - "accuracy": 63.6, - "cost": 0.0, - "robustness": 0.0 - }, - "hard": { - "accuracy": 0.0, - "cost": 0.0, - "robustness": 0 - }, - "all": { - "accuracy": 90.4, - "cost": 0.0001, - "robustness": 60.0 - } - } - }, - "Medicine and health": { - "metrics": { - "easy": { - "accuracy": 93.2, - "cost": 0.0001, - "robustness": 52.9 - }, - "medium": { - "accuracy": 51.7, - "cost": 0.0001, - "robustness": 42.9 - }, - "hard": { - "accuracy": 11.5, - "cost": 0.0001, - "robustness": 0.0 - }, - "all": { - "accuracy": 73.8, - "cost": 0.0001, - "robustness": 44.7 - } - } - } - } - } - } - }, - "sqwish-router": { - "metrics": { - "easy": { - "accuracy": 97.3, - "cost": 0.0002, - "robustness": 51.0 - }, - "medium": { - "accuracy": 79.3, - "cost": 0.0008, - "robustness": 44.2 - }, - "hard": { - "accuracy": 45.2, - "cost": 0.0015, - "robustness": 60.6 - }, - "all": { - "accuracy": 79.9, - "cost": 0.0007, - "robustness": 51.7 - } - }, - "categories": { - "Arts & recreation": { - "metrics": { - "easy": { - "accuracy": 97.0, - "cost": 0.0006, - "robustness": 50.0 - }, - "medium": { - "accuracy": 76.5, - "cost": 0.0019, - "robustness": 9.1 - }, - "hard": { - "accuracy": 33.3, - "cost": 0.0026, - "robustness": 60.0 - }, - "all": { - "accuracy": 76.1, - "cost": 0.0015, - "robustness": 33.3 - } - }, - "subcategories": { - "Arts": { - "metrics": { - "easy": { - "accuracy": 98.3, - "cost": 0.0004, - "robustness": 100.0 - }, - "medium": { - "accuracy": 94.9, - "cost": 0.0014, - "robustness": 0.0 - }, - "hard": { - "accuracy": 33.9, - "cost": 0.0034, - "robustness": 100.0 - }, - "all": { - "accuracy": 74.0, - "cost": 0.0018, - "robustness": 60.0 - } - } - }, - "Music": { - "metrics": { - "easy": { - "accuracy": 97.0, - "cost": 0.0004, - "robustness": 60.0 - }, - "medium": { - "accuracy": 58.6, - "cost": 0.0019, - "robustness": 20.0 - }, - "hard": { - "accuracy": 40.0, - "cost": 0.0029, - "robustness": 0.0 - }, - "all": { - "accuracy": 72.5, - "cost": 0.0014, - "robustness": 36.4 - } - } - }, - "Sports, games and entertainment": { - "metrics": { - "easy": { - "accuracy": 96.4, - "cost": 0.0008, - "robustness": 0.0 - }, - "medium": { - "accuracy": 90.0, - "cost": 0.0022, - "robustness": 0.0 - }, - "hard": { - "accuracy": 30.3, - "cost": 0.0018, - "robustness": 50.0 - }, - "all": { - "accuracy": 80.1, - "cost": 0.0015, - "robustness": 12.5 - } - } - } - } - }, - "Computer science, information, and general works": { - "metrics": { - "easy": { - "accuracy": 98.0, - "cost": 0.0002, - "robustness": 53.8 - }, - "medium": { - "accuracy": 87.2, - "cost": 0.0005, - "robustness": 57.1 - }, - "hard": { - "accuracy": 59.2, - "cost": 0.0012, - "robustness": 60.0 - }, - "all": { - "accuracy": 87.6, - "cost": 0.0005, - "robustness": 56.5 - } - }, - "subcategories": { - "Computer science, knowledge, and systems": { - "metrics": { - "easy": { - "accuracy": 98.6, - "cost": 0.0002, - "robustness": 53.8 - }, - "medium": { - "accuracy": 91.2, - "cost": 0.0006, - "robustness": 52.9 - }, - "hard": { - "accuracy": 66.5, - "cost": 0.0014, - "robustness": 61.5 - }, - "all": { - "accuracy": 89.1, - "cost": 0.0006, - "robustness": 55.8 - } - } - }, - "Library and information sciences": { - "metrics": { - "easy": { - "accuracy": 97.1, - "cost": 0.0002, - "robustness": 53.8 - }, - "medium": { - "accuracy": 69.4, - "cost": 0.0003, - "robustness": 75.0 - }, - "hard": { - "accuracy": 16.7, - "cost": 0.0002, - "robustness": 50.0 - }, - "all": { - "accuracy": 83.8, - "cost": 0.0002, - "robustness": 57.9 - } - } - } - } - }, - "History": { - "metrics": { - "easy": { - "accuracy": 97.6, - "cost": 0.0003, - "robustness": 39.1 - }, - "medium": { - "accuracy": 81.4, - "cost": 0.001, - "robustness": 33.3 - }, - "hard": { - "accuracy": 29.5, - "cost": 0.0021, - "robustness": 40.0 - }, - "all": { - "accuracy": 79.0, - "cost": 0.0009, - "robustness": 38.5 - } - }, - "subcategories": { - "Biography and genealogy": { - "metrics": { - "easy": { - "accuracy": 100.0, - "cost": 0.0003, - "robustness": 0.0 - }, - "medium": { - "accuracy": 100.0, - "cost": 0.0002, - "robustness": 0 - }, - "hard": { - "accuracy": 0.0, - "cost": 0.0001, - "robustness": 0 - }, - "all": { - "accuracy": 98.1, - "cost": 0.0003, - "robustness": 0.0 - } - } - }, - "Geography": { - "metrics": { - "easy": { - "accuracy": 97.2, - "cost": 0.0003, - "robustness": 37.5 - }, - "medium": { - "accuracy": 65.5, - "cost": 0.0014, - "robustness": 0.0 - }, - "hard": { - "accuracy": 58.3, - "cost": 0.0027, - "robustness": 0 - }, - "all": { - "accuracy": 88.0, - "cost": 0.0007, - "robustness": 33.3 - } - } - }, - "History": { - "metrics": { - "easy": { - "accuracy": 97.3, - "cost": 0.0004, - "robustness": 54.5 - }, - "medium": { - "accuracy": 83.5, - "cost": 0.0009, - "robustness": 40.0 - }, - "hard": { - "accuracy": 27.2, - "cost": 0.0021, - "robustness": 40.0 - }, - "all": { - "accuracy": 74.3, - "cost": 0.001, - "robustness": 46.2 - } - } - } - } - }, - "Language": { - "metrics": { - "easy": { - "accuracy": 93.3, - "cost": 0.0002, - "robustness": 54.5 - }, - "medium": { - "accuracy": 55.7, - "cost": 0.0002, - "robustness": 55.6 - }, - "hard": { - "accuracy": 65.1, - "cost": 0.0007, - "robustness": 52.0 - }, - "all": { - "accuracy": 71.3, - "cost": 0.0004, - "robustness": 53.3 - } - }, - "subcategories": { - "Language": { - "metrics": { - "easy": { - "accuracy": 93.3, - "cost": 0.0002, - "robustness": 54.5 - }, - "medium": { - "accuracy": 55.7, - "cost": 0.0002, - "robustness": 55.6 - }, - "hard": { - "accuracy": 65.1, - "cost": 0.0007, - "robustness": 52.0 - }, - "all": { - "accuracy": 71.3, - "cost": 0.0004, - "robustness": 53.3 - } - } - } - } - }, - "Literature": { - "metrics": { - "easy": { - "accuracy": 100.0, - "cost": 0.0003, - "robustness": 66.7 - }, - "medium": { - "accuracy": 88.2, - "cost": 0.0014, - "robustness": 50.0 - }, - "hard": { - "accuracy": 47.1, - "cost": 0.0019, - "robustness": 58.3 - }, - "all": { - "accuracy": 59.7, - "cost": 0.0016, - "robustness": 58.6 - } - }, - "subcategories": { - "Literature, rhetoric and criticism": { - "metrics": { - "easy": { - "accuracy": 100.0, - "cost": 0.0003, - "robustness": 66.7 - }, - "medium": { - "accuracy": 88.2, - "cost": 0.0014, - "robustness": 50.0 - }, - "hard": { - "accuracy": 47.1, - "cost": 0.0019, - "robustness": 58.3 - }, - "all": { - "accuracy": 59.7, - "cost": 0.0016, - "robustness": 58.6 - } - } - } - } - }, - "Philosophy and psychology": { - "metrics": { - "easy": { - "accuracy": 96.3, - "cost": 0.0002, - "robustness": 37.0 - }, - "medium": { - "accuracy": 76.8, - "cost": 0.0005, - "robustness": 30.0 - }, - "hard": { - "accuracy": 38.3, - "cost": 0.002, - "robustness": 42.9 - }, - "all": { - "accuracy": 83.7, - "cost": 0.0005, - "robustness": 36.4 - } - }, - "subcategories": { - "Ethics": { - "metrics": { - "easy": { - "accuracy": 93.0, - "cost": 0.0001, - "robustness": 28.6 - }, - "medium": { - "accuracy": 55.3, - "cost": 0.0002, - "robustness": 0.0 - }, - "hard": { - "accuracy": 0.0, - "cost": 0.0, - "robustness": 50.0 - }, - "all": { - "accuracy": 77.4, - "cost": 0.0002, - "robustness": 23.8 - } - } - }, - "Philosophical logic": { - "metrics": { - "easy": { - "accuracy": 100.0, - "cost": 0.0002, - "robustness": 75.0 - }, - "medium": { - "accuracy": 97.4, - "cost": 0.0004, - "robustness": 66.7 - }, - "hard": { - "accuracy": 60.0, - "cost": 0.0004, - "robustness": 0 - }, - "all": { - "accuracy": 97.2, - "cost": 0.0003, - "robustness": 71.4 - } - } - }, - "Philosophy": { - "metrics": { - "easy": { - "accuracy": 97.3, - "cost": 0.0006, - "robustness": 66.7 - }, - "medium": { - "accuracy": 92.5, - "cost": 0.0012, - "robustness": 50.0 - }, - "hard": { - "accuracy": 48.9, - "cost": 0.0032, - "robustness": 33.3 - }, - "all": { - "accuracy": 77.9, - "cost": 0.0018, - "robustness": 50.0 - } - } - }, - "Psychology": { - "metrics": { - "easy": { - "accuracy": 99.2, - "cost": 0.0003, - "robustness": 16.7 - }, - "medium": { - "accuracy": 85.4, - "cost": 0.0004, - "robustness": 0 - }, - "hard": { - "accuracy": 33.3, - "cost": 0.0007, - "robustness": 50.0 - }, - "all": { - "accuracy": 89.3, - "cost": 0.0004, - "robustness": 25.0 - } - } - } - } - }, - "Science": { - "metrics": { - "easy": { - "accuracy": 99.4, - "cost": 0.0002, - "robustness": 64.1 - }, - "medium": { - "accuracy": 82.9, - "cost": 0.0006, - "robustness": 23.8 - }, - "hard": { - "accuracy": 38.1, - "cost": 0.0014, - "robustness": 77.8 - }, - "all": { - "accuracy": 85.8, - "cost": 0.0005, - "robustness": 53.6 - } - }, - "subcategories": { - "Animals (Zoology)": { - "metrics": { - "easy": { - "accuracy": 100.0, - "cost": 0.0, - "robustness": 100.0 - }, - "medium": { - "accuracy": 60.0, - "cost": 0.0002, - "robustness": 0 - }, - "hard": { - "accuracy": 0, - "cost": 0, - "robustness": 0 - }, - "all": { - "accuracy": 92.0, - "cost": 0.0001, - "robustness": 100.0 - } - } - }, - "Biology": { - "metrics": { - "easy": { - "accuracy": 100.0, - "cost": 0.0002, - "robustness": 71.4 - }, - "medium": { - "accuracy": 76.5, - "cost": 0.0006, - "robustness": 0 - }, - "hard": { - "accuracy": 33.3, - "cost": 0.0014, - "robustness": 0 - }, - "all": { - "accuracy": 93.0, - "cost": 0.0003, - "robustness": 71.4 - } - } - }, - "Chemistry": { - "metrics": { - "easy": { - "accuracy": 100.0, - "cost": 0.0003, - "robustness": 66.7 - }, - "medium": { - "accuracy": 88.2, - "cost": 0.0004, - "robustness": 50.0 - }, - "hard": { - "accuracy": 23.1, - "cost": 0.0005, - "robustness": 0 - }, - "all": { - "accuracy": 84.4, - "cost": 0.0004, - "robustness": 60.0 - } - } - }, - "Earth sciences and geology": { - "metrics": { - "easy": { - "accuracy": 99.2, - "cost": 0.0001, - "robustness": 62.5 - }, - "medium": { - "accuracy": 76.3, - "cost": 0.0004, - "robustness": 16.7 - }, - "hard": { - "accuracy": 23.1, - "cost": 0.0002, - "robustness": 100.0 - }, - "all": { - "accuracy": 92.1, - "cost": 0.0002, - "robustness": 46.7 - } - } - }, - "Mathematics": { - "metrics": { - "easy": { - "accuracy": 99.2, - "cost": 0.0001, - "robustness": 72.7 - }, - "medium": { - "accuracy": 85.2, - "cost": 0.0006, - "robustness": 27.3 - }, - "hard": { - "accuracy": 57.3, - "cost": 0.0013, - "robustness": 80.0 - }, - "all": { - "accuracy": 87.2, - "cost": 0.0005, - "robustness": 55.6 - } - } - }, - "Physics": { - "metrics": { - "easy": { - "accuracy": 97.7, - "cost": 0.0004, - "robustness": 33.3 - }, - "medium": { - "accuracy": 88.1, - "cost": 0.0007, - "robustness": 0.0 - }, - "hard": { - "accuracy": 54.5, - "cost": 0.0007, - "robustness": 0 - }, - "all": { - "accuracy": 88.7, - "cost": 0.0006, - "robustness": 20.0 - } - } - }, - "Science": { - "metrics": { - "easy": { - "accuracy": 100.0, - "cost": 0.0001, - "robustness": 40.0 - }, - "medium": { - "accuracy": 68.0, - "cost": 0.0016, - "robustness": 0 - }, - "hard": { - "accuracy": 13.6, - "cost": 0.0022, - "robustness": 66.7 - }, - "all": { - "accuracy": 48.2, - "cost": 0.0015, - "robustness": 50.0 - } - } - } - } - }, - "Social Science": { - "metrics": { - "easy": { - "accuracy": 97.0, - "cost": 0.0003, - "robustness": 50.0 - }, - "medium": { - "accuracy": 77.9, - "cost": 0.0008, - "robustness": 50.0 - }, - "hard": { - "accuracy": 25.5, - "cost": 0.0013, - "robustness": 85.7 - }, - "all": { - "accuracy": 74.9, - "cost": 0.0007, - "robustness": 56.8 - } - }, - "subcategories": { - "Economics": { - "metrics": { - "easy": { - "accuracy": 98.4, - "cost": 0.0002, - "robustness": 58.3 - }, - "medium": { - "accuracy": 82.8, - "cost": 0.0006, - "robustness": 66.7 - }, - "hard": { - "accuracy": 25.0, - "cost": 0.0008, - "robustness": 66.7 - }, - "all": { - "accuracy": 78.6, - "cost": 0.0005, - "robustness": 61.9 - } - } - }, - "Law": { - "metrics": { - "easy": { - "accuracy": 93.8, - "cost": 0.0006, - "robustness": 33.3 - }, - "medium": { - "accuracy": 74.6, - "cost": 0.0011, - "robustness": 50.0 - }, - "hard": { - "accuracy": 26.7, - "cost": 0.0008, - "robustness": 100.0 - }, - "all": { - "accuracy": 73.4, - "cost": 0.0009, - "robustness": 55.6 - } - } - }, - "Social problems": { - "metrics": { - "easy": { - "accuracy": 95.7, - "cost": 0.0001, - "robustness": 33.3 - }, - "medium": { - "accuracy": 22.2, - "cost": 0.0, - "robustness": 0 - }, - "hard": { - "accuracy": 20.0, - "cost": 0.0, - "robustness": 100.0 - }, - "all": { - "accuracy": 78.7, - "cost": 0.0001, - "robustness": 50.0 - } - } - }, - "Social sciences, sociology, and anthropology": { - "metrics": { - "easy": { - "accuracy": 100.0, - "cost": 0.0008, - "robustness": 0 - }, - "medium": { - "accuracy": 72.2, - "cost": 0.0014, - "robustness": 0.0 - }, - "hard": { - "accuracy": 26.5, - "cost": 0.0029, - "robustness": 100.0 - }, - "all": { - "accuracy": 45.5, - "cost": 0.0023, - "robustness": 33.3 - } - } - } - } - }, - "Technology": { - "metrics": { - "easy": { - "accuracy": 95.9, - "cost": 0.0002, - "robustness": 51.2 - }, - "medium": { - "accuracy": 78.1, - "cost": 0.0007, - "robustness": 71.4 - }, - "hard": { - "accuracy": 33.1, - "cost": 0.0008, - "robustness": 100.0 - }, - "all": { - "accuracy": 83.5, - "cost": 0.0004, - "robustness": 62.0 - } - }, - "subcategories": { - "Engineering": { - "metrics": { - "easy": { - "accuracy": 97.6, - "cost": 0.0004, - "robustness": 60.0 - }, - "medium": { - "accuracy": 93.4, - "cost": 0.0009, - "robustness": 76.9 - }, - "hard": { - "accuracy": 64.3, - "cost": 0.0014, - "robustness": 100.0 - }, - "all": { - "accuracy": 91.3, - "cost": 0.0008, - "robustness": 73.7 - } - } - }, - "Management and public relations": { - "metrics": { - "easy": { - "accuracy": 95.0, - "cost": 0.0001, - "robustness": 50.0 - }, - "medium": { - "accuracy": 45.5, - "cost": 0.0002, - "robustness": 0.0 - }, - "hard": { - "accuracy": 0.0, - "cost": 0.0, - "robustness": 0 - }, - "all": { - "accuracy": 84.9, - "cost": 0.0001, - "robustness": 40.0 - } - } - }, - "Medicine and health": { - "metrics": { - "easy": { - "accuracy": 95.6, - "cost": 0.0002, - "robustness": 50.0 - }, - "medium": { - "accuracy": 69.0, - "cost": 0.0005, - "robustness": 71.4 - }, - "hard": { - "accuracy": 22.1, - "cost": 0.0006, - "robustness": 100.0 - }, - "all": { - "accuracy": 80.7, - "cost": 0.0003, - "robustness": 59.6 - } - } - } - } - } - } - }, - "agentforge": { - "metrics": { - "easy": { - "accuracy": 96.5, - "cost": 0.0001, - "robustness": 40.9 - }, - "medium": { - "accuracy": 73.0, - "cost": 0.0002, - "robustness": 42.5 - }, - "hard": { - "accuracy": 33.0, - "cost": 0.0002, - "robustness": 37.6 - }, - "all": { - "accuracy": 74.8, - "cost": 0.0001, - "robustness": 40.5 - } - }, - "categories": { - "Arts & recreation": { - "metrics": { - "easy": { - "accuracy": 95.6, - "cost": 0.0001, - "robustness": 62.5 - }, - "medium": { - "accuracy": 55.7, - "cost": 0.0002, - "robustness": 27.3 - }, - "hard": { - "accuracy": 14.3, - "cost": 0.0002, - "robustness": 20.0 - }, - "all": { - "accuracy": 64.0, - "cost": 0.0001, - "robustness": 37.5 - } - }, - "subcategories": { - "Arts": { - "metrics": { - "easy": { - "accuracy": 96.6, - "cost": 0.0001, - "robustness": 0.0 - }, - "medium": { - "accuracy": 82.1, - "cost": 0.0001, - "robustness": 50.0 - }, - "hard": { - "accuracy": 8.9, - "cost": 0.0, - "robustness": 50.0 - }, - "all": { - "accuracy": 61.0, - "cost": 0.0001, - "robustness": 40.0 - } - } - }, - "Music": { - "metrics": { - "easy": { - "accuracy": 98.0, - "cost": 0.0001, - "robustness": 80.0 - }, - "medium": { - "accuracy": 45.7, - "cost": 0.0003, - "robustness": 40.0 - }, - "hard": { - "accuracy": 24.0, - "cost": 0.0002, - "robustness": 0.0 - }, - "all": { - "accuracy": 65.0, - "cost": 0.0002, - "robustness": 54.5 - } - } - }, - "Sports, games and entertainment": { - "metrics": { - "easy": { - "accuracy": 93.6, - "cost": 0.0001, - "robustness": 50.0 - }, - "medium": { - "accuracy": 57.0, - "cost": 0.0001, - "robustness": 0.0 - }, - "hard": { - "accuracy": 15.2, - "cost": 0.0003, - "robustness": 0.0 - }, - "all": { - "accuracy": 64.7, - "cost": 0.0001, - "robustness": 12.5 - } - } - } - } - }, - "Computer science, information, and general works": { - "metrics": { - "easy": { - "accuracy": 98.1, - "cost": 0.0001, - "robustness": 34.6 - }, - "medium": { - "accuracy": 82.0, - "cost": 0.0001, - "robustness": 33.3 - }, - "hard": { - "accuracy": 40.0, - "cost": 0.0002, - "robustness": 46.7 - }, - "all": { - "accuracy": 82.6, - "cost": 0.0001, - "robustness": 37.1 - } - }, - "subcategories": { - "Computer science, knowledge, and systems": { - "metrics": { - "easy": { - "accuracy": 98.6, - "cost": 0.0001, - "robustness": 23.1 - }, - "medium": { - "accuracy": 85.4, - "cost": 0.0002, - "robustness": 29.4 - }, - "hard": { - "accuracy": 45.0, - "cost": 0.0003, - "robustness": 46.2 - }, - "all": { - "accuracy": 82.5, - "cost": 0.0001, - "robustness": 32.6 - } - } - }, - "Library and information sciences": { - "metrics": { - "easy": { - "accuracy": 97.5, - "cost": 0.0, - "robustness": 46.2 - }, - "medium": { - "accuracy": 67.1, - "cost": 0.0, - "robustness": 50.0 - }, - "hard": { - "accuracy": 11.1, - "cost": 0.0, - "robustness": 50.0 - }, - "all": { - "accuracy": 83.1, - "cost": 0.0, - "robustness": 47.4 - } - } - } - } - }, - "History": { - "metrics": { - "easy": { - "accuracy": 96.8, - "cost": 0.0001, - "robustness": 60.9 - }, - "medium": { - "accuracy": 71.8, - "cost": 0.0001, - "robustness": 66.7 - }, - "hard": { - "accuracy": 13.4, - "cost": 0.0001, - "robustness": 60.0 - }, - "all": { - "accuracy": 72.7, - "cost": 0.0001, - "robustness": 61.5 - } - }, - "subcategories": { - "Biography and genealogy": { - "metrics": { - "easy": { - "accuracy": 100.0, - "cost": 0.0001, - "robustness": 50.0 - }, - "medium": { - "accuracy": 77.8, - "cost": 0.0001, - "robustness": 0 - }, - "hard": { - "accuracy": 0.0, - "cost": 0.0, - "robustness": 0 - }, - "all": { - "accuracy": 94.2, - "cost": 0.0001, - "robustness": 50.0 - } - } - }, - "Geography": { - "metrics": { - "easy": { - "accuracy": 98.2, - "cost": 0.0001, - "robustness": 87.5 - }, - "medium": { - "accuracy": 58.6, - "cost": 0.0001, - "robustness": 100.0 - }, - "hard": { - "accuracy": 33.3, - "cost": 0.0, - "robustness": 0 - }, - "all": { - "accuracy": 85.3, - "cost": 0.0001, - "robustness": 88.9 - } - } - }, - "History": { - "metrics": { - "easy": { - "accuracy": 95.5, - "cost": 0.0001, - "robustness": 45.5 + "metrics": { + "easy": { + "accuracy": 97.3, + "cost": 0.0004, + "robustness": 54.5 }, "medium": { - "accuracy": 74.1, - "cost": 0.0001, - "robustness": 60.0 + "accuracy": 83.5, + "cost": 0.0009, + "robustness": 40.0 }, "hard": { - "accuracy": 11.8, - "cost": 0.0001, - "robustness": 60.0 + "accuracy": 27.2, + "cost": 0.0021, + "robustness": 40.0 }, "all": { - "accuracy": 66.7, - "cost": 0.0001, - "robustness": 53.8 + "accuracy": 74.3, + "cost": 0.001, + "robustness": 46.2 } } } @@ -15160,48 +13296,48 @@ "Language": { "metrics": { "easy": { - "accuracy": 90.9, - "cost": 0.0001, - "robustness": 27.3 + "accuracy": 93.3, + "cost": 0.0002, + "robustness": 54.5 }, "medium": { - "accuracy": 65.3, - "cost": 0.0001, - "robustness": 77.8 + "accuracy": 55.7, + "cost": 0.0002, + "robustness": 55.6 }, "hard": { - "accuracy": 58.3, - "cost": 0.0001, - "robustness": 32.0 + "accuracy": 65.1, + "cost": 0.0007, + "robustness": 52.0 }, "all": { - "accuracy": 69.7, - "cost": 0.0001, - "robustness": 40.0 + "accuracy": 71.3, + "cost": 0.0004, + "robustness": 53.3 } }, "subcategories": { "Language": { "metrics": { "easy": { - "accuracy": 90.9, - "cost": 0.0001, - "robustness": 27.3 + "accuracy": 93.3, + "cost": 0.0002, + "robustness": 54.5 }, "medium": { - "accuracy": 65.3, - "cost": 0.0001, - "robustness": 77.8 + "accuracy": 55.7, + "cost": 0.0002, + "robustness": 55.6 }, "hard": { - "accuracy": 58.3, - "cost": 0.0001, - "robustness": 32.0 + "accuracy": 65.1, + "cost": 0.0007, + "robustness": 52.0 }, "all": { - "accuracy": 69.7, - "cost": 0.0001, - "robustness": 40.0 + "accuracy": 71.3, + "cost": 0.0004, + "robustness": 53.3 } } } @@ -15211,23 +13347,23 @@ "metrics": { "easy": { "accuracy": 100.0, - "cost": 0.0001, - "robustness": 0.0 + "cost": 0.0003, + "robustness": 66.7 }, "medium": { - "accuracy": 76.8, - "cost": 0.0001, + "accuracy": 88.2, + "cost": 0.0014, "robustness": 50.0 }, "hard": { - "accuracy": 33.9, - "cost": 0.0001, - "robustness": 25.0 + "accuracy": 47.1, + "cost": 0.0019, + "robustness": 58.3 }, "all": { - "accuracy": 48.7, - "cost": 0.0001, - "robustness": 24.1 + "accuracy": 59.7, + "cost": 0.0016, + "robustness": 58.6 } }, "subcategories": { @@ -15235,23 +13371,23 @@ "metrics": { "easy": { "accuracy": 100.0, - "cost": 0.0001, - "robustness": 0.0 + "cost": 0.0003, + "robustness": 66.7 }, "medium": { - "accuracy": 76.8, - "cost": 0.0001, + "accuracy": 88.2, + "cost": 0.0014, "robustness": 50.0 }, "hard": { - "accuracy": 33.9, - "cost": 0.0001, - "robustness": 25.0 + "accuracy": 47.1, + "cost": 0.0019, + "robustness": 58.3 }, "all": { - "accuracy": 48.7, - "cost": 0.0001, - "robustness": 24.1 + "accuracy": 59.7, + "cost": 0.0016, + "robustness": 58.6 } } } @@ -15260,119 +13396,119 @@ "Philosophy and psychology": { "metrics": { "easy": { - "accuracy": 95.1, - "cost": 0.0001, - "robustness": 33.3 + "accuracy": 96.3, + "cost": 0.0002, + "robustness": 37.0 }, "medium": { - "accuracy": 74.4, - "cost": 0.0001, + "accuracy": 76.8, + "cost": 0.0005, "robustness": 30.0 }, "hard": { - "accuracy": 34.6, - "cost": 0.0002, - "robustness": 28.6 + "accuracy": 38.3, + "cost": 0.002, + "robustness": 42.9 }, "all": { - "accuracy": 81.9, - "cost": 0.0001, - "robustness": 31.8 + "accuracy": 83.7, + "cost": 0.0005, + "robustness": 36.4 } }, "subcategories": { "Ethics": { "metrics": { "easy": { - "accuracy": 91.4, + "accuracy": 93.0, "cost": 0.0001, - "robustness": 35.7 + "robustness": 28.6 }, "medium": { - "accuracy": 60.0, - "cost": 0.0001, - "robustness": 40.0 + "accuracy": 55.3, + "cost": 0.0002, + "robustness": 0.0 }, "hard": { - "accuracy": 76.9, + "accuracy": 0.0, "cost": 0.0, "robustness": 50.0 }, "all": { - "accuracy": 81.3, - "cost": 0.0001, - "robustness": 38.1 + "accuracy": 77.4, + "cost": 0.0002, + "robustness": 23.8 } } }, "Philosophical logic": { "metrics": { "easy": { - "accuracy": 98.5, - "cost": 0.0001, - "robustness": 25.0 + "accuracy": 100.0, + "cost": 0.0002, + "robustness": 75.0 }, "medium": { - "accuracy": 94.7, - "cost": 0.0003, - "robustness": 0.0 + "accuracy": 97.4, + "cost": 0.0004, + "robustness": 66.7 }, "hard": { - "accuracy": 80.0, - "cost": 0.0006, + "accuracy": 60.0, + "cost": 0.0004, "robustness": 0 }, "all": { - "accuracy": 96.3, - "cost": 0.0002, - "robustness": 14.3 + "accuracy": 97.2, + "cost": 0.0003, + "robustness": 71.4 } } }, "Philosophy": { "metrics": { "easy": { - "accuracy": 100.0, - "cost": 0.0001, - "robustness": 33.3 + "accuracy": 97.3, + "cost": 0.0006, + "robustness": 66.7 }, "medium": { - "accuracy": 75.0, - "cost": 0.0001, + "accuracy": 92.5, + "cost": 0.0012, "robustness": 50.0 }, "hard": { - "accuracy": 22.2, - "cost": 0.0002, + "accuracy": 48.9, + "cost": 0.0032, "robustness": 33.3 }, "all": { - "accuracy": 63.1, - "cost": 0.0001, - "robustness": 37.5 + "accuracy": 77.9, + "cost": 0.0018, + "robustness": 50.0 } } }, "Psychology": { "metrics": { "easy": { - "accuracy": 97.5, - "cost": 0.0001, - "robustness": 33.3 + "accuracy": 99.2, + "cost": 0.0003, + "robustness": 16.7 }, "medium": { - "accuracy": 83.3, - "cost": 0.0001, + "accuracy": 85.4, + "cost": 0.0004, "robustness": 0 }, "hard": { - "accuracy": 22.2, - "cost": 0.0001, - "robustness": 0.0 + "accuracy": 33.3, + "cost": 0.0007, + "robustness": 50.0 }, "all": { - "accuracy": 86.6, - "cost": 0.0001, + "accuracy": 89.3, + "cost": 0.0004, "robustness": 25.0 } } @@ -15382,24 +13518,24 @@ "Science": { "metrics": { "easy": { - "accuracy": 98.4, - "cost": 0.0001, - "robustness": 43.6 + "accuracy": 99.4, + "cost": 0.0002, + "robustness": 64.1 }, "medium": { - "accuracy": 82.6, - "cost": 0.0002, - "robustness": 38.1 + "accuracy": 82.9, + "cost": 0.0006, + "robustness": 23.8 }, "hard": { - "accuracy": 27.8, - "cost": 0.0006, - "robustness": 66.7 + "accuracy": 38.1, + "cost": 0.0014, + "robustness": 77.8 }, "all": { - "accuracy": 83.8, - "cost": 0.0002, - "robustness": 44.9 + "accuracy": 85.8, + "cost": 0.0005, + "robustness": 53.6 } }, "subcategories": { @@ -15407,12 +13543,12 @@ "metrics": { "easy": { "accuracy": 100.0, - "cost": 0.0001, - "robustness": 50.0 + "cost": 0.0, + "robustness": 100.0 }, "medium": { "accuracy": 60.0, - "cost": 0.0, + "cost": 0.0002, "robustness": 0 }, "hard": { @@ -15423,7 +13559,7 @@ "all": { "accuracy": 92.0, "cost": 0.0001, - "robustness": 50.0 + "robustness": 100.0 } } }, @@ -15431,142 +13567,142 @@ "metrics": { "easy": { "accuracy": 100.0, - "cost": 0.0001, - "robustness": 57.1 + "cost": 0.0002, + "robustness": 71.4 }, "medium": { "accuracy": 76.5, - "cost": 0.0001, + "cost": 0.0006, "robustness": 0 }, "hard": { - "accuracy": 22.2, - "cost": 0.0004, + "accuracy": 33.3, + "cost": 0.0014, "robustness": 0 }, "all": { - "accuracy": 92.3, - "cost": 0.0001, - "robustness": 57.1 + "accuracy": 93.0, + "cost": 0.0003, + "robustness": 71.4 } } }, "Chemistry": { "metrics": { "easy": { - "accuracy": 97.7, - "cost": 0.0001, - "robustness": 33.3 + "accuracy": 100.0, + "cost": 0.0003, + "robustness": 66.7 }, "medium": { - "accuracy": 94.1, - "cost": 0.0002, + "accuracy": 88.2, + "cost": 0.0004, "robustness": 50.0 }, "hard": { - "accuracy": 15.4, - "cost": 0.0002, + "accuracy": 23.1, + "cost": 0.0005, "robustness": 0 }, "all": { "accuracy": 84.4, - "cost": 0.0001, - "robustness": 40.0 + "cost": 0.0004, + "robustness": 60.0 } } }, "Earth sciences and geology": { "metrics": { "easy": { - "accuracy": 98.4, + "accuracy": 99.2, "cost": 0.0001, "robustness": 62.5 }, "medium": { - "accuracy": 72.9, - "cost": 0.0001, - "robustness": 50.0 + "accuracy": 76.3, + "cost": 0.0004, + "robustness": 16.7 }, "hard": { - "accuracy": 15.4, - "cost": 0.0001, + "accuracy": 23.1, + "cost": 0.0002, "robustness": 100.0 }, "all": { - "accuracy": 90.6, - "cost": 0.0001, - "robustness": 60.0 + "accuracy": 92.1, + "cost": 0.0002, + "robustness": 46.7 } } }, "Mathematics": { "metrics": { "easy": { - "accuracy": 97.7, + "accuracy": 99.2, "cost": 0.0001, - "robustness": 36.4 + "robustness": 72.7 }, "medium": { - "accuracy": 84.0, - "cost": 0.0003, - "robustness": 36.4 + "accuracy": 85.2, + "cost": 0.0006, + "robustness": 27.3 }, "hard": { - "accuracy": 44.9, - "cost": 0.0012, - "robustness": 60.0 + "accuracy": 57.3, + "cost": 0.0013, + "robustness": 80.0 }, "all": { - "accuracy": 84.2, - "cost": 0.0003, - "robustness": 40.7 + "accuracy": 87.2, + "cost": 0.0005, + "robustness": 55.6 } } }, "Physics": { "metrics": { "easy": { - "accuracy": 100.0, - "cost": 0.0002, - "robustness": 0.0 + "accuracy": 97.7, + "cost": 0.0004, + "robustness": 33.3 }, "medium": { - "accuracy": 90.5, - "cost": 0.0002, + "accuracy": 88.1, + "cost": 0.0007, "robustness": 0.0 }, "hard": { - "accuracy": 36.4, - "cost": 0.0002, + "accuracy": 54.5, + "cost": 0.0007, "robustness": 0 }, "all": { "accuracy": 88.7, - "cost": 0.0002, - "robustness": 0.0 + "cost": 0.0006, + "robustness": 20.0 } } }, "Science": { "metrics": { "easy": { - "accuracy": 96.7, + "accuracy": 100.0, "cost": 0.0001, "robustness": 40.0 }, "medium": { - "accuracy": 72.0, - "cost": 0.0, + "accuracy": 68.0, + "cost": 0.0016, "robustness": 0 }, "hard": { - "accuracy": 6.8, - "cost": 0.0, + "accuracy": 13.6, + "cost": 0.0022, "robustness": 66.7 }, "all": { - "accuracy": 44.7, - "cost": 0.0, + "accuracy": 48.2, + "cost": 0.0015, "robustness": 50.0 } } @@ -15576,71 +13712,71 @@ "Social Science": { "metrics": { "easy": { - "accuracy": 95.6, - "cost": 0.0001, - "robustness": 27.8 + "accuracy": 97.0, + "cost": 0.0003, + "robustness": 50.0 }, "medium": { - "accuracy": 68.0, - "cost": 0.0002, - "robustness": 41.7 + "accuracy": 77.9, + "cost": 0.0008, + "robustness": 50.0 }, "hard": { - "accuracy": 12.8, - "cost": 0.0002, - "robustness": 42.9 + "accuracy": 25.5, + "cost": 0.0013, + "robustness": 85.7 }, "all": { - "accuracy": 68.0, - "cost": 0.0001, - "robustness": 35.1 + "accuracy": 74.9, + "cost": 0.0007, + "robustness": 56.8 } }, "subcategories": { "Economics": { "metrics": { "easy": { - "accuracy": 97.3, - "cost": 0.0001, - "robustness": 16.7 + "accuracy": 98.4, + "cost": 0.0002, + "robustness": 58.3 }, "medium": { - "accuracy": 74.2, - "cost": 0.0001, - "robustness": 33.3 + "accuracy": 82.8, + "cost": 0.0006, + "robustness": 66.7 }, "hard": { - "accuracy": 13.8, - "cost": 0.0002, - "robustness": 33.3 + "accuracy": 25.0, + "cost": 0.0008, + "robustness": 66.7 }, "all": { - "accuracy": 72.8, - "cost": 0.0001, - "robustness": 23.8 + "accuracy": 78.6, + "cost": 0.0005, + "robustness": 61.9 } } }, "Law": { "metrics": { "easy": { - "accuracy": 90.8, - "cost": 0.0001, - "robustness": 66.7 + "accuracy": 93.8, + "cost": 0.0006, + "robustness": 33.3 }, "medium": { - "accuracy": 58.7, - "cost": 0.0002, + "accuracy": 74.6, + "cost": 0.0011, "robustness": 50.0 }, "hard": { - "accuracy": 16.7, - "cost": 0.0002, - "robustness": 50.0 + "accuracy": 26.7, + "cost": 0.0008, + "robustness": 100.0 }, "all": { - "accuracy": 63.9, - "cost": 0.0002, + "accuracy": 73.4, + "cost": 0.0009, "robustness": 55.6 } } @@ -15649,23 +13785,23 @@ "metrics": { "easy": { "accuracy": 95.7, - "cost": 0.0, + "cost": 0.0001, "robustness": 33.3 }, "medium": { - "accuracy": 11.1, + "accuracy": 22.2, "cost": 0.0, "robustness": 0 }, "hard": { "accuracy": 20.0, "cost": 0.0, - "robustness": 0.0 + "robustness": 100.0 }, "all": { - "accuracy": 77.0, - "cost": 0.0, - "robustness": 25.0 + "accuracy": 78.7, + "cost": 0.0001, + "robustness": 50.0 } } }, @@ -15673,23 +13809,23 @@ "metrics": { "easy": { "accuracy": 100.0, - "cost": 0.0, + "cost": 0.0008, "robustness": 0 }, "medium": { "accuracy": 72.2, - "cost": 0.0, - "robustness": 50.0 + "cost": 0.0014, + "robustness": 0.0 }, "hard": { - "accuracy": 5.9, - "cost": 0.0, + "accuracy": 26.5, + "cost": 0.0029, "robustness": 100.0 }, "all": { - "accuracy": 32.7, - "cost": 0.0, - "robustness": 66.7 + "accuracy": 45.5, + "cost": 0.0023, + "robustness": 33.3 } } } @@ -15698,61 +13834,61 @@ "Technology": { "metrics": { "easy": { - "accuracy": 95.3, - "cost": 0.0001, - "robustness": 44.2 + "accuracy": 95.9, + "cost": 0.0002, + "robustness": 51.2 }, "medium": { - "accuracy": 69.0, - "cost": 0.0002, - "robustness": 47.6 + "accuracy": 78.1, + "cost": 0.0007, + "robustness": 71.4 }, "hard": { - "accuracy": 28.0, - "cost": 0.0003, - "robustness": 28.6 + "accuracy": 33.1, + "cost": 0.0008, + "robustness": 100.0 }, "all": { - "accuracy": 79.9, - "cost": 0.0002, - "robustness": 43.7 + "accuracy": 83.5, + "cost": 0.0004, + "robustness": 62.0 } }, "subcategories": { "Engineering": { "metrics": { "easy": { - "accuracy": 98.4, - "cost": 0.0002, + "accuracy": 97.6, + "cost": 0.0004, "robustness": 60.0 }, "medium": { - "accuracy": 85.0, - "cost": 0.0004, - "robustness": 46.2 + "accuracy": 93.4, + "cost": 0.0009, + "robustness": 76.9 }, "hard": { - "accuracy": 54.8, - "cost": 0.0009, + "accuracy": 64.3, + "cost": 0.0014, "robustness": 100.0 }, "all": { - "accuracy": 86.3, - "cost": 0.0004, - "robustness": 52.6 + "accuracy": 91.3, + "cost": 0.0008, + "robustness": 73.7 } } }, "Management and public relations": { "metrics": { "easy": { - "accuracy": 98.3, + "accuracy": 95.0, "cost": 0.0001, "robustness": 50.0 }, "medium": { - "accuracy": 36.4, - "cost": 0.0001, + "accuracy": 45.5, + "cost": 0.0002, "robustness": 0.0 }, "hard": { @@ -15761,7 +13897,7 @@ "robustness": 0 }, "all": { - "accuracy": 86.3, + "accuracy": 84.9, "cost": 0.0001, "robustness": 40.0 } @@ -15770,24 +13906,24 @@ "Medicine and health": { "metrics": { "easy": { - "accuracy": 94.3, - "cost": 0.0001, - "robustness": 41.2 + "accuracy": 95.6, + "cost": 0.0002, + "robustness": 50.0 }, "medium": { - "accuracy": 59.5, - "cost": 0.0001, - "robustness": 57.1 + "accuracy": 69.0, + "cost": 0.0005, + "robustness": 71.4 }, "hard": { - "accuracy": 18.6, - "cost": 0.0001, - "robustness": 16.7 + "accuracy": 22.1, + "cost": 0.0006, + "robustness": 100.0 }, "all": { - "accuracy": 77.2, - "cost": 0.0001, - "robustness": 40.4 + "accuracy": 80.7, + "cost": 0.0003, + "robustness": 59.6 } } } diff --git a/src/data/routerMetrics/leaderboard.json b/src/data/routerMetrics/leaderboard.json index be9b85d..2ebe402 100644 --- a/src/data/routerMetrics/leaderboard.json +++ b/src/data/routerMetrics/leaderboard.json @@ -120,17 +120,6 @@ "Accuracy": 76.22, "Cost per 1k": 0.36 }, - { - "Router Name": "AgentForge Router", - "Arena Score": 74.2, - "Optimal Selection Score": 18.08, - "Optimal Cost Score": 53.56, - "Optimal Acc. Score": 98.69, - "Robustness Score": 40.48, - "Latency Score": null, - "Accuracy": 74.8, - "Cost per 1k": 0.13 - }, { "Router Name": "NuRoute Artemis", "Arena Score": 73.47, @@ -186,17 +175,6 @@ "Accuracy": 71.46, "Cost per 1k": 0.04 }, - { - "Router Name": "R2-Router", - "Arena Score": 71.86, - "Optimal Selection Score": 66.87, - "Optimal Cost Score": 80.13, - "Optimal Acc. Score": 99.85, - "Robustness Score": 45.71, - "Latency Score": null, - "Accuracy": 71.18, - "Cost per 1k": 0.04 - }, { "Router Name": "cruq-router", "Arena Score": 70.82, diff --git a/src/data/routers.json b/src/data/routers.json index d0882a0..c553c70 100644 --- a/src/data/routers.json +++ b/src/data/routers.json @@ -29,23 +29,6 @@ "websiteUrl": "https://workweave.ai", "githubUrl": "https://github.com/workweave/router" }, - "R2-Router": { - "name": "R2-Router", - "type": "open-source", - "description": "High-performance routing system with optimal accuracy", - "affiliation": "UCF", - "modelPool": [ - "qwen/qwen3-235b-a22b-2507", - "qwen/qwen3-next-80b-a3b-instruct", - "qwen/qwen3-30b-a3b-instruct-2507", - "Qwen/Qwen3-Coder-Next", - "gemini-2.5-flash", - "claude-3-haiku-20240307" - ], - "paperUrl": "https://arxiv.org/abs/2602.02823", - "websiteUrl": "https://arxiv.org/abs/2602.02823", - "githubUrl": "" - }, "RouterDC": { "name": "RouterDC", "type": "open-source", @@ -346,25 +329,6 @@ ], "githubUrl": "https://github.com/cxf2015" }, - "AgentForge Router": { - "name": "AgentForge Router", - "affiliation": "@YangY-Z", - "githubUrl": "https://github.com/YangY-Z", - "type": "open-source", - "description": "Submitted by @YangY-Z.", - "modelPool": [ - "qwen/qwen3-235b-a22b-2507", - "google/gemini-3.1-flash-lite", - "deepseek/deepseek-v4-flash", - "deepseek/deepseek-v3.2", - "qwen/qwen3-next-80b-a3b-instruct", - "Qwen/Qwen3-Coder-Next", - "gemini-2.5-flash", - "qwen/qwen3-30b-a3b-instruct-2507", - "gpt-4o-mini", - "claude-3-haiku-20240307" - ] - }, "Nadir Router": { "name": "Nadir Router", "affiliation": "NadirRouter",