From 9e028bbbd7fd2b409692994d87c8306b8633dc85 Mon Sep 17 00:00:00 2001 From: yl231 <111810457+yl231@users.noreply.github.com> Date: Thu, 29 Oct 2026 22:46:16 +0000 Subject: [PATCH] Sync leaderboard data from RouterArena@483e5571f5f5d5406d042217e162fcfb56f6ef00 --- .../flip_labels/flip_labels_bizrouter.json | 1682 +++++++++++++++++ src/data/flip_labels/flip_labels_divyam.json | 210 +- .../flip_labels_nuroute-artemis.json | 1682 +++++++++++++++++ src/data/routerMetrics/category_scores.json | 672 +++---- src/data/routerMetrics/leaderboard.json | 44 +- src/data/routers.json | 49 + 6 files changed, 3887 insertions(+), 452 deletions(-) create mode 100644 src/data/flip_labels/flip_labels_bizrouter.json create mode 100644 src/data/flip_labels/flip_labels_nuroute-artemis.json diff --git a/src/data/flip_labels/flip_labels_bizrouter.json b/src/data/flip_labels/flip_labels_bizrouter.json new file mode 100644 index 0000000..eef965c --- /dev/null +++ b/src/data/flip_labels/flip_labels_bizrouter.json @@ -0,0 +1,1682 @@ +[ + { + "global index": "AIME_112", + "flip": 0 + }, + { + "global index": "AIME_58", + "flip": 0 + }, + { + "global index": "ArcMMLU_12", + "flip": 0 + }, + { + "global index": "ArcMMLU_123", + "flip": 0 + }, + { + "global index": "ArcMMLU_16", + "flip": 0 + }, + { + "global index": "ArcMMLU_182", + "flip": 0 + }, + { + "global index": "ArcMMLU_230", + "flip": 0 + }, + { + "global index": "ArcMMLU_293", + "flip": 0 + }, + { + "global index": "ArcMMLU_349", + "flip": 0 + }, + { + "global index": "ArcMMLU_378", + "flip": 0 + }, + { + "global index": "ArcMMLU_443", + "flip": 0 + }, + { + "global index": "ArcMMLU_496", + "flip": 0 + }, + { + "global index": "ArcMMLU_631", + "flip": 0 + }, + { + "global index": "ArcMMLU_646", + "flip": 0 + }, + { + "global index": "ArcMMLU_659", + "flip": 0 + }, + { + "global index": "ArcMMLU_676", + "flip": 0 + }, + { + "global index": "ArcMMLU_685", + "flip": 0 + }, + { + "global index": "ArcMMLU_689", + "flip": 0 + }, + { + "global index": "ArcMMLU_702", + "flip": 0 + }, + { + "global index": "ArcMMLU_713", + "flip": 0 + }, + { + "global index": "ArcMMLU_98", + "flip": 0 + }, + { + "global index": "AsDiv_1165", + "flip": 0 + }, + { + "global index": "AsDiv_1347", + "flip": 0 + }, + { + "global index": "AsDiv_472", + "flip": 0 + }, + { + "global index": "AsDiv_733", + "flip": 0 + }, + { + "global index": "ChessInstruct_0", + "flip": 0 + }, + { + "global index": "ChessInstruct_107", + "flip": 0 + }, + { + "global index": "ChessInstruct_144", + "flip": 1 + }, + { + "global index": "ChessInstruct_42", + "flip": 0 + }, + { + "global index": "ChessInstruct_58", + "flip": 0 + }, + { + "global index": "ChessInstruct_71", + "flip": 0 + }, + { + "global index": "ChessInstruct_84", + "flip": 0 + }, + { + "global index": "Ethics_commonsense_28", + "flip": 0 + }, + { + "global index": "Ethics_commonsense_51", + "flip": 0 + }, + { + "global index": "Ethics_commonsense_6", + "flip": 0 + }, + { + "global index": "Ethics_commonsense_62", + "flip": 0 + }, + { + "global index": "Ethics_commonsense_70", + "flip": 0 + }, + { + "global index": "Ethics_commonsense_85", + "flip": 0 + }, + { + "global index": "Ethics_commonsense_90", + "flip": 0 + }, + { + "global index": "Ethics_deontology_0", + "flip": 0 + }, + { + "global index": "Ethics_deontology_2", + "flip": 0 + }, + { + "global index": "Ethics_deontology_31", + "flip": 0 + }, + { + "global index": "Ethics_deontology_32", + "flip": 0 + }, + { + "global index": "Ethics_deontology_56", + "flip": 0 + }, + { + "global index": "Ethics_justice_1", + "flip": 0 + }, + { + "global index": "Ethics_justice_45", + "flip": 0 + }, + { + "global index": "Ethics_justice_76", + "flip": 0 + }, + { + "global index": "Ethics_justice_84", + "flip": 0 + }, + { + "global index": "Ethics_virtue_14", + "flip": 0 + }, + { + "global index": "Ethics_virtue_26", + "flip": 0 + }, + { + "global index": "Ethics_virtue_30", + "flip": 0 + }, + { + "global index": "Ethics_virtue_48", + "flip": 0 + }, + { + "global index": "Ethics_virtue_51", + "flip": 0 + }, + { + "global index": "FinQA_149", + "flip": 0 + }, + { + "global index": "FinQA_208", + "flip": 0 + }, + { + "global index": "FinQA_56", + "flip": 0 + }, + { + "global index": "FinQA_60", + "flip": 0 + }, + { + "global index": "GSM8K_43", + "flip": 0 + }, + { + "global index": "GeoBench_1002", + "flip": 0 + }, + { + "global index": "GeoBench_1094", + "flip": 0 + }, + { + "global index": "GeoBench_1102", + "flip": 0 + }, + { + "global index": "GeoBench_1113", + "flip": 0 + }, + { + "global index": "GeoBench_124", + "flip": 0 + }, + { + "global index": "GeoBench_1243", + "flip": 0 + }, + { + "global index": "GeoBench_30", + "flip": 0 + }, + { + "global index": "GeoBench_502", + "flip": 0 + }, + { + "global index": "GeoBench_526", + "flip": 0 + }, + { + "global index": "GeoBench_591", + "flip": 0 + }, + { + "global index": "GeoBench_766", + "flip": 0 + }, + { + "global index": "GeoBench_87", + "flip": 0 + }, + { + "global index": "GeoBench_915", + "flip": 0 + }, + { + "global index": "GeoBench_944", + "flip": 0 + }, + { + "global index": "GeoBench_968", + "flip": 0 + }, + { + "global index": "GeoGraphyData_100k_27", + "flip": 0 + }, + { + "global index": "GeoGraphyData_100k_42", + "flip": 0 + }, + { + "global index": "LiveCodeBench_105", + "flip": 0 + }, + { + "global index": "LiveCodeBench_114", + "flip": 0 + }, + { + "global index": "LiveCodeBench_118", + "flip": 0 + }, + { + "global index": "LiveCodeBench_131", + "flip": 0 + }, + { + "global index": "LiveCodeBench_136", + "flip": 0 + }, + { + "global index": "LiveCodeBench_181", + "flip": 0 + }, + { + "global index": "LiveCodeBench_237", + "flip": 0 + }, + { + "global index": "LiveCodeBench_271", + "flip": 1 + }, + { + "global index": "LiveCodeBench_350", + "flip": 0 + }, + { + "global index": "LiveCodeBench_386", + "flip": 0 + }, + { + "global index": "LiveCodeBench_405", + "flip": 0 + }, + { + "global index": "LiveCodeBench_43", + "flip": 0 + }, + { + "global index": "LiveCodeBench_431", + "flip": 0 + }, + { + "global index": "LiveCodeBench_437", + "flip": 0 + }, + { + "global index": "LiveCodeBench_476", + "flip": 0 + }, + { + "global index": "LiveCodeBench_485", + "flip": 0 + }, + { + "global index": "LiveCodeBench_49", + "flip": 0 + }, + { + "global index": "LiveCodeBench_491", + "flip": 0 + }, + { + "global index": "LiveCodeBench_499", + "flip": 0 + }, + { + "global index": "MATH_108", + "flip": 0 + }, + { + "global index": "MATH_442", + "flip": 0 + }, + { + "global index": "MATH_53", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_2808", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_2912", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_2980", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_2985", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_3188", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_3215", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_3225", + "flip": 0 + }, + { + "global index": "MMLUPro_business_226", + "flip": 0 + }, + { + "global index": "MMLUPro_business_294", + "flip": 0 + }, + { + "global index": "MMLUPro_business_378", + "flip": 0 + }, + { + "global index": "MMLUPro_business_430", + "flip": 0 + }, + { + "global index": "MMLUPro_business_503", + "flip": 0 + }, + { + "global index": "MMLUPro_business_507", + "flip": 0 + }, + { + "global index": "MMLUPro_business_6", + "flip": 0 + }, + { + "global index": "MMLUPro_business_784", + "flip": 0 + }, + { + "global index": "MMLUPro_chemistry_3796", + "flip": 0 + }, + { + "global index": "MMLUPro_chemistry_3837", + "flip": 1 + }, + { + "global index": "MMLUPro_chemistry_3974", + "flip": 0 + }, + { + "global index": "MMLUPro_chemistry_4067", + "flip": 0 + }, + { + "global index": "MMLUPro_chemistry_4407", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9086", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9110", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9136", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9138", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9149", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9200", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9212", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9239", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9264", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9285", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9289", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9414", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9415", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9430", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9452", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9471", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9475", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_5769", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_5907", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_5931", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_5965", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_6114", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_6122", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_6135", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_6325", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_6353", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10076", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10125", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10179", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10195", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10199", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10298", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10342", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10395", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10428", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10432", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10473", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10537", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10701", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10823", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10864", + "flip": 0 + }, + { + "global index": "MMLUPro_health_4885", + "flip": 0 + }, + { + "global index": "MMLUPro_health_4973", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5093", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5144", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5214", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5215", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5261", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5473", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5514", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4486", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4490", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4497", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4509", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4517", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4523", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4605", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4629", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4638", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4717", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4749", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4752", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4774", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4810", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4833", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4836", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4841", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1007", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1031", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1386", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1462", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1484", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1518", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1818", + "flip": 0 + }, + { + "global index": "MMLUPro_law_806", + "flip": 0 + }, + { + "global index": "MMLUPro_law_899", + "flip": 0 + }, + { + "global index": "MMLUPro_math_6429", + "flip": 0 + }, + { + "global index": "MMLUPro_math_6526", + "flip": 0 + }, + { + "global index": "MMLUPro_math_6623", + "flip": 0 + }, + { + "global index": "MMLUPro_math_6848", + "flip": 0 + }, + { + "global index": "MMLUPro_math_7101", + "flip": 0 + }, + { + "global index": "MMLUPro_math_7249", + "flip": 0 + }, + { + "global index": "MMLUPro_math_7284", + "flip": 0 + }, + { + "global index": "MMLUPro_math_7451", + "flip": 0 + }, + { + "global index": "MMLUPro_math_7577", + "flip": 0 + }, + { + "global index": "MMLUPro_philosophy_9510", + "flip": 0 + }, + { + "global index": "MMLUPro_philosophy_9536", + "flip": 0 + }, + { + "global index": "MMLUPro_philosophy_9663", + "flip": 0 + }, + { + "global index": "MMLUPro_philosophy_9672", + "flip": 0 + }, + { + "global index": "MMLUPro_philosophy_9943", + "flip": 0 + }, + { + "global index": "MMLUPro_physics_7773", + "flip": 0 + }, + { + "global index": "MMLUPro_physics_7887", + "flip": 0 + }, + { + "global index": "MMLUPro_physics_7893", + "flip": 0 + }, + { + "global index": "MMLUPro_physics_8888", + "flip": 0 + }, + { + "global index": "MMLUPro_physics_9017", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2005", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2186", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2329", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2406", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2420", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2450", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2457", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2524", + "flip": 0 + }, + { + "global index": "MMLU_formal_logic_121", + "flip": 0 + }, + { + "global index": "MMLU_formal_logic_16", + "flip": 0 + }, + { + "global index": "MMLU_formal_logic_32", + "flip": 0 + }, + { + "global index": "MMLU_formal_logic_63", + "flip": 0 + }, + { + "global index": "MMLU_formal_logic_7", + "flip": 0 + }, + { + "global index": "MMLU_formal_logic_70", + "flip": 0 + }, + { + "global index": "MMLU_formal_logic_85", + "flip": 0 + }, + { + "global index": "MMLU_management_4", + "flip": 0 + }, + { + "global index": "MMLU_management_41", + "flip": 0 + }, + { + "global index": "MMLU_management_77", + "flip": 0 + }, + { + "global index": "MMLU_management_91", + "flip": 0 + }, + { + "global index": "MMLU_management_93", + "flip": 0 + }, + { + "global index": "MathQA_158", + "flip": 0 + }, + { + "global index": "MathQA_1742", + "flip": 0 + }, + { + "global index": "MathQA_202", + "flip": 0 + }, + { + "global index": "MathQA_2092", + "flip": 0 + }, + { + "global index": "MathQA_2102", + "flip": 0 + }, + { + "global index": "MathQA_2851", + "flip": 0 + }, + { + "global index": "MathQA_827", + "flip": 0 + }, + { + "global index": "MathQA_84", + "flip": 0 + }, + { + "global index": "MedMCQA_1005", + "flip": 0 + }, + { + "global index": "MedMCQA_1054", + "flip": 0 + }, + { + "global index": "MedMCQA_1298", + "flip": 0 + }, + { + "global index": "MedMCQA_1309", + "flip": 0 + }, + { + "global index": "MedMCQA_1362", + "flip": 0 + }, + { + "global index": "MedMCQA_145", + "flip": 0 + }, + { + "global index": "MedMCQA_2010", + "flip": 0 + }, + { + "global index": "MedMCQA_2323", + "flip": 0 + }, + { + "global index": "MedMCQA_2366", + "flip": 0 + }, + { + "global index": "MedMCQA_2581", + "flip": 0 + }, + { + "global index": "MedMCQA_511", + "flip": 0 + }, + { + "global index": "MedMCQA_59", + "flip": 0 + }, + { + "global index": "MedMCQA_643", + "flip": 0 + }, + { + "global index": "MedMCQA_853", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_126", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_14", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_147", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_152", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_188", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_189", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_240", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_33", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_337", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_340", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_70", + "flip": 0 + }, + { + "global index": "NarrativeQA_131", + "flip": 0 + }, + { + "global index": "NarrativeQA_1683", + "flip": 0 + }, + { + "global index": "NarrativeQA_2474", + "flip": 0 + }, + { + "global index": "NarrativeQA_2820", + "flip": 0 + }, + { + "global index": "NarrativeQA_3282", + "flip": 0 + }, + { + "global index": "NarrativeQA_4102", + "flip": 0 + }, + { + "global index": "NarrativeQA_4128", + "flip": 0 + }, + { + "global index": "NarrativeQA_4347", + "flip": 0 + }, + { + "global index": "NarrativeQA_4540", + "flip": 0 + }, + { + "global index": "NarrativeQA_5022", + "flip": 0 + }, + { + "global index": "NarrativeQA_5259", + "flip": 0 + }, + { + "global index": "NarrativeQA_533", + "flip": 0 + }, + { + "global index": "NarrativeQA_5894", + "flip": 0 + }, + { + "global index": "NarrativeQA_6829", + "flip": 0 + }, + { + "global index": "NarrativeQA_7678", + "flip": 0 + }, + { + "global index": "NarrativeQA_7964", + "flip": 0 + }, + { + "global index": "NarrativeQA_8215", + "flip": 0 + }, + { + "global index": "NarrativeQA_8598", + "flip": 0 + }, + { + "global index": "NarrativeQA_927", + "flip": 0 + }, + { + "global index": "OpenTDB_Animals_2545", + "flip": 0 + }, + { + "global index": "OpenTDB_Animals_262", + "flip": 0 + }, + { + "global index": "OpenTDB_Art_1429", + "flip": 0 + }, + { + "global index": "OpenTDB_Celebrities_1078", + "flip": 0 + }, + { + "global index": "OpenTDB_Celebrities_1456", + "flip": 0 + }, + { + "global index": "OpenTDB_Celebrities_3577", + "flip": 0 + }, + { + "global index": "OpenTDB_Celebrities_3968", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_1178", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_1807", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_1957", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_2181", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_3404", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_3727", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_4019", + "flip": 0 + }, + { + "global index": "OpenTDB_Geography_2099", + "flip": 0 + }, + { + "global index": "OpenTDB_Geography_2346", + "flip": 0 + }, + { + "global index": "OpenTDB_Geography_3880", + "flip": 0 + }, + { + "global index": "OpenTDB_Geography_792", + "flip": 0 + }, + { + "global index": "OpenTDB_History_1162", + "flip": 0 + }, + { + "global index": "OpenTDB_History_2026", + "flip": 0 + }, + { + "global index": "OpenTDB_History_3712", + "flip": 0 + }, + { + "global index": "OpenTDB_History_3902", + "flip": 0 + }, + { + "global index": "OpenTDB_Science & Nature_1175", + "flip": 0 + }, + { + "global index": "OpenTDB_Science & Nature_1560", + "flip": 0 + }, + { + "global index": "OpenTDB_Science & Nature_3716", + "flip": 0 + }, + { + "global index": "OpenTDB_Science & Nature_476", + "flip": 0 + }, + { + "global index": "OpenTDB_Sports_2289", + "flip": 0 + }, + { + "global index": "OpenTDB_Vehicles_1173", + "flip": 0 + }, + { + "global index": "OpenTDB_Vehicles_1419", + "flip": 0 + }, + { + "global index": "OpenTDB_Vehicles_2519", + "flip": 0 + }, + { + "global index": "OpenTDB_Vehicles_3234", + "flip": 0 + }, + { + "global index": "PubMedQA_0", + "flip": 0 + }, + { + "global index": "PubMedQA_154", + "flip": 0 + }, + { + "global index": "PubMedQA_18", + "flip": 0 + }, + { + "global index": "PubMedQA_238", + "flip": 0 + }, + { + "global index": "PubMedQA_250", + "flip": 0 + }, + { + "global index": "PubMedQA_337", + "flip": 0 + }, + { + "global index": "PubMedQA_362", + "flip": 0 + }, + { + "global index": "PubMedQA_437", + "flip": 0 + }, + { + "global index": "PubMedQA_510", + "flip": 0 + }, + { + "global index": "PubMedQA_520", + "flip": 0 + }, + { + "global index": "PubMedQA_575", + "flip": 0 + }, + { + "global index": "PubMedQA_582", + "flip": 0 + }, + { + "global index": "PubMedQA_588", + "flip": 0 + }, + { + "global index": "PubMedQA_610", + "flip": 0 + }, + { + "global index": "PubMedQA_63", + "flip": 0 + }, + { + "global index": "PubMedQA_643", + "flip": 0 + }, + { + "global index": "PubMedQA_687", + "flip": 0 + }, + { + "global index": "PubMedQA_722", + "flip": 0 + }, + { + "global index": "PubMedQA_73", + "flip": 0 + }, + { + "global index": "PubMedQA_755", + "flip": 0 + }, + { + "global index": "PubMedQA_8", + "flip": 0 + }, + { + "global index": "PubMedQA_81", + "flip": 0 + }, + { + "global index": "PubMedQA_854", + "flip": 0 + }, + { + "global index": "PubMedQA_905", + "flip": 0 + }, + { + "global index": "QANTA_Fine Arts_1212", + "flip": 0 + }, + { + "global index": "QANTA_Fine Arts_1702", + "flip": 0 + }, + { + "global index": "QANTA_Fine Arts_828", + "flip": 0 + }, + { + "global index": "QANTA_Fine Arts_865", + "flip": 0 + }, + { + "global index": "QANTA_Geography_1023", + "flip": 0 + }, + { + "global index": "QANTA_Geography_1555", + "flip": 0 + }, + { + "global index": "QANTA_Geography_304", + "flip": 0 + }, + { + "global index": "QANTA_History_1084", + "flip": 0 + }, + { + "global index": "QANTA_History_1154", + "flip": 0 + }, + { + "global index": "QANTA_History_433", + "flip": 0 + }, + { + "global index": "QANTA_History_473", + "flip": 0 + }, + { + "global index": "QANTA_History_926", + "flip": 0 + }, + { + "global index": "QANTA_Literature_1045", + "flip": 0 + }, + { + "global index": "QANTA_Literature_1073", + "flip": 0 + }, + { + "global index": "QANTA_Literature_1239", + "flip": 0 + }, + { + "global index": "QANTA_Literature_1326", + "flip": 0 + }, + { + "global index": "QANTA_Literature_1727", + "flip": 0 + }, + { + "global index": "QANTA_Literature_1843", + "flip": 0 + }, + { + "global index": "QANTA_Literature_386", + "flip": 0 + }, + { + "global index": "QANTA_Literature_408", + "flip": 1 + }, + { + "global index": "QANTA_Literature_475", + "flip": 0 + }, + { + "global index": "QANTA_Literature_833", + "flip": 1 + }, + { + "global index": "QANTA_Philosophy_1270", + "flip": 0 + }, + { + "global index": "QANTA_Philosophy_499", + "flip": 0 + }, + { + "global index": "QANTA_Philosophy_91", + "flip": 0 + }, + { + "global index": "QANTA_Science_1360", + "flip": 0 + }, + { + "global index": "QANTA_Science_1473", + "flip": 0 + }, + { + "global index": "QANTA_Science_308", + "flip": 0 + }, + { + "global index": "QANTA_Science_619", + "flip": 0 + }, + { + "global index": "QANTA_Social Science_1847", + "flip": 0 + }, + { + "global index": "QANTA_Social Science_2", + "flip": 0 + }, + { + "global index": "QANTA_Social Science_77", + "flip": 0 + }, + { + "global index": "SocialiQA_13810", + "flip": 0 + }, + { + "global index": "SocialiQA_22095", + "flip": 0 + }, + { + "global index": "SocialiQA_26846", + "flip": 0 + }, + { + "global index": "SocialiQA_7839", + "flip": 0 + }, + { + "global index": "SuperGLUE-CausalReasoning_4526", + "flip": 0 + }, + { + "global index": "SuperGLUE-ClozeTest_12894", + "flip": 0 + }, + { + "global index": "SuperGLUE-ClozeTest_17965", + "flip": 0 + }, + { + "global index": "SuperGLUE-ClozeTest_18766", + "flip": 0 + }, + { + "global index": "SuperGLUE-Entailment_19410", + "flip": 0 + }, + { + "global index": "SuperGLUE-Entailment_19567", + "flip": 0 + }, + { + "global index": "SuperGLUE-Entailment_522", + "flip": 0 + }, + { + "global index": "SuperGLUE-Entailment_767", + "flip": 0 + }, + { + "global index": "SuperGLUE-QA_1408", + "flip": 0 + }, + { + "global index": "SuperGLUE-QA_3137", + "flip": 0 + }, + { + "global index": "SuperGLUE-QA_4046", + "flip": 0 + }, + { + "global index": "SuperGLUE-QA_4102", + "flip": 0 + }, + { + "global index": "SuperGLUE-QA_4160", + "flip": 0 + }, + { + "global index": "SuperGLUE-RC_7725", + "flip": 0 + }, + { + "global index": "SuperGLUE-RC_7738", + "flip": 0 + }, + { + "global index": "SuperGLUE-RC_8531", + "flip": 0 + }, + { + "global index": "SuperGLUE-Wic_19695", + "flip": 0 + }, + { + "global index": "SuperGLUE-Wic_19738", + "flip": 1 + }, + { + "global index": "SuperGLUE-Wic_20079", + "flip": 0 + }, + { + "global index": "SuperGLUE-Wic_20189", + "flip": 1 + }, + { + "global index": "SuperGLUE-Wic_20253", + "flip": 1 + }, + { + "global index": "SuperGLUE-Wsc_20368", + "flip": 0 + }, + { + "global index": "SuperGLUE-Wsc_20370", + "flip": 0 + }, + { + "global index": "WMT19-cs-en_156", + "flip": 0 + }, + { + "global index": "WMT19-cs-en_246", + "flip": 0 + }, + { + "global index": "WMT19-cs-en_568", + "flip": 0 + }, + { + "global index": "WMT19-de-en_46", + "flip": 0 + }, + { + "global index": "WMT19-de-en_715", + "flip": 0 + }, + { + "global index": "WMT19-de-en_883", + "flip": 1 + }, + { + "global index": "WMT19-fi-en_222", + "flip": 0 + }, + { + "global index": "WMT19-fi-en_610", + "flip": 0 + }, + { + "global index": "WMT19-gu-en_116", + "flip": 0 + }, + { + "global index": "WMT19-gu-en_123", + "flip": 0 + }, + { + "global index": "WMT19-gu-en_191", + "flip": 0 + }, + { + "global index": "WMT19-gu-en_491", + "flip": 0 + }, + { + "global index": "WMT19-gu-en_968", + "flip": 0 + }, + { + "global index": "WMT19-kk-en_528", + "flip": 1 + }, + { + "global index": "WMT19-kk-en_826", + "flip": 0 + }, + { + "global index": "WMT19-lt-en_135", + "flip": 0 + }, + { + "global index": "WMT19-lt-en_269", + "flip": 0 + }, + { + "global index": "WMT19-lt-en_636", + "flip": 0 + }, + { + "global index": "WMT19-ru-en_222", + "flip": 0 + }, + { + "global index": "WMT19-zh-en_218", + "flip": 0 + }, + { + "global index": "WMT19-zh-en_252", + "flip": 1 + }, + { + "global index": "WMT19-zh-en_59", + "flip": 0 + } +] diff --git a/src/data/flip_labels/flip_labels_divyam.json b/src/data/flip_labels/flip_labels_divyam.json index 1d7dc21..368ef93 100644 --- a/src/data/flip_labels/flip_labels_divyam.json +++ b/src/data/flip_labels/flip_labels_divyam.json @@ -13,7 +13,7 @@ }, { "global index": "ArcMMLU_123", - "flip": 0 + "flip": 1 }, { "global index": "ArcMMLU_16", @@ -21,7 +21,7 @@ }, { "global index": "ArcMMLU_182", - "flip": 0 + "flip": 1 }, { "global index": "ArcMMLU_230", @@ -33,7 +33,7 @@ }, { "global index": "ArcMMLU_349", - "flip": 0 + "flip": 1 }, { "global index": "ArcMMLU_378", @@ -45,7 +45,7 @@ }, { "global index": "ArcMMLU_496", - "flip": 0 + "flip": 1 }, { "global index": "ArcMMLU_631", @@ -73,7 +73,7 @@ }, { "global index": "ArcMMLU_702", - "flip": 0 + "flip": 1 }, { "global index": "ArcMMLU_713", @@ -85,7 +85,7 @@ }, { "global index": "AsDiv_1165", - "flip": 0 + "flip": 1 }, { "global index": "AsDiv_1347", @@ -133,15 +133,15 @@ }, { "global index": "Ethics_commonsense_51", - "flip": 0 + "flip": 1 }, { "global index": "Ethics_commonsense_6", - "flip": 0 + "flip": 1 }, { "global index": "Ethics_commonsense_62", - "flip": 0 + "flip": 1 }, { "global index": "Ethics_commonsense_70", @@ -157,7 +157,7 @@ }, { "global index": "Ethics_deontology_0", - "flip": 0 + "flip": 1 }, { "global index": "Ethics_deontology_2", @@ -169,11 +169,11 @@ }, { "global index": "Ethics_deontology_32", - "flip": 0 + "flip": 1 }, { "global index": "Ethics_deontology_56", - "flip": 0 + "flip": 1 }, { "global index": "Ethics_justice_1", @@ -181,11 +181,11 @@ }, { "global index": "Ethics_justice_45", - "flip": 0 + "flip": 1 }, { "global index": "Ethics_justice_76", - "flip": 0 + "flip": 1 }, { "global index": "Ethics_justice_84", @@ -197,7 +197,7 @@ }, { "global index": "Ethics_virtue_26", - "flip": 1 + "flip": 0 }, { "global index": "Ethics_virtue_30", @@ -237,11 +237,11 @@ }, { "global index": "GeoBench_1094", - "flip": 0 + "flip": 1 }, { "global index": "GeoBench_1102", - "flip": 0 + "flip": 1 }, { "global index": "GeoBench_1113", @@ -265,7 +265,7 @@ }, { "global index": "GeoBench_526", - "flip": 0 + "flip": 1 }, { "global index": "GeoBench_591", @@ -273,7 +273,7 @@ }, { "global index": "GeoBench_766", - "flip": 0 + "flip": 1 }, { "global index": "GeoBench_87", @@ -293,11 +293,11 @@ }, { "global index": "GeoGraphyData_100k_27", - "flip": 0 + "flip": 1 }, { "global index": "GeoGraphyData_100k_42", - "flip": 0 + "flip": 1 }, { "global index": "LiveCodeBench_105", @@ -309,7 +309,7 @@ }, { "global index": "LiveCodeBench_118", - "flip": 1 + "flip": 0 }, { "global index": "LiveCodeBench_131", @@ -329,7 +329,7 @@ }, { "global index": "LiveCodeBench_271", - "flip": 0 + "flip": 1 }, { "global index": "LiveCodeBench_350", @@ -345,11 +345,11 @@ }, { "global index": "LiveCodeBench_43", - "flip": 0 + "flip": 1 }, { "global index": "LiveCodeBench_431", - "flip": 0 + "flip": 1 }, { "global index": "LiveCodeBench_437", @@ -405,11 +405,11 @@ }, { "global index": "MMLUPro_biology_3188", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_biology_3215", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_biology_3225", @@ -421,7 +421,7 @@ }, { "global index": "MMLUPro_business_294", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_business_378", @@ -429,7 +429,7 @@ }, { "global index": "MMLUPro_business_430", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_business_503", @@ -437,7 +437,7 @@ }, { "global index": "MMLUPro_business_507", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_business_6", @@ -449,7 +449,7 @@ }, { "global index": "MMLUPro_chemistry_3796", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_chemistry_3837", @@ -457,11 +457,11 @@ }, { "global index": "MMLUPro_chemistry_3974", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_chemistry_4067", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_chemistry_4407", @@ -481,7 +481,7 @@ }, { "global index": "MMLUPro_computer science_9138", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_computer science_9149", @@ -497,7 +497,7 @@ }, { "global index": "MMLUPro_computer science_9239", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_computer science_9264", @@ -561,7 +561,7 @@ }, { "global index": "MMLUPro_economics_6135", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_economics_6325", @@ -581,7 +581,7 @@ }, { "global index": "MMLUPro_engineering_10179", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_engineering_10195", @@ -597,7 +597,7 @@ }, { "global index": "MMLUPro_engineering_10342", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_engineering_10395", @@ -609,7 +609,7 @@ }, { "global index": "MMLUPro_engineering_10432", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_engineering_10473", @@ -633,15 +633,15 @@ }, { "global index": "MMLUPro_health_4885", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_health_4973", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_health_5093", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_health_5144", @@ -661,11 +661,11 @@ }, { "global index": "MMLUPro_health_5473", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_health_5514", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_history_4486", @@ -681,11 +681,11 @@ }, { "global index": "MMLUPro_history_4509", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_history_4517", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_history_4523", @@ -705,23 +705,23 @@ }, { "global index": "MMLUPro_history_4717", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_history_4749", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_history_4752", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_history_4774", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_history_4810", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_history_4833", @@ -733,7 +733,7 @@ }, { "global index": "MMLUPro_history_4841", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_law_1007", @@ -785,7 +785,7 @@ }, { "global index": "MMLUPro_math_6848", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_math_7101", @@ -793,11 +793,11 @@ }, { "global index": "MMLUPro_math_7249", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_math_7284", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_math_7451", @@ -809,7 +809,7 @@ }, { "global index": "MMLUPro_philosophy_9510", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_philosophy_9536", @@ -825,11 +825,11 @@ }, { "global index": "MMLUPro_philosophy_9943", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_physics_7773", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_physics_7887", @@ -837,7 +837,7 @@ }, { "global index": "MMLUPro_physics_7893", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_physics_8888", @@ -845,11 +845,11 @@ }, { "global index": "MMLUPro_physics_9017", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_psychology_2005", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_psychology_2186", @@ -861,7 +861,7 @@ }, { "global index": "MMLUPro_psychology_2406", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_psychology_2420", @@ -897,15 +897,15 @@ }, { "global index": "MMLU_formal_logic_7", - "flip": 0 + "flip": 1 }, { "global index": "MMLU_formal_logic_70", - "flip": 0 + "flip": 1 }, { "global index": "MMLU_formal_logic_85", - "flip": 0 + "flip": 1 }, { "global index": "MMLU_management_4", @@ -921,7 +921,7 @@ }, { "global index": "MMLU_management_91", - "flip": 0 + "flip": 1 }, { "global index": "MMLU_management_93", @@ -985,7 +985,7 @@ }, { "global index": "MedMCQA_2010", - "flip": 0 + "flip": 1 }, { "global index": "MedMCQA_2323", @@ -1005,7 +1005,7 @@ }, { "global index": "MedMCQA_59", - "flip": 0 + "flip": 1 }, { "global index": "MedMCQA_643", @@ -1065,11 +1065,11 @@ }, { "global index": "NarrativeQA_1683", - "flip": 0 + "flip": 1 }, { "global index": "NarrativeQA_2474", - "flip": 0 + "flip": 1 }, { "global index": "NarrativeQA_2820", @@ -1089,7 +1089,7 @@ }, { "global index": "NarrativeQA_4347", - "flip": 0 + "flip": 1 }, { "global index": "NarrativeQA_4540", @@ -1105,7 +1105,7 @@ }, { "global index": "NarrativeQA_533", - "flip": 0 + "flip": 1 }, { "global index": "NarrativeQA_5894", @@ -1121,7 +1121,7 @@ }, { "global index": "NarrativeQA_7964", - "flip": 0 + "flip": 1 }, { "global index": "NarrativeQA_8215", @@ -1133,7 +1133,7 @@ }, { "global index": "NarrativeQA_927", - "flip": 0 + "flip": 1 }, { "global index": "OpenTDB_Animals_2545", @@ -1141,7 +1141,7 @@ }, { "global index": "OpenTDB_Animals_262", - "flip": 0 + "flip": 1 }, { "global index": "OpenTDB_Art_1429", @@ -1153,7 +1153,7 @@ }, { "global index": "OpenTDB_Celebrities_1456", - "flip": 0 + "flip": 1 }, { "global index": "OpenTDB_Celebrities_3577", @@ -1173,7 +1173,7 @@ }, { "global index": "OpenTDB_General Knowledge_1957", - "flip": 0 + "flip": 1 }, { "global index": "OpenTDB_General Knowledge_2181", @@ -1189,11 +1189,11 @@ }, { "global index": "OpenTDB_General Knowledge_4019", - "flip": 0 + "flip": 1 }, { "global index": "OpenTDB_Geography_2099", - "flip": 0 + "flip": 1 }, { "global index": "OpenTDB_Geography_2346", @@ -1209,7 +1209,7 @@ }, { "global index": "OpenTDB_History_1162", - "flip": 0 + "flip": 1 }, { "global index": "OpenTDB_History_2026", @@ -1237,7 +1237,7 @@ }, { "global index": "OpenTDB_Science & Nature_476", - "flip": 0 + "flip": 1 }, { "global index": "OpenTDB_Sports_2289", @@ -1281,7 +1281,7 @@ }, { "global index": "PubMedQA_337", - "flip": 0 + "flip": 1 }, { "global index": "PubMedQA_362", @@ -1289,7 +1289,7 @@ }, { "global index": "PubMedQA_437", - "flip": 0 + "flip": 1 }, { "global index": "PubMedQA_510", @@ -1313,7 +1313,7 @@ }, { "global index": "PubMedQA_610", - "flip": 0 + "flip": 1 }, { "global index": "PubMedQA_63", @@ -1329,7 +1329,7 @@ }, { "global index": "PubMedQA_722", - "flip": 0 + "flip": 1 }, { "global index": "PubMedQA_73", @@ -1337,7 +1337,7 @@ }, { "global index": "PubMedQA_755", - "flip": 0 + "flip": 1 }, { "global index": "PubMedQA_8", @@ -1497,11 +1497,11 @@ }, { "global index": "SocialiQA_7839", - "flip": 0 + "flip": 1 }, { "global index": "SuperGLUE-CausalReasoning_4526", - "flip": 0 + "flip": 1 }, { "global index": "SuperGLUE-ClozeTest_12894", @@ -1517,11 +1517,11 @@ }, { "global index": "SuperGLUE-Entailment_19410", - "flip": 0 + "flip": 1 }, { "global index": "SuperGLUE-Entailment_19567", - "flip": 0 + "flip": 1 }, { "global index": "SuperGLUE-Entailment_522", @@ -1541,7 +1541,7 @@ }, { "global index": "SuperGLUE-QA_4046", - "flip": 0 + "flip": 1 }, { "global index": "SuperGLUE-QA_4102", @@ -1549,7 +1549,7 @@ }, { "global index": "SuperGLUE-QA_4160", - "flip": 0 + "flip": 1 }, { "global index": "SuperGLUE-RC_7725", @@ -1569,7 +1569,7 @@ }, { "global index": "SuperGLUE-Wic_19738", - "flip": 0 + "flip": 1 }, { "global index": "SuperGLUE-Wic_20079", @@ -1577,7 +1577,7 @@ }, { "global index": "SuperGLUE-Wic_20189", - "flip": 0 + "flip": 1 }, { "global index": "SuperGLUE-Wic_20253", @@ -1585,7 +1585,7 @@ }, { "global index": "SuperGLUE-Wsc_20368", - "flip": 0 + "flip": 1 }, { "global index": "SuperGLUE-Wsc_20370", @@ -1601,7 +1601,7 @@ }, { "global index": "WMT19-cs-en_568", - "flip": 0 + "flip": 1 }, { "global index": "WMT19-de-en_46", @@ -1609,11 +1609,11 @@ }, { "global index": "WMT19-de-en_715", - "flip": 0 + "flip": 1 }, { "global index": "WMT19-de-en_883", - "flip": 0 + "flip": 1 }, { "global index": "WMT19-fi-en_222", @@ -1621,7 +1621,7 @@ }, { "global index": "WMT19-fi-en_610", - "flip": 1 + "flip": 0 }, { "global index": "WMT19-gu-en_116", @@ -1637,7 +1637,7 @@ }, { "global index": "WMT19-gu-en_491", - "flip": 0 + "flip": 1 }, { "global index": "WMT19-gu-en_968", @@ -1645,11 +1645,11 @@ }, { "global index": "WMT19-kk-en_528", - "flip": 0 + "flip": 1 }, { "global index": "WMT19-kk-en_826", - "flip": 0 + "flip": 1 }, { "global index": "WMT19-lt-en_135", @@ -1661,7 +1661,7 @@ }, { "global index": "WMT19-lt-en_636", - "flip": 0 + "flip": 1 }, { "global index": "WMT19-ru-en_222", @@ -1673,10 +1673,10 @@ }, { "global index": "WMT19-zh-en_252", - "flip": 0 + "flip": 1 }, { "global index": "WMT19-zh-en_59", - "flip": 0 + "flip": 1 } ] diff --git a/src/data/flip_labels/flip_labels_nuroute-artemis.json b/src/data/flip_labels/flip_labels_nuroute-artemis.json new file mode 100644 index 0000000..780db25 --- /dev/null +++ b/src/data/flip_labels/flip_labels_nuroute-artemis.json @@ -0,0 +1,1682 @@ +[ + { + "global index": "AIME_112", + "flip": 0 + }, + { + "global index": "AIME_58", + "flip": 0 + }, + { + "global index": "ArcMMLU_12", + "flip": 0 + }, + { + "global index": "ArcMMLU_123", + "flip": 0 + }, + { + "global index": "ArcMMLU_16", + "flip": 0 + }, + { + "global index": "ArcMMLU_182", + "flip": 0 + }, + { + "global index": "ArcMMLU_230", + "flip": 0 + }, + { + "global index": "ArcMMLU_293", + "flip": 0 + }, + { + "global index": "ArcMMLU_349", + "flip": 0 + }, + { + "global index": "ArcMMLU_378", + "flip": 0 + }, + { + "global index": "ArcMMLU_443", + "flip": 1 + }, + { + "global index": "ArcMMLU_496", + "flip": 0 + }, + { + "global index": "ArcMMLU_631", + "flip": 0 + }, + { + "global index": "ArcMMLU_646", + "flip": 0 + }, + { + "global index": "ArcMMLU_659", + "flip": 0 + }, + { + "global index": "ArcMMLU_676", + "flip": 0 + }, + { + "global index": "ArcMMLU_685", + "flip": 0 + }, + { + "global index": "ArcMMLU_689", + "flip": 0 + }, + { + "global index": "ArcMMLU_702", + "flip": 0 + }, + { + "global index": "ArcMMLU_713", + "flip": 1 + }, + { + "global index": "ArcMMLU_98", + "flip": 1 + }, + { + "global index": "AsDiv_1165", + "flip": 1 + }, + { + "global index": "AsDiv_1347", + "flip": 1 + }, + { + "global index": "AsDiv_472", + "flip": 1 + }, + { + "global index": "AsDiv_733", + "flip": 1 + }, + { + "global index": "ChessInstruct_0", + "flip": 0 + }, + { + "global index": "ChessInstruct_107", + "flip": 0 + }, + { + "global index": "ChessInstruct_144", + "flip": 0 + }, + { + "global index": "ChessInstruct_42", + "flip": 0 + }, + { + "global index": "ChessInstruct_58", + "flip": 0 + }, + { + "global index": "ChessInstruct_71", + "flip": 0 + }, + { + "global index": "ChessInstruct_84", + "flip": 1 + }, + { + "global index": "Ethics_commonsense_28", + "flip": 1 + }, + { + "global index": "Ethics_commonsense_51", + "flip": 1 + }, + { + "global index": "Ethics_commonsense_6", + "flip": 1 + }, + { + "global index": "Ethics_commonsense_62", + "flip": 1 + }, + { + "global index": "Ethics_commonsense_70", + "flip": 1 + }, + { + "global index": "Ethics_commonsense_85", + "flip": 0 + }, + { + "global index": "Ethics_commonsense_90", + "flip": 1 + }, + { + "global index": "Ethics_deontology_0", + "flip": 0 + }, + { + "global index": "Ethics_deontology_2", + "flip": 0 + }, + { + "global index": "Ethics_deontology_31", + "flip": 0 + }, + { + "global index": "Ethics_deontology_32", + "flip": 0 + }, + { + "global index": "Ethics_deontology_56", + "flip": 0 + }, + { + "global index": "Ethics_justice_1", + "flip": 1 + }, + { + "global index": "Ethics_justice_45", + "flip": 1 + }, + { + "global index": "Ethics_justice_76", + "flip": 1 + }, + { + "global index": "Ethics_justice_84", + "flip": 1 + }, + { + "global index": "Ethics_virtue_14", + "flip": 0 + }, + { + "global index": "Ethics_virtue_26", + "flip": 0 + }, + { + "global index": "Ethics_virtue_30", + "flip": 1 + }, + { + "global index": "Ethics_virtue_48", + "flip": 0 + }, + { + "global index": "Ethics_virtue_51", + "flip": 0 + }, + { + "global index": "FinQA_149", + "flip": 0 + }, + { + "global index": "FinQA_208", + "flip": 0 + }, + { + "global index": "FinQA_56", + "flip": 0 + }, + { + "global index": "FinQA_60", + "flip": 0 + }, + { + "global index": "GSM8K_43", + "flip": 0 + }, + { + "global index": "GeoBench_1002", + "flip": 0 + }, + { + "global index": "GeoBench_1094", + "flip": 0 + }, + { + "global index": "GeoBench_1102", + "flip": 0 + }, + { + "global index": "GeoBench_1113", + "flip": 0 + }, + { + "global index": "GeoBench_124", + "flip": 0 + }, + { + "global index": "GeoBench_1243", + "flip": 0 + }, + { + "global index": "GeoBench_30", + "flip": 0 + }, + { + "global index": "GeoBench_502", + "flip": 1 + }, + { + "global index": "GeoBench_526", + "flip": 0 + }, + { + "global index": "GeoBench_591", + "flip": 0 + }, + { + "global index": "GeoBench_766", + "flip": 0 + }, + { + "global index": "GeoBench_87", + "flip": 1 + }, + { + "global index": "GeoBench_915", + "flip": 0 + }, + { + "global index": "GeoBench_944", + "flip": 0 + }, + { + "global index": "GeoBench_968", + "flip": 0 + }, + { + "global index": "GeoGraphyData_100k_27", + "flip": 0 + }, + { + "global index": "GeoGraphyData_100k_42", + "flip": 0 + }, + { + "global index": "LiveCodeBench_105", + "flip": 1 + }, + { + "global index": "LiveCodeBench_114", + "flip": 0 + }, + { + "global index": "LiveCodeBench_118", + "flip": 0 + }, + { + "global index": "LiveCodeBench_131", + "flip": 0 + }, + { + "global index": "LiveCodeBench_136", + "flip": 1 + }, + { + "global index": "LiveCodeBench_181", + "flip": 1 + }, + { + "global index": "LiveCodeBench_237", + "flip": 0 + }, + { + "global index": "LiveCodeBench_271", + "flip": 0 + }, + { + "global index": "LiveCodeBench_350", + "flip": 0 + }, + { + "global index": "LiveCodeBench_386", + "flip": 0 + }, + { + "global index": "LiveCodeBench_405", + "flip": 0 + }, + { + "global index": "LiveCodeBench_43", + "flip": 1 + }, + { + "global index": "LiveCodeBench_431", + "flip": 0 + }, + { + "global index": "LiveCodeBench_437", + "flip": 0 + }, + { + "global index": "LiveCodeBench_476", + "flip": 0 + }, + { + "global index": "LiveCodeBench_485", + "flip": 0 + }, + { + "global index": "LiveCodeBench_49", + "flip": 0 + }, + { + "global index": "LiveCodeBench_491", + "flip": 0 + }, + { + "global index": "LiveCodeBench_499", + "flip": 1 + }, + { + "global index": "MATH_108", + "flip": 1 + }, + { + "global index": "MATH_442", + "flip": 1 + }, + { + "global index": "MATH_53", + "flip": 1 + }, + { + "global index": "MMLUPro_biology_2808", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_2912", + "flip": 1 + }, + { + "global index": "MMLUPro_biology_2980", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_2985", + "flip": 1 + }, + { + "global index": "MMLUPro_biology_3188", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_3215", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_3225", + "flip": 0 + }, + { + "global index": "MMLUPro_business_226", + "flip": 0 + }, + { + "global index": "MMLUPro_business_294", + "flip": 0 + }, + { + "global index": "MMLUPro_business_378", + "flip": 0 + }, + { + "global index": "MMLUPro_business_430", + "flip": 0 + }, + { + "global index": "MMLUPro_business_503", + "flip": 0 + }, + { + "global index": "MMLUPro_business_507", + "flip": 0 + }, + { + "global index": "MMLUPro_business_6", + "flip": 1 + }, + { + "global index": "MMLUPro_business_784", + "flip": 1 + }, + { + "global index": "MMLUPro_chemistry_3796", + "flip": 0 + }, + { + "global index": "MMLUPro_chemistry_3837", + "flip": 0 + }, + { + "global index": "MMLUPro_chemistry_3974", + "flip": 0 + }, + { + "global index": "MMLUPro_chemistry_4067", + "flip": 0 + }, + { + "global index": "MMLUPro_chemistry_4407", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9086", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9110", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9136", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9138", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9149", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9200", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9212", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9239", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9264", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9285", + "flip": 1 + }, + { + "global index": "MMLUPro_computer science_9289", + "flip": 1 + }, + { + "global index": "MMLUPro_computer science_9414", + "flip": 1 + }, + { + "global index": "MMLUPro_computer science_9415", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9430", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9452", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9471", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9475", + "flip": 1 + }, + { + "global index": "MMLUPro_economics_5769", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_5907", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_5931", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_5965", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_6114", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_6122", + "flip": 1 + }, + { + "global index": "MMLUPro_economics_6135", + "flip": 1 + }, + { + "global index": "MMLUPro_economics_6325", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_6353", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10076", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10125", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10179", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10195", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10199", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10298", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10342", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10395", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10428", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10432", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10473", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10537", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10701", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10823", + "flip": 1 + }, + { + "global index": "MMLUPro_engineering_10864", + "flip": 0 + }, + { + "global index": "MMLUPro_health_4885", + "flip": 0 + }, + { + "global index": "MMLUPro_health_4973", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5093", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5144", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5214", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5215", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5261", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5473", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5514", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4486", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4490", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4497", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4509", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4517", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4523", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4605", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4629", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4638", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4717", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4749", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4752", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4774", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4810", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4833", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4836", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4841", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1007", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1031", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1386", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1462", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1484", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1518", + "flip": 1 + }, + { + "global index": "MMLUPro_law_1818", + "flip": 0 + }, + { + "global index": "MMLUPro_law_806", + "flip": 0 + }, + { + "global index": "MMLUPro_law_899", + "flip": 0 + }, + { + "global index": "MMLUPro_math_6429", + "flip": 1 + }, + { + "global index": "MMLUPro_math_6526", + "flip": 0 + }, + { + "global index": "MMLUPro_math_6623", + "flip": 0 + }, + { + "global index": "MMLUPro_math_6848", + "flip": 0 + }, + { + "global index": "MMLUPro_math_7101", + "flip": 0 + }, + { + "global index": "MMLUPro_math_7249", + "flip": 0 + }, + { + "global index": "MMLUPro_math_7284", + "flip": 0 + }, + { + "global index": "MMLUPro_math_7451", + "flip": 0 + }, + { + "global index": "MMLUPro_math_7577", + "flip": 0 + }, + { + "global index": "MMLUPro_philosophy_9510", + "flip": 0 + }, + { + "global index": "MMLUPro_philosophy_9536", + "flip": 0 + }, + { + "global index": "MMLUPro_philosophy_9663", + "flip": 0 + }, + { + "global index": "MMLUPro_philosophy_9672", + "flip": 1 + }, + { + "global index": "MMLUPro_philosophy_9943", + "flip": 0 + }, + { + "global index": "MMLUPro_physics_7773", + "flip": 0 + }, + { + "global index": "MMLUPro_physics_7887", + "flip": 0 + }, + { + "global index": "MMLUPro_physics_7893", + "flip": 1 + }, + { + "global index": "MMLUPro_physics_8888", + "flip": 1 + }, + { + "global index": "MMLUPro_physics_9017", + "flip": 1 + }, + { + "global index": "MMLUPro_psychology_2005", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2186", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2329", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2406", + "flip": 1 + }, + { + "global index": "MMLUPro_psychology_2420", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2450", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2457", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2524", + "flip": 0 + }, + { + "global index": "MMLU_formal_logic_121", + "flip": 0 + }, + { + "global index": "MMLU_formal_logic_16", + "flip": 0 + }, + { + "global index": "MMLU_formal_logic_32", + "flip": 1 + }, + { + "global index": "MMLU_formal_logic_63", + "flip": 0 + }, + { + "global index": "MMLU_formal_logic_7", + "flip": 0 + }, + { + "global index": "MMLU_formal_logic_70", + "flip": 0 + }, + { + "global index": "MMLU_formal_logic_85", + "flip": 1 + }, + { + "global index": "MMLU_management_4", + "flip": 0 + }, + { + "global index": "MMLU_management_41", + "flip": 1 + }, + { + "global index": "MMLU_management_77", + "flip": 0 + }, + { + "global index": "MMLU_management_91", + "flip": 0 + }, + { + "global index": "MMLU_management_93", + "flip": 0 + }, + { + "global index": "MathQA_158", + "flip": 0 + }, + { + "global index": "MathQA_1742", + "flip": 0 + }, + { + "global index": "MathQA_202", + "flip": 0 + }, + { + "global index": "MathQA_2092", + "flip": 0 + }, + { + "global index": "MathQA_2102", + "flip": 0 + }, + { + "global index": "MathQA_2851", + "flip": 1 + }, + { + "global index": "MathQA_827", + "flip": 1 + }, + { + "global index": "MathQA_84", + "flip": 1 + }, + { + "global index": "MedMCQA_1005", + "flip": 0 + }, + { + "global index": "MedMCQA_1054", + "flip": 0 + }, + { + "global index": "MedMCQA_1298", + "flip": 0 + }, + { + "global index": "MedMCQA_1309", + "flip": 0 + }, + { + "global index": "MedMCQA_1362", + "flip": 0 + }, + { + "global index": "MedMCQA_145", + "flip": 0 + }, + { + "global index": "MedMCQA_2010", + "flip": 0 + }, + { + "global index": "MedMCQA_2323", + "flip": 0 + }, + { + "global index": "MedMCQA_2366", + "flip": 1 + }, + { + "global index": "MedMCQA_2581", + "flip": 0 + }, + { + "global index": "MedMCQA_511", + "flip": 0 + }, + { + "global index": "MedMCQA_59", + "flip": 1 + }, + { + "global index": "MedMCQA_643", + "flip": 0 + }, + { + "global index": "MedMCQA_853", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_126", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_14", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_147", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_152", + "flip": 1 + }, + { + "global index": "MusicTheoryBench_188", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_189", + "flip": 1 + }, + { + "global index": "MusicTheoryBench_240", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_33", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_337", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_340", + "flip": 1 + }, + { + "global index": "MusicTheoryBench_70", + "flip": 0 + }, + { + "global index": "NarrativeQA_131", + "flip": 0 + }, + { + "global index": "NarrativeQA_1683", + "flip": 0 + }, + { + "global index": "NarrativeQA_2474", + "flip": 0 + }, + { + "global index": "NarrativeQA_2820", + "flip": 1 + }, + { + "global index": "NarrativeQA_3282", + "flip": 0 + }, + { + "global index": "NarrativeQA_4102", + "flip": 1 + }, + { + "global index": "NarrativeQA_4128", + "flip": 1 + }, + { + "global index": "NarrativeQA_4347", + "flip": 1 + }, + { + "global index": "NarrativeQA_4540", + "flip": 1 + }, + { + "global index": "NarrativeQA_5022", + "flip": 1 + }, + { + "global index": "NarrativeQA_5259", + "flip": 0 + }, + { + "global index": "NarrativeQA_533", + "flip": 0 + }, + { + "global index": "NarrativeQA_5894", + "flip": 1 + }, + { + "global index": "NarrativeQA_6829", + "flip": 1 + }, + { + "global index": "NarrativeQA_7678", + "flip": 1 + }, + { + "global index": "NarrativeQA_7964", + "flip": 0 + }, + { + "global index": "NarrativeQA_8215", + "flip": 0 + }, + { + "global index": "NarrativeQA_8598", + "flip": 0 + }, + { + "global index": "NarrativeQA_927", + "flip": 0 + }, + { + "global index": "OpenTDB_Animals_2545", + "flip": 0 + }, + { + "global index": "OpenTDB_Animals_262", + "flip": 0 + }, + { + "global index": "OpenTDB_Art_1429", + "flip": 0 + }, + { + "global index": "OpenTDB_Celebrities_1078", + "flip": 0 + }, + { + "global index": "OpenTDB_Celebrities_1456", + "flip": 0 + }, + { + "global index": "OpenTDB_Celebrities_3577", + "flip": 0 + }, + { + "global index": "OpenTDB_Celebrities_3968", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_1178", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_1807", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_1957", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_2181", + "flip": 1 + }, + { + "global index": "OpenTDB_General Knowledge_3404", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_3727", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_4019", + "flip": 0 + }, + { + "global index": "OpenTDB_Geography_2099", + "flip": 0 + }, + { + "global index": "OpenTDB_Geography_2346", + "flip": 0 + }, + { + "global index": "OpenTDB_Geography_3880", + "flip": 0 + }, + { + "global index": "OpenTDB_Geography_792", + "flip": 0 + }, + { + "global index": "OpenTDB_History_1162", + "flip": 0 + }, + { + "global index": "OpenTDB_History_2026", + "flip": 0 + }, + { + "global index": "OpenTDB_History_3712", + "flip": 0 + }, + { + "global index": "OpenTDB_History_3902", + "flip": 0 + }, + { + "global index": "OpenTDB_Science & Nature_1175", + "flip": 0 + }, + { + "global index": "OpenTDB_Science & Nature_1560", + "flip": 0 + }, + { + "global index": "OpenTDB_Science & Nature_3716", + "flip": 0 + }, + { + "global index": "OpenTDB_Science & Nature_476", + "flip": 0 + }, + { + "global index": "OpenTDB_Sports_2289", + "flip": 0 + }, + { + "global index": "OpenTDB_Vehicles_1173", + "flip": 1 + }, + { + "global index": "OpenTDB_Vehicles_1419", + "flip": 0 + }, + { + "global index": "OpenTDB_Vehicles_2519", + "flip": 0 + }, + { + "global index": "OpenTDB_Vehicles_3234", + "flip": 0 + }, + { + "global index": "PubMedQA_0", + "flip": 0 + }, + { + "global index": "PubMedQA_154", + "flip": 1 + }, + { + "global index": "PubMedQA_18", + "flip": 0 + }, + { + "global index": "PubMedQA_238", + "flip": 0 + }, + { + "global index": "PubMedQA_250", + "flip": 0 + }, + { + "global index": "PubMedQA_337", + "flip": 1 + }, + { + "global index": "PubMedQA_362", + "flip": 0 + }, + { + "global index": "PubMedQA_437", + "flip": 0 + }, + { + "global index": "PubMedQA_510", + "flip": 0 + }, + { + "global index": "PubMedQA_520", + "flip": 1 + }, + { + "global index": "PubMedQA_575", + "flip": 0 + }, + { + "global index": "PubMedQA_582", + "flip": 0 + }, + { + "global index": "PubMedQA_588", + "flip": 0 + }, + { + "global index": "PubMedQA_610", + "flip": 0 + }, + { + "global index": "PubMedQA_63", + "flip": 0 + }, + { + "global index": "PubMedQA_643", + "flip": 0 + }, + { + "global index": "PubMedQA_687", + "flip": 0 + }, + { + "global index": "PubMedQA_722", + "flip": 0 + }, + { + "global index": "PubMedQA_73", + "flip": 1 + }, + { + "global index": "PubMedQA_755", + "flip": 0 + }, + { + "global index": "PubMedQA_8", + "flip": 0 + }, + { + "global index": "PubMedQA_81", + "flip": 0 + }, + { + "global index": "PubMedQA_854", + "flip": 0 + }, + { + "global index": "PubMedQA_905", + "flip": 0 + }, + { + "global index": "QANTA_Fine Arts_1212", + "flip": 0 + }, + { + "global index": "QANTA_Fine Arts_1702", + "flip": 0 + }, + { + "global index": "QANTA_Fine Arts_828", + "flip": 0 + }, + { + "global index": "QANTA_Fine Arts_865", + "flip": 0 + }, + { + "global index": "QANTA_Geography_1023", + "flip": 0 + }, + { + "global index": "QANTA_Geography_1555", + "flip": 0 + }, + { + "global index": "QANTA_Geography_304", + "flip": 0 + }, + { + "global index": "QANTA_History_1084", + "flip": 0 + }, + { + "global index": "QANTA_History_1154", + "flip": 0 + }, + { + "global index": "QANTA_History_433", + "flip": 0 + }, + { + "global index": "QANTA_History_473", + "flip": 0 + }, + { + "global index": "QANTA_History_926", + "flip": 0 + }, + { + "global index": "QANTA_Literature_1045", + "flip": 0 + }, + { + "global index": "QANTA_Literature_1073", + "flip": 0 + }, + { + "global index": "QANTA_Literature_1239", + "flip": 0 + }, + { + "global index": "QANTA_Literature_1326", + "flip": 0 + }, + { + "global index": "QANTA_Literature_1727", + "flip": 0 + }, + { + "global index": "QANTA_Literature_1843", + "flip": 0 + }, + { + "global index": "QANTA_Literature_386", + "flip": 0 + }, + { + "global index": "QANTA_Literature_408", + "flip": 0 + }, + { + "global index": "QANTA_Literature_475", + "flip": 0 + }, + { + "global index": "QANTA_Literature_833", + "flip": 1 + }, + { + "global index": "QANTA_Philosophy_1270", + "flip": 0 + }, + { + "global index": "QANTA_Philosophy_499", + "flip": 0 + }, + { + "global index": "QANTA_Philosophy_91", + "flip": 1 + }, + { + "global index": "QANTA_Science_1360", + "flip": 0 + }, + { + "global index": "QANTA_Science_1473", + "flip": 0 + }, + { + "global index": "QANTA_Science_308", + "flip": 0 + }, + { + "global index": "QANTA_Science_619", + "flip": 1 + }, + { + "global index": "QANTA_Social Science_1847", + "flip": 0 + }, + { + "global index": "QANTA_Social Science_2", + "flip": 0 + }, + { + "global index": "QANTA_Social Science_77", + "flip": 0 + }, + { + "global index": "SocialiQA_13810", + "flip": 1 + }, + { + "global index": "SocialiQA_22095", + "flip": 0 + }, + { + "global index": "SocialiQA_26846", + "flip": 0 + }, + { + "global index": "SocialiQA_7839", + "flip": 1 + }, + { + "global index": "SuperGLUE-CausalReasoning_4526", + "flip": 0 + }, + { + "global index": "SuperGLUE-ClozeTest_12894", + "flip": 0 + }, + { + "global index": "SuperGLUE-ClozeTest_17965", + "flip": 0 + }, + { + "global index": "SuperGLUE-ClozeTest_18766", + "flip": 0 + }, + { + "global index": "SuperGLUE-Entailment_19410", + "flip": 0 + }, + { + "global index": "SuperGLUE-Entailment_19567", + "flip": 0 + }, + { + "global index": "SuperGLUE-Entailment_522", + "flip": 0 + }, + { + "global index": "SuperGLUE-Entailment_767", + "flip": 0 + }, + { + "global index": "SuperGLUE-QA_1408", + "flip": 0 + }, + { + "global index": "SuperGLUE-QA_3137", + "flip": 0 + }, + { + "global index": "SuperGLUE-QA_4046", + "flip": 0 + }, + { + "global index": "SuperGLUE-QA_4102", + "flip": 0 + }, + { + "global index": "SuperGLUE-QA_4160", + "flip": 0 + }, + { + "global index": "SuperGLUE-RC_7725", + "flip": 1 + }, + { + "global index": "SuperGLUE-RC_7738", + "flip": 0 + }, + { + "global index": "SuperGLUE-RC_8531", + "flip": 0 + }, + { + "global index": "SuperGLUE-Wic_19695", + "flip": 1 + }, + { + "global index": "SuperGLUE-Wic_19738", + "flip": 0 + }, + { + "global index": "SuperGLUE-Wic_20079", + "flip": 0 + }, + { + "global index": "SuperGLUE-Wic_20189", + "flip": 0 + }, + { + "global index": "SuperGLUE-Wic_20253", + "flip": 0 + }, + { + "global index": "SuperGLUE-Wsc_20368", + "flip": 0 + }, + { + "global index": "SuperGLUE-Wsc_20370", + "flip": 0 + }, + { + "global index": "WMT19-cs-en_156", + "flip": 0 + }, + { + "global index": "WMT19-cs-en_246", + "flip": 0 + }, + { + "global index": "WMT19-cs-en_568", + "flip": 0 + }, + { + "global index": "WMT19-de-en_46", + "flip": 0 + }, + { + "global index": "WMT19-de-en_715", + "flip": 0 + }, + { + "global index": "WMT19-de-en_883", + "flip": 1 + }, + { + "global index": "WMT19-fi-en_222", + "flip": 0 + }, + { + "global index": "WMT19-fi-en_610", + "flip": 0 + }, + { + "global index": "WMT19-gu-en_116", + "flip": 0 + }, + { + "global index": "WMT19-gu-en_123", + "flip": 0 + }, + { + "global index": "WMT19-gu-en_191", + "flip": 0 + }, + { + "global index": "WMT19-gu-en_491", + "flip": 0 + }, + { + "global index": "WMT19-gu-en_968", + "flip": 0 + }, + { + "global index": "WMT19-kk-en_528", + "flip": 0 + }, + { + "global index": "WMT19-kk-en_826", + "flip": 0 + }, + { + "global index": "WMT19-lt-en_135", + "flip": 0 + }, + { + "global index": "WMT19-lt-en_269", + "flip": 0 + }, + { + "global index": "WMT19-lt-en_636", + "flip": 0 + }, + { + "global index": "WMT19-ru-en_222", + "flip": 0 + }, + { + "global index": "WMT19-zh-en_218", + "flip": 0 + }, + { + "global index": "WMT19-zh-en_252", + "flip": 1 + }, + { + "global index": "WMT19-zh-en_59", + "flip": 0 + } +] diff --git a/src/data/routerMetrics/category_scores.json b/src/data/routerMetrics/category_scores.json index 07b4840..9c6f70f 100644 --- a/src/data/routerMetrics/category_scores.json +++ b/src/data/routerMetrics/category_scores.json @@ -23254,47 +23254,47 @@ "divyam": { "metrics": { "easy": { - "accuracy": 97.6, - "cost": 0.0003, - "robustness": 98.0 + "accuracy": 97.4, + "cost": 0.0004, + "robustness": 72.7 }, "medium": { - "accuracy": 78.3, - "cost": 0.0006, - "robustness": 99.1 + "accuracy": 81.2, + "cost": 0.0008, + "robustness": 77.0 }, "hard": { - "accuracy": 40.5, - "cost": 0.0007, - "robustness": 98.2 + "accuracy": 45.1, + "cost": 0.0011, + "robustness": 79.8 }, "all": { - "accuracy": 78.6, - "cost": 0.0005, - "robustness": 98.3 + "accuracy": 80.4, + "cost": 0.0007, + "robustness": 75.7 } }, "categories": { "Arts & recreation": { "metrics": { "easy": { - "accuracy": 97.3, - "cost": 0.0003, + "accuracy": 97.0, + "cost": 0.0005, "robustness": 100.0 }, "medium": { - "accuracy": 71.8, - "cost": 0.0005, + "accuracy": 81.2, + "cost": 0.0016, "robustness": 100.0 }, "hard": { - "accuracy": 26.5, - "cost": 0.0006, + "accuracy": 34.0, + "cost": 0.0022, "robustness": 100.0 }, "all": { - "accuracy": 73.1, - "cost": 0.0005, + "accuracy": 78.0, + "cost": 0.0013, "robustness": 100.0 } }, @@ -23303,22 +23303,22 @@ "metrics": { "easy": { "accuracy": 96.6, - "cost": 0.0002, + "cost": 0.0003, "robustness": 100.0 }, "medium": { "accuracy": 94.9, - "cost": 0.0003, + "cost": 0.0006, "robustness": 100.0 }, "hard": { - "accuracy": 26.8, - "cost": 0.0003, + "accuracy": 33.9, + "cost": 0.0011, "robustness": 100.0 }, "all": { - "accuracy": 70.8, - "cost": 0.0003, + "accuracy": 73.4, + "cost": 0.0007, "robustness": 100.0 } } @@ -23326,23 +23326,23 @@ "Music": { "metrics": { "easy": { - "accuracy": 96.0, - "cost": 0.0003, + "accuracy": 98.0, + "cost": 0.0005, "robustness": 100.0 }, "medium": { - "accuracy": 56.0, - "cost": 0.0006, + "accuracy": 69.0, + "cost": 0.0022, "robustness": 100.0 }, "hard": { "accuracy": 24.0, - "cost": 0.0006, + "cost": 0.0034, "robustness": 100.0 }, "all": { - "accuracy": 69.2, - "cost": 0.0005, + "accuracy": 76.2, + "cost": 0.0016, "robustness": 100.0 } } @@ -23350,23 +23350,23 @@ "Sports, games and entertainment": { "metrics": { "easy": { - "accuracy": 98.6, - "cost": 0.0004, + "accuracy": 96.4, + "cost": 0.0006, "robustness": 100.0 }, "medium": { - "accuracy": 81.0, - "cost": 0.0006, + "accuracy": 90.0, + "cost": 0.0013, "robustness": 100.0 }, "hard": { - "accuracy": 27.3, - "cost": 0.0009, + "accuracy": 37.9, + "cost": 0.0028, "robustness": 100.0 }, "all": { - "accuracy": 77.5, - "cost": 0.0006, + "accuracy": 81.7, + "cost": 0.0013, "robustness": 100.0 } } @@ -23376,48 +23376,48 @@ "Computer science, information, and general works": { "metrics": { "easy": { - "accuracy": 99.0, - "cost": 0.0003, - "robustness": 100.0 + "accuracy": 98.4, + "cost": 0.0004, + "robustness": 73.1 }, "medium": { - "accuracy": 90.9, - "cost": 0.0008, - "robustness": 100.0 + "accuracy": 89.2, + "cost": 0.0006, + "robustness": 76.2 }, "hard": { - "accuracy": 65.7, - "cost": 0.0016, - "robustness": 93.3 + "accuracy": 53.9, + "cost": 0.0013, + "robustness": 100.0 }, "all": { - "accuracy": 90.5, - "cost": 0.0007, - "robustness": 98.4 + "accuracy": 87.6, + "cost": 0.0006, + "robustness": 80.6 } }, "subcategories": { "Computer science, knowledge, and systems": { "metrics": { "easy": { - "accuracy": 99.8, + "accuracy": 98.8, "cost": 0.0004, - "robustness": 100.0 + "robustness": 76.9 }, "medium": { - "accuracy": 93.6, - "cost": 0.0009, - "robustness": 100.0 + "accuracy": 90.7, + "cost": 0.0006, + "robustness": 76.5 }, "hard": { - "accuracy": 73.7, - "cost": 0.0018, - "robustness": 92.3 + "accuracy": 59.3, + "cost": 0.0015, + "robustness": 100.0 }, "all": { - "accuracy": 92.0, - "cost": 0.0009, - "robustness": 97.7 + "accuracy": 87.5, + "cost": 0.0007, + "robustness": 83.7 } } }, @@ -23425,23 +23425,23 @@ "metrics": { "easy": { "accuracy": 97.8, - "cost": 0.0003, - "robustness": 100.0 + "cost": 0.0004, + "robustness": 69.2 }, "medium": { - "accuracy": 78.8, - "cost": 0.0003, - "robustness": 100.0 + "accuracy": 82.4, + "cost": 0.0005, + "robustness": 75.0 }, "hard": { - "accuracy": 19.4, - "cost": 0.0003, + "accuracy": 22.2, + "cost": 0.0006, "robustness": 100.0 }, "all": { - "accuracy": 86.6, - "cost": 0.0003, - "robustness": 100.0 + "accuracy": 87.6, + "cost": 0.0004, + "robustness": 73.7 } } } @@ -23450,24 +23450,24 @@ "History": { "metrics": { "easy": { - "accuracy": 98.1, - "cost": 0.0003, - "robustness": 95.7 + "accuracy": 98.7, + "cost": 0.0004, + "robustness": 65.2 }, "medium": { - "accuracy": 83.1, - "cost": 0.0004, - "robustness": 100.0 + "accuracy": 86.4, + "cost": 0.0006, + "robustness": 83.3 }, "hard": { - "accuracy": 28.9, - "cost": 0.0004, - "robustness": 100.0 + "accuracy": 37.6, + "cost": 0.0009, + "robustness": 70.0 }, "all": { - "accuracy": 79.6, - "cost": 0.0003, - "robustness": 97.4 + "accuracy": 82.6, + "cost": 0.0005, + "robustness": 69.2 } }, "subcategories": { @@ -23475,23 +23475,23 @@ "metrics": { "easy": { "accuracy": 97.6, - "cost": 0.0002, - "robustness": 100.0 + "cost": 0.0003, + "robustness": 75.0 }, "medium": { "accuracy": 100.0, - "cost": 0.0002, + "cost": 0.0003, "robustness": 0 }, "hard": { "accuracy": 0.0, - "cost": 0.0003, + "cost": 0.0, "robustness": 0 }, "all": { "accuracy": 96.2, - "cost": 0.0002, - "robustness": 100.0 + "cost": 0.0003, + "robustness": 75.0 } } }, @@ -23500,46 +23500,46 @@ "easy": { "accuracy": 99.1, "cost": 0.0002, - "robustness": 100.0 + "robustness": 62.5 }, "medium": { - "accuracy": 62.1, - "cost": 0.0002, + "accuracy": 65.5, + "cost": 0.0004, "robustness": 100.0 }, "hard": { - "accuracy": 58.3, - "cost": 0.0002, + "accuracy": 66.7, + "cost": 0.0007, "robustness": 0 }, "all": { - "accuracy": 88.7, - "cost": 0.0002, - "robustness": 100.0 + "accuracy": 90.0, + "cost": 0.0003, + "robustness": 66.7 } } }, "History": { "metrics": { "easy": { - "accuracy": 97.8, - "cost": 0.0004, - "robustness": 90.9 + "accuracy": 98.7, + "cost": 0.0005, + "robustness": 63.6 }, "medium": { - "accuracy": 86.3, - "cost": 0.0004, - "robustness": 100.0 + "accuracy": 89.9, + "cost": 0.0007, + "robustness": 80.0 }, "hard": { - "accuracy": 26.5, - "cost": 0.0004, - "robustness": 100.0 + "accuracy": 35.3, + "cost": 0.0009, + "robustness": 70.0 }, "all": { - "accuracy": 75.1, - "cost": 0.0004, - "robustness": 96.2 + "accuracy": 78.9, + "cost": 0.0006, + "robustness": 69.2 } } } @@ -23550,22 +23550,22 @@ "easy": { "accuracy": 93.3, "cost": 0.0003, - "robustness": 100.0 + "robustness": 54.5 }, "medium": { - "accuracy": 50.3, - "cost": 0.0004, - "robustness": 100.0 + "accuracy": 52.1, + "cost": 0.0006, + "robustness": 66.7 }, "hard": { - "accuracy": 56.2, + "accuracy": 57.8, "cost": 0.0005, - "robustness": 96.0 + "robustness": 64.0 }, "all": { - "accuracy": 65.9, - "cost": 0.0004, - "robustness": 97.8 + "accuracy": 67.0, + "cost": 0.0005, + "robustness": 62.2 } }, "subcategories": { @@ -23574,22 +23574,22 @@ "easy": { "accuracy": 93.3, "cost": 0.0003, - "robustness": 100.0 + "robustness": 54.5 }, "medium": { - "accuracy": 50.3, - "cost": 0.0004, - "robustness": 100.0 + "accuracy": 52.1, + "cost": 0.0006, + "robustness": 66.7 }, "hard": { - "accuracy": 56.2, + "accuracy": 57.8, "cost": 0.0005, - "robustness": 96.0 + "robustness": 64.0 }, "all": { - "accuracy": 65.9, - "cost": 0.0004, - "robustness": 97.8 + "accuracy": 67.0, + "cost": 0.0005, + "robustness": 62.2 } } } @@ -23600,22 +23600,22 @@ "easy": { "accuracy": 100.0, "cost": 0.0003, - "robustness": 100.0 + "robustness": 66.7 }, "medium": { - "accuracy": 83.4, - "cost": 0.0003, - "robustness": 100.0 + "accuracy": 90.5, + "cost": 0.0006, + "robustness": 50.0 }, "hard": { - "accuracy": 40.1, - "cost": 0.0005, - "robustness": 100.0 + "accuracy": 49.2, + "cost": 0.0011, + "robustness": 83.3 }, "all": { - "accuracy": 53.9, - "cost": 0.0005, - "robustness": 100.0 + "accuracy": 61.4, + "cost": 0.0009, + "robustness": 79.3 } }, "subcategories": { @@ -23624,22 +23624,22 @@ "easy": { "accuracy": 100.0, "cost": 0.0003, - "robustness": 100.0 + "robustness": 66.7 }, "medium": { - "accuracy": 83.4, - "cost": 0.0003, - "robustness": 100.0 + "accuracy": 90.5, + "cost": 0.0006, + "robustness": 50.0 }, "hard": { - "accuracy": 40.1, - "cost": 0.0005, - "robustness": 100.0 + "accuracy": 49.2, + "cost": 0.0011, + "robustness": 83.3 }, "all": { - "accuracy": 53.9, - "cost": 0.0005, - "robustness": 100.0 + "accuracy": 61.4, + "cost": 0.0009, + "robustness": 79.3 } } } @@ -23648,96 +23648,96 @@ "Philosophy and psychology": { "metrics": { "easy": { - "accuracy": 97.1, + "accuracy": 95.8, "cost": 0.0003, - "robustness": 92.6 + "robustness": 66.7 }, "medium": { - "accuracy": 71.1, - "cost": 0.0004, - "robustness": 100.0 + "accuracy": 75.4, + "cost": 0.0006, + "robustness": 80.0 }, "hard": { - "accuracy": 35.8, - "cost": 0.0004, - "robustness": 100.0 + "accuracy": 37.0, + "cost": 0.0009, + "robustness": 57.1 }, "all": { - "accuracy": 82.1, - "cost": 0.0003, - "robustness": 95.5 + "accuracy": 82.9, + "cost": 0.0005, + "robustness": 68.2 } }, "subcategories": { "Ethics": { "metrics": { "easy": { - "accuracy": 95.7, - "cost": 0.0003, - "robustness": 92.9 + "accuracy": 93.0, + "cost": 0.0002, + "robustness": 64.3 }, "medium": { - "accuracy": 42.4, - "cost": 0.0003, - "robustness": 100.0 + "accuracy": 48.2, + "cost": 0.0006, + "robustness": 80.0 }, "hard": { "accuracy": 0.0, "cost": 0.0003, - "robustness": 100.0 + "robustness": 0.0 }, "all": { "accuracy": 75.3, "cost": 0.0003, - "robustness": 95.2 + "robustness": 61.9 } } }, "Philosophical logic": { "metrics": { "easy": { - "accuracy": 98.5, + "accuracy": 100.0, "cost": 0.0004, - "robustness": 100.0 + "robustness": 25.0 }, "medium": { - "accuracy": 84.2, - "cost": 0.0005, + "accuracy": 94.7, + "cost": 0.0009, "robustness": 100.0 }, "hard": { - "accuracy": 60.0, - "cost": 0.0007, + "accuracy": 80.0, + "cost": 0.0017, "robustness": 0 }, "all": { - "accuracy": 91.7, - "cost": 0.0004, - "robustness": 100.0 + "accuracy": 97.2, + "cost": 0.0006, + "robustness": 57.1 } } }, "Philosophy": { "metrics": { "easy": { - "accuracy": 97.3, - "cost": 0.0003, - "robustness": 100.0 + "accuracy": 94.6, + "cost": 0.0005, + "robustness": 66.7 }, "medium": { "accuracy": 95.0, - "cost": 0.0004, - "robustness": 100.0 + "cost": 0.0008, + "robustness": 50.0 }, "hard": { - "accuracy": 42.2, - "cost": 0.0004, + "accuracy": 44.4, + "cost": 0.0011, "robustness": 100.0 }, "all": { "accuracy": 76.2, - "cost": 0.0004, - "robustness": 100.0 + "cost": 0.0008, + "robustness": 75.0 } } }, @@ -23745,8 +23745,8 @@ "metrics": { "easy": { "accuracy": 98.3, - "cost": 0.0003, - "robustness": 83.3 + "cost": 0.0004, + "robustness": 100.0 }, "medium": { "accuracy": 91.7, @@ -23754,12 +23754,12 @@ "robustness": 0 }, "hard": { - "accuracy": 38.9, - "cost": 0.0004, - "robustness": 100.0 + "accuracy": 33.3, + "cost": 0.0005, + "robustness": 50.0 }, "all": { - "accuracy": 90.9, + "accuracy": 90.4, "cost": 0.0004, "robustness": 87.5 } @@ -23770,24 +23770,24 @@ "Science": { "metrics": { "easy": { - "accuracy": 98.6, + "accuracy": 99.0, "cost": 0.0003, - "robustness": 100.0 + "robustness": 69.2 }, "medium": { - "accuracy": 83.3, + "accuracy": 84.3, "cost": 0.0006, - "robustness": 100.0 + "robustness": 76.2 }, "hard": { - "accuracy": 26.3, - "cost": 0.0008, - "robustness": 100.0 + "accuracy": 41.2, + "cost": 0.0011, + "robustness": 88.9 }, "all": { - "accuracy": 83.9, + "accuracy": 86.4, "cost": 0.0005, - "robustness": 100.0 + "robustness": 73.9 } }, "subcategories": { @@ -23796,7 +23796,7 @@ "easy": { "accuracy": 100.0, "cost": 0.0002, - "robustness": 100.0 + "robustness": 50.0 }, "medium": { "accuracy": 80.0, @@ -23811,7 +23811,7 @@ "all": { "accuracy": 96.0, "cost": 0.0002, - "robustness": 100.0 + "robustness": 50.0 } } }, @@ -23820,11 +23820,11 @@ "easy": { "accuracy": 100.0, "cost": 0.0004, - "robustness": 100.0 + "robustness": 71.4 }, "medium": { "accuracy": 100.0, - "cost": 0.0005, + "cost": 0.0006, "robustness": 0 }, "hard": { @@ -23835,7 +23835,7 @@ "all": { "accuracy": 96.5, "cost": 0.0004, - "robustness": 100.0 + "robustness": 71.4 } } }, @@ -23844,22 +23844,22 @@ "easy": { "accuracy": 100.0, "cost": 0.0004, - "robustness": 100.0 + "robustness": 33.3 }, "medium": { - "accuracy": 94.1, + "accuracy": 91.2, "cost": 0.0006, - "robustness": 100.0 + "robustness": 50.0 }, "hard": { - "accuracy": 23.1, - "cost": 0.0005, + "accuracy": 38.5, + "cost": 0.001, "robustness": 0 }, "all": { - "accuracy": 86.7, + "accuracy": 87.8, "cost": 0.0005, - "robustness": 100.0 + "robustness": 40.0 } } }, @@ -23868,70 +23868,70 @@ "easy": { "accuracy": 98.4, "cost": 0.0003, - "robustness": 100.0 + "robustness": 75.0 }, "medium": { - "accuracy": 69.5, - "cost": 0.0003, - "robustness": 100.0 + "accuracy": 72.9, + "cost": 0.0005, + "robustness": 83.3 }, "hard": { - "accuracy": 23.1, - "cost": 0.0003, - "robustness": 100.0 + "accuracy": 46.2, + "cost": 0.0008, + "robustness": 0.0 }, "all": { - "accuracy": 90.3, + "accuracy": 91.8, "cost": 0.0003, - "robustness": 100.0 + "robustness": 73.3 } } }, "Mathematics": { "metrics": { "easy": { - "accuracy": 97.7, - "cost": 0.0004, - "robustness": 100.0 + "accuracy": 98.5, + "cost": 0.0002, + "robustness": 81.8 }, "medium": { - "accuracy": 85.2, - "cost": 0.0007, - "robustness": 100.0 + "accuracy": 84.8, + "cost": 0.0006, + "robustness": 81.8 }, "hard": { - "accuracy": 31.5, - "cost": 0.0013, + "accuracy": 55.1, + "cost": 0.0016, "robustness": 100.0 }, "all": { - "accuracy": 82.7, - "cost": 0.0007, - "robustness": 100.0 + "accuracy": 86.4, + "cost": 0.0006, + "robustness": 85.2 } } }, "Physics": { "metrics": { "easy": { - "accuracy": 97.7, - "cost": 0.0003, - "robustness": 100.0 + "accuracy": 100.0, + "cost": 0.0004, + "robustness": 33.3 }, "medium": { - "accuracy": 88.1, - "cost": 0.0006, - "robustness": 100.0 + "accuracy": 85.7, + "cost": 0.0007, + "robustness": 50.0 }, "hard": { - "accuracy": 45.5, - "cost": 0.0005, + "accuracy": 36.4, + "cost": 0.0006, "robustness": 0 }, "all": { - "accuracy": 87.6, + "accuracy": 86.6, "cost": 0.0005, - "robustness": 100.0 + "robustness": 40.0 } } }, @@ -23939,23 +23939,23 @@ "metrics": { "easy": { "accuracy": 100.0, - "cost": 0.0003, - "robustness": 100.0 + "cost": 0.0002, + "robustness": 80.0 }, "medium": { - "accuracy": 64.0, - "cost": 0.0003, + "accuracy": 84.0, + "cost": 0.0006, "robustness": 0 }, "hard": { - "accuracy": 13.6, - "cost": 0.0003, + "accuracy": 20.3, + "cost": 0.0008, "robustness": 100.0 }, "all": { - "accuracy": 47.4, - "cost": 0.0003, - "robustness": 100.0 + "accuracy": 55.3, + "cost": 0.0006, + "robustness": 87.5 } } } @@ -23964,71 +23964,71 @@ "Social Science": { "metrics": { "easy": { - "accuracy": 96.6, - "cost": 0.0004, - "robustness": 100.0 + "accuracy": 97.0, + "cost": 0.0005, + "robustness": 88.9 }, "medium": { - "accuracy": 77.9, - "cost": 0.0005, - "robustness": 91.7 + "accuracy": 80.6, + "cost": 0.0008, + "robustness": 75.0 }, "hard": { - "accuracy": 22.8, - "cost": 0.0005, - "robustness": 100.0 + "accuracy": 26.2, + "cost": 0.0011, + "robustness": 85.7 }, "all": { - "accuracy": 74.1, - "cost": 0.0005, - "robustness": 97.3 + "accuracy": 76.0, + "cost": 0.0007, + "robustness": 83.8 } }, "subcategories": { "Economics": { "metrics": { "easy": { - "accuracy": 96.2, - "cost": 0.0004, - "robustness": 100.0 + "accuracy": 97.3, + "cost": 0.0005, + "robustness": 83.3 }, "medium": { - "accuracy": 77.9, - "cost": 0.0006, - "robustness": 83.3 + "accuracy": 83.4, + "cost": 0.0008, + "robustness": 50.0 }, "hard": { - "accuracy": 22.5, - "cost": 0.0006, + "accuracy": 26.2, + "cost": 0.0012, "robustness": 100.0 }, "all": { - "accuracy": 75.4, - "cost": 0.0005, - "robustness": 95.2 + "accuracy": 78.6, + "cost": 0.0008, + "robustness": 76.2 } } }, "Law": { "metrics": { "easy": { - "accuracy": 98.5, - "cost": 0.0005, + "accuracy": 96.9, + "cost": 0.0008, "robustness": 100.0 }, "medium": { - "accuracy": 82.5, - "cost": 0.0005, + "accuracy": 77.8, + "cost": 0.001, "robustness": 100.0 }, "hard": { - "accuracy": 33.3, - "cost": 0.0005, + "accuracy": 23.3, + "cost": 0.001, "robustness": 100.0 }, "all": { - "accuracy": 79.7, - "cost": 0.0005, + "accuracy": 75.3, + "cost": 0.0009, "robustness": 100.0 } } @@ -24037,23 +24037,23 @@ "metrics": { "easy": { "accuracy": 95.7, - "cost": 0.0002, + "cost": 0.0001, "robustness": 100.0 }, "medium": { "accuracy": 33.3, - "cost": 0.0002, + "cost": 0.0001, "robustness": 0 }, "hard": { - "accuracy": 0.0, - "cost": 0.0002, - "robustness": 100.0 + "accuracy": 20.0, + "cost": 0.0001, + "robustness": 0.0 }, "all": { - "accuracy": 78.7, - "cost": 0.0002, - "robustness": 100.0 + "accuracy": 80.3, + "cost": 0.0001, + "robustness": 75.0 } } }, @@ -24061,22 +24061,22 @@ "metrics": { "easy": { "accuracy": 100.0, - "cost": 0.0002, + "cost": 0.0005, "robustness": 0 }, "medium": { - "accuracy": 83.3, - "cost": 0.0003, + "accuracy": 88.9, + "cost": 0.0007, "robustness": 100.0 }, "hard": { - "accuracy": 17.6, - "cost": 0.0003, + "accuracy": 29.4, + "cost": 0.001, "robustness": 100.0 }, "all": { - "accuracy": 43.6, - "cost": 0.0003, + "accuracy": 52.7, + "cost": 0.0009, "robustness": 100.0 } } @@ -24086,48 +24086,48 @@ "Technology": { "metrics": { "easy": { - "accuracy": 97.0, - "cost": 0.0004, - "robustness": 97.7 + "accuracy": 96.1, + "cost": 0.0005, + "robustness": 76.7 }, "medium": { - "accuracy": 75.5, - "cost": 0.0006, - "robustness": 100.0 + "accuracy": 80.5, + "cost": 0.0008, + "robustness": 71.4 }, "hard": { - "accuracy": 30.6, - "cost": 0.0005, - "robustness": 100.0 + "accuracy": 36.3, + "cost": 0.0009, + "robustness": 85.7 }, "all": { - "accuracy": 83.1, - "cost": 0.0005, - "robustness": 98.6 + "accuracy": 84.7, + "cost": 0.0006, + "robustness": 76.1 } }, "subcategories": { "Engineering": { "metrics": { "easy": { - "accuracy": 97.6, - "cost": 0.0005, + "accuracy": 96.8, + "cost": 0.0004, "robustness": 100.0 }, "medium": { - "accuracy": 89.2, + "accuracy": 91.6, "cost": 0.0008, - "robustness": 100.0 + "robustness": 76.9 }, "hard": { - "accuracy": 54.8, - "cost": 0.0008, + "accuracy": 61.9, + "cost": 0.0012, "robustness": 100.0 }, "all": { - "accuracy": 88.1, + "accuracy": 89.9, "cost": 0.0007, - "robustness": 100.0 + "robustness": 84.2 } } }, @@ -24136,11 +24136,11 @@ "easy": { "accuracy": 96.7, "cost": 0.0003, - "robustness": 100.0 + "robustness": 75.0 }, "medium": { - "accuracy": 63.6, - "cost": 0.0003, + "accuracy": 72.7, + "cost": 0.0005, "robustness": 100.0 }, "hard": { @@ -24149,33 +24149,33 @@ "robustness": 0 }, "all": { - "accuracy": 90.4, + "accuracy": 91.8, "cost": 0.0003, - "robustness": 100.0 + "robustness": 80.0 } } }, "Medicine and health": { "metrics": { "easy": { - "accuracy": 96.9, - "cost": 0.0004, - "robustness": 97.1 + "accuracy": 95.9, + "cost": 0.0005, + "robustness": 73.5 }, "medium": { - "accuracy": 66.5, - "cost": 0.0004, - "robustness": 100.0 + "accuracy": 73.1, + "cost": 0.0007, + "robustness": 57.1 }, "hard": { - "accuracy": 21.2, - "cost": 0.0004, - "robustness": 100.0 + "accuracy": 26.5, + "cost": 0.0009, + "robustness": 83.3 }, "all": { - "accuracy": 80.8, - "cost": 0.0004, - "robustness": 97.9 + "accuracy": 82.5, + "cost": 0.0006, + "robustness": 72.3 } } } diff --git a/src/data/routerMetrics/leaderboard.json b/src/data/routerMetrics/leaderboard.json index 33c91fc..7676abd 100644 --- a/src/data/routerMetrics/leaderboard.json +++ b/src/data/routerMetrics/leaderboard.json @@ -1,4 +1,26 @@ [ + { + "Router Name": "BizRouter", + "Arena Score": 77.04, + "Optimal Selection Score": null, + "Optimal Cost Score": null, + "Optimal Acc. Score": null, + "Robustness Score": 97.38, + "Latency Score": null, + "Accuracy": 81.61, + "Cost per 1k": 1.0 + }, + { + "Router Name": "Divyam", + "Arena Score": 76.88, + "Optimal Selection Score": 6.57, + "Optimal Cost Score": 16.91, + "Optimal Acc. Score": 92.85, + "Robustness Score": 75.71, + "Latency Score": null, + "Accuracy": 80.43, + "Cost per 1k": 0.65 + }, { "Router Name": "KT-ModelRouter", "Arena Score": 76.28, @@ -32,17 +54,6 @@ "Accuracy": 78.94, "Cost per 1k": 0.48 }, - { - "Router Name": "Divyam", - "Arena Score": 75.85, - "Optimal Selection Score": 4.93, - "Optimal Cost Score": 16.03, - "Optimal Acc. Score": 93.43, - "Robustness Score": 98.33, - "Latency Score": null, - "Accuracy": 78.59, - "Cost per 1k": 0.48 - }, { "Router Name": "Cross-Router", "Arena Score": 75.75, @@ -120,6 +131,17 @@ "Accuracy": 75.72, "Cost per 1k": 0.36 }, + { + "Router Name": "NuRoute Artemis", + "Arena Score": 72.87, + "Optimal Selection Score": null, + "Optimal Cost Score": null, + "Optimal Acc. Score": null, + "Robustness Score": 80.0, + "Latency Score": null, + "Accuracy": 75.47, + "Cost per 1k": 0.6 + }, { "Router Name": "Weave Router", "Arena Score": 72.82, diff --git a/src/data/routers.json b/src/data/routers.json index a456213..d0882a0 100644 --- a/src/data/routers.json +++ b/src/data/routers.json @@ -561,5 +561,54 @@ "deepseek/deepseek-v4-flash", "Qwen/Qwen3-Coder-Next" ] + }, + "BizRouter": { + "name": "BizRouter", + "affiliation": "BizRouter", + "githubUrl": "https://bizrouter.ai", + "type": "closed-source", + "description": "Submitted by BizRouter.", + "modelPool": [ + "MiniMax-M3", + "anthropic/claude-opus-5.5", + "anthropic/claude-sonnet-5.5", + "claude-haiku-4.5", + "gemini-2.5-flash-lite", + "gemini-3-flash-preview", + "google/gemini-3.1-flash-lite", + "google/gemini-3.5-flash-lite", + "google/gemini-3.7-flash", + "google/gemini-3.8-flash", + "google/gemma-4-31b-it", + "gpt-5.4", + "gpt-5.4-mini", + "gpt-5.4-nano", + "moonshotai/kimi-k3", + "openai/gpt-5.6-luna", + "openai/gpt-5.6-sol", + "openai/gpt-5.6-terra", + "openai/gpt-6-luna", + "openai/gpt-6-sol", + "openai/gpt-6.1-sol", + "upstage/solar-mini", + "upstage/solar-pro-2", + "upstage/solar-pro3", + "upstage/solar-pro4", + "z-ai/glm-5.3-flash" + ] + }, + "NuRoute Artemis": { + "name": "NuRoute Artemis", + "affiliation": "NuRoute", + "githubUrl": "https://nuroute.ai", + "type": "closed-source", + "description": "Submitted by NuRoute.", + "modelPool": [ + "deepseek-ai/DeepSeek-V4-Flash-0731", + "gpt-5.6-luna", + "ministral-8b-latest", + "moonshotai/Kimi-K2.6", + "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B" + ] } }