diff --git a/new_results/figures/opt_java_metrics_comparison_ctx_0.2.pdf b/new_results/figures/opt_java_metrics_comparison_ctx_0.2.pdf new file mode 100644 index 0000000..806a3ba Binary files /dev/null and b/new_results/figures/opt_java_metrics_comparison_ctx_0.2.pdf differ diff --git a/new_results/figures/plot_pareto.py b/new_results/figures/plot_pareto.py new file mode 100644 index 0000000..77cdbeb --- /dev/null +++ b/new_results/figures/plot_pareto.py @@ -0,0 +1,19 @@ +import matplotlib.pyplot as plt +import os +energy = [12.13, 14.65, 19.32, 24.01, 28.00, 32.55, 37.07, 45.85, 55.01, 64.01, 71.89] +codebleu = [0.3085, 0.3116, 0.3173, 0.3300, 0.3312, 0.3322, 0.3298, 0.3329, 0.3464, 0.3627, 0.3991] +labels = ["L3", "L5", "L7", "L9", "L11", "L13", "L15", "L19", "L23", "L27", "L31"] +plt.figure(figsize=(9, 6)) +plt.plot(energy, codebleu, marker='o', linestyle='-', linewidth=2, markersize=8, color='#1f77b4') +for i, txt in enumerate(labels): + offset = (0, 8) if i % 2 == 0 else (0, -15) + plt.annotate(txt, (energy[i], codebleu[i]), textcoords="offset points", xytext=offset, ha='center', fontsize=10, fontweight='bold') +plt.title("OPT-2.7B Java Fixed-Exit Pareto Frontier", fontsize=14, pad=15) +plt.xlabel("Average Energy (mJ)", fontsize=12) +plt.ylabel("CodeBLEU Score", fontsize=12) +plt.grid(True, linestyle='--', alpha=0.7) +plt.plot(energy[-1], codebleu[-1], marker='*', color='gold', markersize=15, markeredgecolor='black') +plt.tight_layout() +out_path = os.path.expanduser('~/greencode/results/figures/opt_java_metrics_comparison_ctx_0.2.pdf') +plt.savefig(out_path, dpi=300) +print(f"Plot saved successfully to: {out_path}") diff --git a/new_results/replication/opt_java/eval_all_fixed_exits.json b/new_results/replication/opt_java/eval_all_fixed_exits.json new file mode 100644 index 0000000..d9aebb2 --- /dev/null +++ b/new_results/replication/opt_java/eval_all_fixed_exits.json @@ -0,0 +1,530 @@ +[ + { + "exit_index": 3, + "codebleu": { + "codebleu": 0.3085419550981881, + "ngram_match_score": 0.19067456838371866, + "weighted_ngram_match_score": 0.22549509718509428, + "syntax_match_score": 0.35543130990415334, + "dataflow_match_score": 0.4625668449197861 + }, + "chrf": { + "score": 26.346815923235134, + "char_order": 6, + "word_order": 0, + "beta": 2 + }, + "rouge": { + "rouge1": 0.23502866734167188, + "rouge2": 0.12437429066666715, + "rougeL": 0.23316514868917615, + "rougeLsum": 0.23375735943570625 + }, + "bleu": { + "bleu": 0.18841011747985517, + "precisions": [ + 0.408708999838423, + 0.20435967302452315, + 0.1417558061096656, + 0.10643062813266503 + ], + "brevity_penalty": 1.0, + "length_ratio": 1.1674054512873715, + "translation_length": 12378, + "reference_length": 10603 + }, + "rest": { + "exact_match": 0.003996003996003996, + "lev_dist": 38.09090909090909, + "mean_time": 0.0835209745508093, + "mean_energy": 12.131913788013586, + "std_time": 0.014508562537882033, + "std_energy": 2.393643656542935, + "total_energy": 12144.045701801599, + "total_time": 83.60449552536011, + "std_through": 22.600805926068823, + "mean_through": 183.3797429552793 + }, + "eval_time_s": 97.11258339881897 + }, + { + "exit_index": 5, + "codebleu": { + "codebleu": 0.3116000451857528, + "ngram_match_score": 0.19583596908350961, + "weighted_ngram_match_score": 0.2322465680176707, + "syntax_match_score": 0.35575079872204474, + "dataflow_match_score": 0.4625668449197861 + }, + "chrf": { + "score": 27.509499364602476, + "char_order": 6, + "word_order": 0, + "beta": 2 + }, + "rouge": { + "rouge1": 0.24760287562334615, + "rouge2": 0.1287703806703771, + "rougeL": 0.24618569927591427, + "rougeLsum": 0.24666538936015608 + }, + "bleu": { + "bleu": 0.19200024717807793, + "precisions": [ + 0.41269202927692433, + 0.20932470258922323, + 0.14560966257668712, + 0.10803647158608991 + ], + "brevity_penalty": 1.0, + "length_ratio": 1.1725926624540224, + "translation_length": 12433, + "reference_length": 10603 + }, + "rest": { + "exact_match": 0.001998001998001998, + "lev_dist": 37.7022977022977, + "mean_time": 0.11263286245690955, + "mean_energy": 14.654149850147528, + "std_time": 0.01926744220321838, + "std_energy": 5.193031602609825, + "total_energy": 14668.803999997675, + "total_time": 112.74549531936646, + "std_through": 16.709589051719462, + "mean_through": 135.96795867233908 + }, + "eval_time_s": 123.29629802703857 + }, + { + "exit_index": 7, + "codebleu": { + "codebleu": 0.31729770479159014, + "ngram_match_score": 0.21048480986273896, + "weighted_ngram_match_score": 0.24257524099281183, + "syntax_match_score": 0.3669329073482428, + "dataflow_match_score": 0.44919786096256686 + }, + "chrf": { + "score": 27.994852690575495, + "char_order": 6, + "word_order": 0, + "beta": 2 + }, + "rouge": { + "rouge1": 0.2526891860010755, + "rouge2": 0.13690362274116008, + "rougeL": 0.25116825853929314, + "rougeLsum": 0.25160714377163584 + }, + "bleu": { + "bleu": 0.2074365721453364, + "precisions": [ + 0.42798115858193536, + 0.2245945945945946, + 0.15939015939015938, + 0.12085255987695012 + ], + "brevity_penalty": 1.0, + "length_ratio": 1.1412807695935112, + "translation_length": 12101, + "reference_length": 10603 + }, + "rest": { + "exact_match": 0.003996003996003996, + "lev_dist": 36.77622377622377, + "mean_time": 0.1412087634845928, + "mean_energy": 19.32034965033655, + "std_time": 0.024858495063354007, + "std_energy": 7.952233459576671, + "total_energy": 19339.669999986887, + "total_time": 141.3499722480774, + "std_through": 13.564945231147227, + "mean_through": 108.57195790893184 + }, + "eval_time_s": 151.60858178138733 + }, + { + "exit_index": 9, + "codebleu": { + "codebleu": 0.3299713858015275, + "ngram_match_score": 0.2140210547595856, + "weighted_ngram_match_score": 0.2500299734032142, + "syntax_match_score": 0.3611821086261981, + "dataflow_match_score": 0.4946524064171123 + }, + "chrf": { + "score": 28.69955699307067, + "char_order": 6, + "word_order": 0, + "beta": 2 + }, + "rouge": { + "rouge1": 0.2584162173104967, + "rouge2": 0.14148841902303644, + "rougeL": 0.2568476311374282, + "rougeLsum": 0.2574396627317782 + }, + "bleu": { + "bleu": 0.21199501532394802, + "precisions": [ + 0.4338598223099704, + 0.22904527117884357, + 0.16336779911373708, + 0.12441288913162206 + ], + "brevity_penalty": 1.0, + "length_ratio": 1.1464679807601623, + "translation_length": 12156, + "reference_length": 10603 + }, + "rest": { + "exact_match": 0.005994005994005994, + "lev_dist": 36.64435564435564, + "mean_time": 0.1701601042256846, + "mean_energy": 24.00939460540866, + "std_time": 0.030554575006484837, + "std_energy": 8.554045285677534, + "total_energy": 24033.404000014067, + "total_time": 170.33026432991028, + "std_through": 11.413561760382295, + "mean_through": 90.1649261119857 + }, + "eval_time_s": 180.87501645088196 + }, + { + "exit_index": 11, + "codebleu": { + "codebleu": 0.33120149846093827, + "ngram_match_score": 0.20701923639901434, + "weighted_ngram_match_score": 0.24382509610288555, + "syntax_match_score": 0.37396166134185305, + "dataflow_match_score": 0.5 + }, + "chrf": { + "score": 28.82049173604128, + "char_order": 6, + "word_order": 0, + "beta": 2 + }, + "rouge": { + "rouge1": 0.25585947109616974, + "rouge2": 0.13293999263030254, + "rougeL": 0.2542704145066956, + "rougeLsum": 0.2547394210624174 + }, + "bleu": { + "bleu": 0.20618886027628378, + "precisions": [ + 0.43295929232533376, + 0.22376873661670235, + 0.1565592240619183, + 0.11916141646752118 + ], + "brevity_penalty": 1.0, + "length_ratio": 1.1514665660662078, + "translation_length": 12209, + "reference_length": 10603 + }, + "rest": { + "exact_match": 0.003996003996003996, + "lev_dist": 36.396603396603396, + "mean_time": 0.19899982481926948, + "mean_energy": 27.99878421578404, + "std_time": 0.036461068208114095, + "std_energy": 8.586884540226105, + "total_energy": 28026.78299999982, + "total_time": 199.19882464408875, + "std_through": 9.934190896429508, + "mean_through": 77.16365839145136 + }, + "eval_time_s": 209.05940532684326 + }, + { + "exit_index": 13, + "codebleu": { + "codebleu": 0.33221771923670806, + "ngram_match_score": 0.2104466754551522, + "weighted_ngram_match_score": 0.24788551259178088, + "syntax_match_score": 0.3865814696485623, + "dataflow_match_score": 0.4839572192513369 + }, + "chrf": { + "score": 29.288377230375783, + "char_order": 6, + "word_order": 0, + "beta": 2 + }, + "rouge": { + "rouge1": 0.2650741303270669, + "rouge2": 0.13803171936288838, + "rougeL": 0.26144219694427273, + "rougeLsum": 0.2616926126452208 + }, + "bleu": { + "bleu": 0.21055753937088106, + "precisions": [ + 0.4401471790678659, + 0.23109804969275982, + 0.16041055718475072, + 0.12046365507528979 + ], + "brevity_penalty": 1.0, + "length_ratio": 1.1534471376025652, + "translation_length": 12230, + "reference_length": 10603 + }, + "rest": { + "exact_match": 0.003996003996003996, + "lev_dist": 35.75924075924076, + "mean_time": 0.22799557167571502, + "mean_energy": 32.54528171828874, + "std_time": 0.04219700284697554, + "std_energy": 9.127248044267345, + "total_energy": 32577.827000007033, + "total_time": 228.22356724739075, + "std_through": 8.73251354572754, + "mean_through": 67.37692714441317 + }, + "eval_time_s": 238.83869338035583 + }, + { + "exit_index": 15, + "codebleu": { + "codebleu": 0.32981240935779493, + "ngram_match_score": 0.21022796177535755, + "weighted_ngram_match_score": 0.24786604872308576, + "syntax_match_score": 0.3798722044728435, + "dataflow_match_score": 0.48128342245989303 + }, + "chrf": { + "score": 29.37609781524177, + "char_order": 6, + "word_order": 0, + "beta": 2 + }, + "rouge": { + "rouge1": 0.2663668773809412, + "rouge2": 0.13770367161975577, + "rougeL": 0.26332676304480884, + "rougeLsum": 0.2643884092531812 + }, + "bleu": { + "bleu": 0.2094901792722669, + "precisions": [ + 0.438232421875, + 0.22831576149552582, + 0.15950622083981336, + 0.12068037463666703 + ], + "brevity_penalty": 1.0, + "length_ratio": 1.1589172875601246, + "translation_length": 12288, + "reference_length": 10603 + }, + "rest": { + "exact_match": 0.005994005994005994, + "lev_dist": 35.54645354645355, + "mean_time": 0.2570528831634369, + "mean_energy": 37.07245754245129, + "std_time": 0.04800471696727656, + "std_energy": 10.962740650949112, + "total_energy": 37109.52999999374, + "total_time": 257.30993604660034, + "std_through": 7.813241217013678, + "mean_through": 59.78180284233567 + }, + "eval_time_s": 267.66468238830566 + }, + { + "exit_index": 19, + "codebleu": { + "codebleu": 0.33292683979730503, + "ngram_match_score": 0.2050238350348228, + "weighted_ngram_match_score": 0.23658767750902993, + "syntax_match_score": 0.3900958466453674, + "dataflow_match_score": 0.5 + }, + "chrf": { + "score": 29.928778613235725, + "char_order": 6, + "word_order": 0, + "beta": 2 + }, + "rouge": { + "rouge1": 0.258132344421675, + "rouge2": 0.12731499848867794, + "rougeL": 0.25625157353050265, + "rougeLsum": 0.2562569533657273 + }, + "bleu": { + "bleu": 0.20768210905216453, + "precisions": [ + 0.4447119375726382, + 0.22634676324128564, + 0.15568385427035636, + 0.11871338565270255 + ], + "brevity_penalty": 1.0, + "length_ratio": 1.1360935584268603, + "translation_length": 12046, + "reference_length": 10603 + }, + "rest": { + "exact_match": 0.011988011988011988, + "lev_dist": 35.75924075924076, + "mean_time": 0.3145247920528873, + "mean_energy": 45.84630869130244, + "std_time": 0.0602302781392137, + "std_energy": 12.234466055263793, + "total_energy": 45892.15499999374, + "total_time": 314.8393168449402, + "std_through": 6.5245759425711265, + "mean_through": 48.786965227281975 + }, + "eval_time_s": 326.15760707855225 + }, + { + "exit_index": 23, + "codebleu": { + "codebleu": 0.3463692551603624, + "ngram_match_score": 0.22670813177777038, + "weighted_ngram_match_score": 0.25655894883190106, + "syntax_match_score": 0.420926517571885, + "dataflow_match_score": 0.48128342245989303 + }, + "chrf": { + "score": 32.89891416039852, + "char_order": 6, + "word_order": 0, + "beta": 2 + }, + "rouge": { + "rouge1": 0.2901406447162942, + "rouge2": 0.14277519898369662, + "rougeL": 0.2873019930085162, + "rougeLsum": 0.287600276754196 + }, + "bleu": { + "bleu": 0.2306103978206576, + "precisions": [ + 0.47367086467754205, + 0.2528162511542013, + 0.17550106826737205, + 0.13457181694608064 + ], + "brevity_penalty": 1.0, + "length_ratio": 1.1158162784117702, + "translation_length": 11831, + "reference_length": 10603 + }, + "rest": { + "exact_match": 0.013986013986013986, + "lev_dist": 34.19180819180819, + "mean_time": 0.3732499042591015, + "mean_energy": 55.00681618381511, + "std_time": 0.07158166555203227, + "std_energy": 14.931885141495968, + "total_energy": 55061.82299999893, + "total_time": 373.6231541633606, + "std_through": 5.527807783240863, + "mean_through": 41.218827695156804 + }, + "eval_time_s": 384.3628373146057 + }, + { + "exit_index": 27, + "codebleu": { + "codebleu": 0.3627157562055855, + "ngram_match_score": 0.2464942151229068, + "weighted_ngram_match_score": 0.2809624096720993, + "syntax_match_score": 0.42875399361022365, + "dataflow_match_score": 0.4946524064171123 + }, + "chrf": { + "score": 34.76999760879607, + "char_order": 6, + "word_order": 0, + "beta": 2 + }, + "rouge": { + "rouge1": 0.3107995751636172, + "rouge2": 0.1661677227332164, + "rougeL": 0.30788978023979907, + "rougeLsum": 0.308413718917109 + }, + "bleu": { + "bleu": 0.24907409850058812, + "precisions": [ + 0.4838899144151703, + 0.27168636072181, + 0.19433239209358613, + 0.15064498037016263 + ], + "brevity_penalty": 1.0, + "length_ratio": 1.1240215033481091, + "translation_length": 11918, + "reference_length": 10603 + }, + "rest": { + "exact_match": 0.016983016983016984, + "lev_dist": 33.38061938061938, + "mean_time": 0.43135855390832617, + "mean_energy": 64.00646053945327, + "std_time": 0.08339636601453994, + "std_energy": 16.417450825348574, + "total_energy": 64070.46699999273, + "total_time": 431.7899124622345, + "std_through": 4.823876445521664, + "mean_through": 35.67643863236937 + }, + "eval_time_s": 442.76964235305786 + }, + { + "exit_index": 31, + "codebleu": { + "codebleu": 0.3991263642967176, + "ngram_match_score": 0.27604689925592335, + "weighted_ngram_match_score": 0.31123951161361074, + "syntax_match_score": 0.4450479233226837, + "dataflow_match_score": 0.5641711229946524 + }, + "chrf": { + "score": 36.88821883531422, + "char_order": 6, + "word_order": 0, + "beta": 2 + }, + "rouge": { + "rouge1": 0.33839891481634476, + "rouge2": 0.19533754485053206, + "rougeL": 0.3353110278328256, + "rougeLsum": 0.3356657620541966 + }, + "bleu": { + "bleu": 0.2783191796452495, + "precisions": [ + 0.49826843483402317, + 0.29885587746816755, + 0.22364542035173327, + 0.18017202354006337 + ], + "brevity_penalty": 1.0, + "length_ratio": 1.1165707818541921, + "translation_length": 11839, + "reference_length": 10603 + }, + "rest": { + "exact_match": 0.028971028971028972, + "lev_dist": 32.09390609390609, + "mean_time": 0.48922354739148183, + "mean_energy": 71.8862967032876, + "std_time": 0.09483094935342311, + "std_energy": 19.764103419251658, + "total_energy": 71958.18299999088, + "total_time": 489.7127709388733, + "std_through": 4.268431025505318, + "mean_through": 31.447385129456674 + }, + "eval_time_s": 502.876544713974 + } +] \ No newline at end of file diff --git a/new_results/replication/opt_java/eval_exit_11.json b/new_results/replication/opt_java/eval_exit_11.json new file mode 100644 index 0000000..fed5cd0 --- /dev/null +++ b/new_results/replication/opt_java/eval_exit_11.json @@ -0,0 +1,48 @@ +{ + "exit_index": 11, + "codebleu": { + "codebleu": 0.33120149846093827, + "ngram_match_score": 0.20701923639901434, + "weighted_ngram_match_score": 0.24382509610288555, + "syntax_match_score": 0.37396166134185305, + "dataflow_match_score": 0.5 + }, + "chrf": { + "score": 28.82049173604128, + "char_order": 6, + "word_order": 0, + "beta": 2 + }, + "rouge": { + "rouge1": 0.25585947109616974, + "rouge2": 0.13293999263030254, + "rougeL": 0.2542704145066956, + "rougeLsum": 0.2547394210624174 + }, + "bleu": { + "bleu": 0.20618886027628378, + "precisions": [ + 0.43295929232533376, + 0.22376873661670235, + 0.1565592240619183, + 0.11916141646752118 + ], + "brevity_penalty": 1.0, + "length_ratio": 1.1514665660662078, + "translation_length": 12209, + "reference_length": 10603 + }, + "rest": { + "exact_match": 0.003996003996003996, + "lev_dist": 36.396603396603396, + "mean_time": 0.19899982481926948, + "mean_energy": 27.99878421578404, + "std_time": 0.036461068208114095, + "std_energy": 8.586884540226105, + "total_energy": 28026.78299999982, + "total_time": 199.19882464408875, + "std_through": 9.934190896429508, + "mean_through": 77.16365839145136 + }, + "eval_time_s": 209.05940532684326 +} \ No newline at end of file diff --git a/new_results/replication/opt_java/eval_exit_13.json b/new_results/replication/opt_java/eval_exit_13.json new file mode 100644 index 0000000..f7d30cb --- /dev/null +++ b/new_results/replication/opt_java/eval_exit_13.json @@ -0,0 +1,48 @@ +{ + "exit_index": 13, + "codebleu": { + "codebleu": 0.33221771923670806, + "ngram_match_score": 0.2104466754551522, + "weighted_ngram_match_score": 0.24788551259178088, + "syntax_match_score": 0.3865814696485623, + "dataflow_match_score": 0.4839572192513369 + }, + "chrf": { + "score": 29.288377230375783, + "char_order": 6, + "word_order": 0, + "beta": 2 + }, + "rouge": { + "rouge1": 0.2650741303270669, + "rouge2": 0.13803171936288838, + "rougeL": 0.26144219694427273, + "rougeLsum": 0.2616926126452208 + }, + "bleu": { + "bleu": 0.21055753937088106, + "precisions": [ + 0.4401471790678659, + 0.23109804969275982, + 0.16041055718475072, + 0.12046365507528979 + ], + "brevity_penalty": 1.0, + "length_ratio": 1.1534471376025652, + "translation_length": 12230, + "reference_length": 10603 + }, + "rest": { + "exact_match": 0.003996003996003996, + "lev_dist": 35.75924075924076, + "mean_time": 0.22799557167571502, + "mean_energy": 32.54528171828874, + "std_time": 0.04219700284697554, + "std_energy": 9.127248044267345, + "total_energy": 32577.827000007033, + "total_time": 228.22356724739075, + "std_through": 8.73251354572754, + "mean_through": 67.37692714441317 + }, + "eval_time_s": 238.83869338035583 +} \ No newline at end of file diff --git a/new_results/replication/opt_java/eval_exit_15.json b/new_results/replication/opt_java/eval_exit_15.json new file mode 100644 index 0000000..18177aa --- /dev/null +++ b/new_results/replication/opt_java/eval_exit_15.json @@ -0,0 +1,48 @@ +{ + "exit_index": 15, + "codebleu": { + "codebleu": 0.32981240935779493, + "ngram_match_score": 0.21022796177535755, + "weighted_ngram_match_score": 0.24786604872308576, + "syntax_match_score": 0.3798722044728435, + "dataflow_match_score": 0.48128342245989303 + }, + "chrf": { + "score": 29.37609781524177, + "char_order": 6, + "word_order": 0, + "beta": 2 + }, + "rouge": { + "rouge1": 0.2663668773809412, + "rouge2": 0.13770367161975577, + "rougeL": 0.26332676304480884, + "rougeLsum": 0.2643884092531812 + }, + "bleu": { + "bleu": 0.2094901792722669, + "precisions": [ + 0.438232421875, + 0.22831576149552582, + 0.15950622083981336, + 0.12068037463666703 + ], + "brevity_penalty": 1.0, + "length_ratio": 1.1589172875601246, + "translation_length": 12288, + "reference_length": 10603 + }, + "rest": { + "exact_match": 0.005994005994005994, + "lev_dist": 35.54645354645355, + "mean_time": 0.2570528831634369, + "mean_energy": 37.07245754245129, + "std_time": 0.04800471696727656, + "std_energy": 10.962740650949112, + "total_energy": 37109.52999999374, + "total_time": 257.30993604660034, + "std_through": 7.813241217013678, + "mean_through": 59.78180284233567 + }, + "eval_time_s": 267.66468238830566 +} \ No newline at end of file diff --git a/new_results/replication/opt_java/eval_exit_19.json b/new_results/replication/opt_java/eval_exit_19.json new file mode 100644 index 0000000..769f33f --- /dev/null +++ b/new_results/replication/opt_java/eval_exit_19.json @@ -0,0 +1,48 @@ +{ + "exit_index": 19, + "codebleu": { + "codebleu": 0.33292683979730503, + "ngram_match_score": 0.2050238350348228, + "weighted_ngram_match_score": 0.23658767750902993, + "syntax_match_score": 0.3900958466453674, + "dataflow_match_score": 0.5 + }, + "chrf": { + "score": 29.928778613235725, + "char_order": 6, + "word_order": 0, + "beta": 2 + }, + "rouge": { + "rouge1": 0.258132344421675, + "rouge2": 0.12731499848867794, + "rougeL": 0.25625157353050265, + "rougeLsum": 0.2562569533657273 + }, + "bleu": { + "bleu": 0.20768210905216453, + "precisions": [ + 0.4447119375726382, + 0.22634676324128564, + 0.15568385427035636, + 0.11871338565270255 + ], + "brevity_penalty": 1.0, + "length_ratio": 1.1360935584268603, + "translation_length": 12046, + "reference_length": 10603 + }, + "rest": { + "exact_match": 0.011988011988011988, + "lev_dist": 35.75924075924076, + "mean_time": 0.3145247920528873, + "mean_energy": 45.84630869130244, + "std_time": 0.0602302781392137, + "std_energy": 12.234466055263793, + "total_energy": 45892.15499999374, + "total_time": 314.8393168449402, + "std_through": 6.5245759425711265, + "mean_through": 48.786965227281975 + }, + "eval_time_s": 326.15760707855225 +} \ No newline at end of file diff --git a/new_results/replication/opt_java/eval_exit_23.json b/new_results/replication/opt_java/eval_exit_23.json new file mode 100644 index 0000000..f0f6e92 --- /dev/null +++ b/new_results/replication/opt_java/eval_exit_23.json @@ -0,0 +1,48 @@ +{ + "exit_index": 23, + "codebleu": { + "codebleu": 0.3463692551603624, + "ngram_match_score": 0.22670813177777038, + "weighted_ngram_match_score": 0.25655894883190106, + "syntax_match_score": 0.420926517571885, + "dataflow_match_score": 0.48128342245989303 + }, + "chrf": { + "score": 32.89891416039852, + "char_order": 6, + "word_order": 0, + "beta": 2 + }, + "rouge": { + "rouge1": 0.2901406447162942, + "rouge2": 0.14277519898369662, + "rougeL": 0.2873019930085162, + "rougeLsum": 0.287600276754196 + }, + "bleu": { + "bleu": 0.2306103978206576, + "precisions": [ + 0.47367086467754205, + 0.2528162511542013, + 0.17550106826737205, + 0.13457181694608064 + ], + "brevity_penalty": 1.0, + "length_ratio": 1.1158162784117702, + "translation_length": 11831, + "reference_length": 10603 + }, + "rest": { + "exact_match": 0.013986013986013986, + "lev_dist": 34.19180819180819, + "mean_time": 0.3732499042591015, + "mean_energy": 55.00681618381511, + "std_time": 0.07158166555203227, + "std_energy": 14.931885141495968, + "total_energy": 55061.82299999893, + "total_time": 373.6231541633606, + "std_through": 5.527807783240863, + "mean_through": 41.218827695156804 + }, + "eval_time_s": 384.3628373146057 +} \ No newline at end of file diff --git a/new_results/replication/opt_java/eval_exit_27.json b/new_results/replication/opt_java/eval_exit_27.json new file mode 100644 index 0000000..3b429b6 --- /dev/null +++ b/new_results/replication/opt_java/eval_exit_27.json @@ -0,0 +1,48 @@ +{ + "exit_index": 27, + "codebleu": { + "codebleu": 0.3627157562055855, + "ngram_match_score": 0.2464942151229068, + "weighted_ngram_match_score": 0.2809624096720993, + "syntax_match_score": 0.42875399361022365, + "dataflow_match_score": 0.4946524064171123 + }, + "chrf": { + "score": 34.76999760879607, + "char_order": 6, + "word_order": 0, + "beta": 2 + }, + "rouge": { + "rouge1": 0.3107995751636172, + "rouge2": 0.1661677227332164, + "rougeL": 0.30788978023979907, + "rougeLsum": 0.308413718917109 + }, + "bleu": { + "bleu": 0.24907409850058812, + "precisions": [ + 0.4838899144151703, + 0.27168636072181, + 0.19433239209358613, + 0.15064498037016263 + ], + "brevity_penalty": 1.0, + "length_ratio": 1.1240215033481091, + "translation_length": 11918, + "reference_length": 10603 + }, + "rest": { + "exact_match": 0.016983016983016984, + "lev_dist": 33.38061938061938, + "mean_time": 0.43135855390832617, + "mean_energy": 64.00646053945327, + "std_time": 0.08339636601453994, + "std_energy": 16.417450825348574, + "total_energy": 64070.46699999273, + "total_time": 431.7899124622345, + "std_through": 4.823876445521664, + "mean_through": 35.67643863236937 + }, + "eval_time_s": 442.76964235305786 +} \ No newline at end of file diff --git a/new_results/replication/opt_java/eval_exit_3.json b/new_results/replication/opt_java/eval_exit_3.json new file mode 100644 index 0000000..0002375 --- /dev/null +++ b/new_results/replication/opt_java/eval_exit_3.json @@ -0,0 +1,48 @@ +{ + "exit_index": 3, + "codebleu": { + "codebleu": 0.3085419550981881, + "ngram_match_score": 0.19067456838371866, + "weighted_ngram_match_score": 0.22549509718509428, + "syntax_match_score": 0.35543130990415334, + "dataflow_match_score": 0.4625668449197861 + }, + "chrf": { + "score": 26.346815923235134, + "char_order": 6, + "word_order": 0, + "beta": 2 + }, + "rouge": { + "rouge1": 0.23502866734167188, + "rouge2": 0.12437429066666715, + "rougeL": 0.23316514868917615, + "rougeLsum": 0.23375735943570625 + }, + "bleu": { + "bleu": 0.18841011747985517, + "precisions": [ + 0.408708999838423, + 0.20435967302452315, + 0.1417558061096656, + 0.10643062813266503 + ], + "brevity_penalty": 1.0, + "length_ratio": 1.1674054512873715, + "translation_length": 12378, + "reference_length": 10603 + }, + "rest": { + "exact_match": 0.003996003996003996, + "lev_dist": 38.09090909090909, + "mean_time": 0.0835209745508093, + "mean_energy": 12.131913788013586, + "std_time": 0.014508562537882033, + "std_energy": 2.393643656542935, + "total_energy": 12144.045701801599, + "total_time": 83.60449552536011, + "std_through": 22.600805926068823, + "mean_through": 183.3797429552793 + }, + "eval_time_s": 97.11258339881897 +} \ No newline at end of file diff --git a/new_results/replication/opt_java/eval_exit_31.json b/new_results/replication/opt_java/eval_exit_31.json new file mode 100644 index 0000000..ceeec16 --- /dev/null +++ b/new_results/replication/opt_java/eval_exit_31.json @@ -0,0 +1,48 @@ +{ + "exit_index": 31, + "codebleu": { + "codebleu": 0.3991263642967176, + "ngram_match_score": 0.27604689925592335, + "weighted_ngram_match_score": 0.31123951161361074, + "syntax_match_score": 0.4450479233226837, + "dataflow_match_score": 0.5641711229946524 + }, + "chrf": { + "score": 36.88821883531422, + "char_order": 6, + "word_order": 0, + "beta": 2 + }, + "rouge": { + "rouge1": 0.33839891481634476, + "rouge2": 0.19533754485053206, + "rougeL": 0.3353110278328256, + "rougeLsum": 0.3356657620541966 + }, + "bleu": { + "bleu": 0.2783191796452495, + "precisions": [ + 0.49826843483402317, + 0.29885587746816755, + 0.22364542035173327, + 0.18017202354006337 + ], + "brevity_penalty": 1.0, + "length_ratio": 1.1165707818541921, + "translation_length": 11839, + "reference_length": 10603 + }, + "rest": { + "exact_match": 0.028971028971028972, + "lev_dist": 32.09390609390609, + "mean_time": 0.48922354739148183, + "mean_energy": 71.8862967032876, + "std_time": 0.09483094935342311, + "std_energy": 19.764103419251658, + "total_energy": 71958.18299999088, + "total_time": 489.7127709388733, + "std_through": 4.268431025505318, + "mean_through": 31.447385129456674 + }, + "eval_time_s": 502.876544713974 +} \ No newline at end of file diff --git a/new_results/replication/opt_java/eval_exit_5.json b/new_results/replication/opt_java/eval_exit_5.json new file mode 100644 index 0000000..595e54c --- /dev/null +++ b/new_results/replication/opt_java/eval_exit_5.json @@ -0,0 +1,48 @@ +{ + "exit_index": 5, + "codebleu": { + "codebleu": 0.3116000451857528, + "ngram_match_score": 0.19583596908350961, + "weighted_ngram_match_score": 0.2322465680176707, + "syntax_match_score": 0.35575079872204474, + "dataflow_match_score": 0.4625668449197861 + }, + "chrf": { + "score": 27.509499364602476, + "char_order": 6, + "word_order": 0, + "beta": 2 + }, + "rouge": { + "rouge1": 0.24760287562334615, + "rouge2": 0.1287703806703771, + "rougeL": 0.24618569927591427, + "rougeLsum": 0.24666538936015608 + }, + "bleu": { + "bleu": 0.19200024717807793, + "precisions": [ + 0.41269202927692433, + 0.20932470258922323, + 0.14560966257668712, + 0.10803647158608991 + ], + "brevity_penalty": 1.0, + "length_ratio": 1.1725926624540224, + "translation_length": 12433, + "reference_length": 10603 + }, + "rest": { + "exact_match": 0.001998001998001998, + "lev_dist": 37.7022977022977, + "mean_time": 0.11263286245690955, + "mean_energy": 14.654149850147528, + "std_time": 0.01926744220321838, + "std_energy": 5.193031602609825, + "total_energy": 14668.803999997675, + "total_time": 112.74549531936646, + "std_through": 16.709589051719462, + "mean_through": 135.96795867233908 + }, + "eval_time_s": 123.29629802703857 +} \ No newline at end of file diff --git a/new_results/replication/opt_java/eval_exit_7.json b/new_results/replication/opt_java/eval_exit_7.json new file mode 100644 index 0000000..da8fbd6 --- /dev/null +++ b/new_results/replication/opt_java/eval_exit_7.json @@ -0,0 +1,48 @@ +{ + "exit_index": 7, + "codebleu": { + "codebleu": 0.31729770479159014, + "ngram_match_score": 0.21048480986273896, + "weighted_ngram_match_score": 0.24257524099281183, + "syntax_match_score": 0.3669329073482428, + "dataflow_match_score": 0.44919786096256686 + }, + "chrf": { + "score": 27.994852690575495, + "char_order": 6, + "word_order": 0, + "beta": 2 + }, + "rouge": { + "rouge1": 0.2526891860010755, + "rouge2": 0.13690362274116008, + "rougeL": 0.25116825853929314, + "rougeLsum": 0.25160714377163584 + }, + "bleu": { + "bleu": 0.2074365721453364, + "precisions": [ + 0.42798115858193536, + 0.2245945945945946, + 0.15939015939015938, + 0.12085255987695012 + ], + "brevity_penalty": 1.0, + "length_ratio": 1.1412807695935112, + "translation_length": 12101, + "reference_length": 10603 + }, + "rest": { + "exact_match": 0.003996003996003996, + "lev_dist": 36.77622377622377, + "mean_time": 0.1412087634845928, + "mean_energy": 19.32034965033655, + "std_time": 0.024858495063354007, + "std_energy": 7.952233459576671, + "total_energy": 19339.669999986887, + "total_time": 141.3499722480774, + "std_through": 13.564945231147227, + "mean_through": 108.57195790893184 + }, + "eval_time_s": 151.60858178138733 +} \ No newline at end of file diff --git a/new_results/replication/opt_java/eval_exit_9.json b/new_results/replication/opt_java/eval_exit_9.json new file mode 100644 index 0000000..f286bfe --- /dev/null +++ b/new_results/replication/opt_java/eval_exit_9.json @@ -0,0 +1,48 @@ +{ + "exit_index": 9, + "codebleu": { + "codebleu": 0.3299713858015275, + "ngram_match_score": 0.2140210547595856, + "weighted_ngram_match_score": 0.2500299734032142, + "syntax_match_score": 0.3611821086261981, + "dataflow_match_score": 0.4946524064171123 + }, + "chrf": { + "score": 28.69955699307067, + "char_order": 6, + "word_order": 0, + "beta": 2 + }, + "rouge": { + "rouge1": 0.2584162173104967, + "rouge2": 0.14148841902303644, + "rougeL": 0.2568476311374282, + "rougeLsum": 0.2574396627317782 + }, + "bleu": { + "bleu": 0.21199501532394802, + "precisions": [ + 0.4338598223099704, + 0.22904527117884357, + 0.16336779911373708, + 0.12441288913162206 + ], + "brevity_penalty": 1.0, + "length_ratio": 1.1464679807601623, + "translation_length": 12156, + "reference_length": 10603 + }, + "rest": { + "exact_match": 0.005994005994005994, + "lev_dist": 36.64435564435564, + "mean_time": 0.1701601042256846, + "mean_energy": 24.00939460540866, + "std_time": 0.030554575006484837, + "std_energy": 8.554045285677534, + "total_energy": 24033.404000014067, + "total_time": 170.33026432991028, + "std_through": 11.413561760382295, + "mean_through": 90.1649261119857 + }, + "eval_time_s": 180.87501645088196 +} \ No newline at end of file diff --git a/new_results/replication/opt_java/phase1_log.txt b/new_results/replication/opt_java/phase1_log.txt new file mode 100644 index 0000000..7aec1d7 --- /dev/null +++ b/new_results/replication/opt_java/phase1_log.txt @@ -0,0 +1,150220 @@ +============================================================ +Phase 1: Fixed-Exit Evaluation — OPT-2.7B x JavaCorpus +Exit layers: [3, 5, 7, 9, 11, 13, 15, 19, 23, 27, 31] +============================================================ +Dataset: 8268 test samples + +--- Evaluating exit_index = 3 --- + Loading checkpoint shards: 0%| | 0/3 [00:00 + codebleu, chrf, rouge, bleu, rest = evaluator.evaluate( + ^^^^^^^^^^^^^^^^^^^ + File "/home/mluser/greencode/repo/src/eval/evaluator.py", line 84, in evaluate + output = self.model.generate(input_ids=input_ids, attention_mask=attention_mask, + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/mluser/greencode/venv/lib/python3.12/site-packages/torch/utils/_contextlib.py", line 116, in decorate_context + return func(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^ + File "/home/mluser/greencode/venv/lib/python3.12/site-packages/transformers/generation/utils.py", line 2047, in generate + result = self._sample( + ^^^^^^^^^^^^^ + File "/home/mluser/greencode/venv/lib/python3.12/site-packages/transformers/generation/utils.py", line 3007, in _sample + outputs = self(**model_inputs, return_dict=True) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/mluser/greencode/venv/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1553, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/mluser/greencode/venv/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1562, in _call_impl + return forward_call(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/mluser/greencode/repo/src/models/opt/modeling_opt_single_head_fixed_exit.py", line 576, in forward + raise ValueError(f"mode {self.mode} not supported") +ValueError: mode infer_ee not supported diff --git a/results/figures/opt_java_metrics_comparison_ctx_0.2.pdf b/results/figures/opt_java_metrics_comparison_ctx_0.2.pdf new file mode 100644 index 0000000..806a3ba Binary files /dev/null and b/results/figures/opt_java_metrics_comparison_ctx_0.2.pdf differ diff --git a/results/figures/plot_pareto.py b/results/figures/plot_pareto.py new file mode 100644 index 0000000..77cdbeb --- /dev/null +++ b/results/figures/plot_pareto.py @@ -0,0 +1,19 @@ +import matplotlib.pyplot as plt +import os +energy = [12.13, 14.65, 19.32, 24.01, 28.00, 32.55, 37.07, 45.85, 55.01, 64.01, 71.89] +codebleu = [0.3085, 0.3116, 0.3173, 0.3300, 0.3312, 0.3322, 0.3298, 0.3329, 0.3464, 0.3627, 0.3991] +labels = ["L3", "L5", "L7", "L9", "L11", "L13", "L15", "L19", "L23", "L27", "L31"] +plt.figure(figsize=(9, 6)) +plt.plot(energy, codebleu, marker='o', linestyle='-', linewidth=2, markersize=8, color='#1f77b4') +for i, txt in enumerate(labels): + offset = (0, 8) if i % 2 == 0 else (0, -15) + plt.annotate(txt, (energy[i], codebleu[i]), textcoords="offset points", xytext=offset, ha='center', fontsize=10, fontweight='bold') +plt.title("OPT-2.7B Java Fixed-Exit Pareto Frontier", fontsize=14, pad=15) +plt.xlabel("Average Energy (mJ)", fontsize=12) +plt.ylabel("CodeBLEU Score", fontsize=12) +plt.grid(True, linestyle='--', alpha=0.7) +plt.plot(energy[-1], codebleu[-1], marker='*', color='gold', markersize=15, markeredgecolor='black') +plt.tight_layout() +out_path = os.path.expanduser('~/greencode/results/figures/opt_java_metrics_comparison_ctx_0.2.pdf') +plt.savefig(out_path, dpi=300) +print(f"Plot saved successfully to: {out_path}") diff --git a/results/replication/opt_java/eval_all_fixed_exits.json b/results/replication/opt_java/eval_all_fixed_exits.json new file mode 100644 index 0000000..d9aebb2 --- /dev/null +++ b/results/replication/opt_java/eval_all_fixed_exits.json @@ -0,0 +1,530 @@ +[ + { + "exit_index": 3, + "codebleu": { + "codebleu": 0.3085419550981881, + "ngram_match_score": 0.19067456838371866, + "weighted_ngram_match_score": 0.22549509718509428, + "syntax_match_score": 0.35543130990415334, + "dataflow_match_score": 0.4625668449197861 + }, + "chrf": { + "score": 26.346815923235134, + "char_order": 6, + "word_order": 0, + "beta": 2 + }, + "rouge": { + "rouge1": 0.23502866734167188, + "rouge2": 0.12437429066666715, + "rougeL": 0.23316514868917615, + "rougeLsum": 0.23375735943570625 + }, + "bleu": { + "bleu": 0.18841011747985517, + "precisions": [ + 0.408708999838423, + 0.20435967302452315, + 0.1417558061096656, + 0.10643062813266503 + ], + "brevity_penalty": 1.0, + "length_ratio": 1.1674054512873715, + "translation_length": 12378, + "reference_length": 10603 + }, + "rest": { + "exact_match": 0.003996003996003996, + "lev_dist": 38.09090909090909, + "mean_time": 0.0835209745508093, + "mean_energy": 12.131913788013586, + "std_time": 0.014508562537882033, + "std_energy": 2.393643656542935, + "total_energy": 12144.045701801599, + "total_time": 83.60449552536011, + "std_through": 22.600805926068823, + "mean_through": 183.3797429552793 + }, + "eval_time_s": 97.11258339881897 + }, + { + "exit_index": 5, + "codebleu": { + "codebleu": 0.3116000451857528, + "ngram_match_score": 0.19583596908350961, + "weighted_ngram_match_score": 0.2322465680176707, + "syntax_match_score": 0.35575079872204474, + "dataflow_match_score": 0.4625668449197861 + }, + "chrf": { + "score": 27.509499364602476, + "char_order": 6, + "word_order": 0, + "beta": 2 + }, + "rouge": { + "rouge1": 0.24760287562334615, + "rouge2": 0.1287703806703771, + "rougeL": 0.24618569927591427, + "rougeLsum": 0.24666538936015608 + }, + "bleu": { + "bleu": 0.19200024717807793, + "precisions": [ + 0.41269202927692433, + 0.20932470258922323, + 0.14560966257668712, + 0.10803647158608991 + ], + "brevity_penalty": 1.0, + "length_ratio": 1.1725926624540224, + "translation_length": 12433, + "reference_length": 10603 + }, + "rest": { + "exact_match": 0.001998001998001998, + "lev_dist": 37.7022977022977, + "mean_time": 0.11263286245690955, + "mean_energy": 14.654149850147528, + "std_time": 0.01926744220321838, + "std_energy": 5.193031602609825, + "total_energy": 14668.803999997675, + "total_time": 112.74549531936646, + "std_through": 16.709589051719462, + "mean_through": 135.96795867233908 + }, + "eval_time_s": 123.29629802703857 + }, + { + "exit_index": 7, + "codebleu": { + "codebleu": 0.31729770479159014, + "ngram_match_score": 0.21048480986273896, + "weighted_ngram_match_score": 0.24257524099281183, + "syntax_match_score": 0.3669329073482428, + "dataflow_match_score": 0.44919786096256686 + }, + "chrf": { + "score": 27.994852690575495, + "char_order": 6, + "word_order": 0, + "beta": 2 + }, + "rouge": { + "rouge1": 0.2526891860010755, + "rouge2": 0.13690362274116008, + "rougeL": 0.25116825853929314, + "rougeLsum": 0.25160714377163584 + }, + "bleu": { + "bleu": 0.2074365721453364, + "precisions": [ + 0.42798115858193536, + 0.2245945945945946, + 0.15939015939015938, + 0.12085255987695012 + ], + "brevity_penalty": 1.0, + "length_ratio": 1.1412807695935112, + "translation_length": 12101, + "reference_length": 10603 + }, + "rest": { + "exact_match": 0.003996003996003996, + "lev_dist": 36.77622377622377, + "mean_time": 0.1412087634845928, + "mean_energy": 19.32034965033655, + "std_time": 0.024858495063354007, + "std_energy": 7.952233459576671, + "total_energy": 19339.669999986887, + "total_time": 141.3499722480774, + "std_through": 13.564945231147227, + "mean_through": 108.57195790893184 + }, + "eval_time_s": 151.60858178138733 + }, + { + "exit_index": 9, + "codebleu": { + "codebleu": 0.3299713858015275, + "ngram_match_score": 0.2140210547595856, + "weighted_ngram_match_score": 0.2500299734032142, + "syntax_match_score": 0.3611821086261981, + "dataflow_match_score": 0.4946524064171123 + }, + "chrf": { + "score": 28.69955699307067, + "char_order": 6, + "word_order": 0, + "beta": 2 + }, + "rouge": { + "rouge1": 0.2584162173104967, + "rouge2": 0.14148841902303644, + "rougeL": 0.2568476311374282, + "rougeLsum": 0.2574396627317782 + }, + "bleu": { + "bleu": 0.21199501532394802, + "precisions": [ + 0.4338598223099704, + 0.22904527117884357, + 0.16336779911373708, + 0.12441288913162206 + ], + "brevity_penalty": 1.0, + "length_ratio": 1.1464679807601623, + "translation_length": 12156, + "reference_length": 10603 + }, + "rest": { + "exact_match": 0.005994005994005994, + "lev_dist": 36.64435564435564, + "mean_time": 0.1701601042256846, + "mean_energy": 24.00939460540866, + "std_time": 0.030554575006484837, + "std_energy": 8.554045285677534, + "total_energy": 24033.404000014067, + "total_time": 170.33026432991028, + "std_through": 11.413561760382295, + "mean_through": 90.1649261119857 + }, + "eval_time_s": 180.87501645088196 + }, + { + "exit_index": 11, + "codebleu": { + "codebleu": 0.33120149846093827, + "ngram_match_score": 0.20701923639901434, + "weighted_ngram_match_score": 0.24382509610288555, + "syntax_match_score": 0.37396166134185305, + "dataflow_match_score": 0.5 + }, + "chrf": { + "score": 28.82049173604128, + "char_order": 6, + "word_order": 0, + "beta": 2 + }, + "rouge": { + "rouge1": 0.25585947109616974, + "rouge2": 0.13293999263030254, + "rougeL": 0.2542704145066956, + "rougeLsum": 0.2547394210624174 + }, + "bleu": { + "bleu": 0.20618886027628378, + "precisions": [ + 0.43295929232533376, + 0.22376873661670235, + 0.1565592240619183, + 0.11916141646752118 + ], + "brevity_penalty": 1.0, + "length_ratio": 1.1514665660662078, + "translation_length": 12209, + "reference_length": 10603 + }, + "rest": { + "exact_match": 0.003996003996003996, + "lev_dist": 36.396603396603396, + "mean_time": 0.19899982481926948, + "mean_energy": 27.99878421578404, + "std_time": 0.036461068208114095, + "std_energy": 8.586884540226105, + "total_energy": 28026.78299999982, + "total_time": 199.19882464408875, + "std_through": 9.934190896429508, + "mean_through": 77.16365839145136 + }, + "eval_time_s": 209.05940532684326 + }, + { + "exit_index": 13, + "codebleu": { + "codebleu": 0.33221771923670806, + "ngram_match_score": 0.2104466754551522, + "weighted_ngram_match_score": 0.24788551259178088, + "syntax_match_score": 0.3865814696485623, + "dataflow_match_score": 0.4839572192513369 + }, + "chrf": { + "score": 29.288377230375783, + "char_order": 6, + "word_order": 0, + "beta": 2 + }, + "rouge": { + "rouge1": 0.2650741303270669, + "rouge2": 0.13803171936288838, + "rougeL": 0.26144219694427273, + "rougeLsum": 0.2616926126452208 + }, + "bleu": { + "bleu": 0.21055753937088106, + "precisions": [ + 0.4401471790678659, + 0.23109804969275982, + 0.16041055718475072, + 0.12046365507528979 + ], + "brevity_penalty": 1.0, + "length_ratio": 1.1534471376025652, + "translation_length": 12230, + "reference_length": 10603 + }, + "rest": { + "exact_match": 0.003996003996003996, + "lev_dist": 35.75924075924076, + "mean_time": 0.22799557167571502, + "mean_energy": 32.54528171828874, + "std_time": 0.04219700284697554, + "std_energy": 9.127248044267345, + "total_energy": 32577.827000007033, + "total_time": 228.22356724739075, + "std_through": 8.73251354572754, + "mean_through": 67.37692714441317 + }, + "eval_time_s": 238.83869338035583 + }, + { + "exit_index": 15, + "codebleu": { + "codebleu": 0.32981240935779493, + "ngram_match_score": 0.21022796177535755, + "weighted_ngram_match_score": 0.24786604872308576, + "syntax_match_score": 0.3798722044728435, + "dataflow_match_score": 0.48128342245989303 + }, + "chrf": { + "score": 29.37609781524177, + "char_order": 6, + "word_order": 0, + "beta": 2 + }, + "rouge": { + "rouge1": 0.2663668773809412, + "rouge2": 0.13770367161975577, + "rougeL": 0.26332676304480884, + "rougeLsum": 0.2643884092531812 + }, + "bleu": { + "bleu": 0.2094901792722669, + "precisions": [ + 0.438232421875, + 0.22831576149552582, + 0.15950622083981336, + 0.12068037463666703 + ], + "brevity_penalty": 1.0, + "length_ratio": 1.1589172875601246, + "translation_length": 12288, + "reference_length": 10603 + }, + "rest": { + "exact_match": 0.005994005994005994, + "lev_dist": 35.54645354645355, + "mean_time": 0.2570528831634369, + "mean_energy": 37.07245754245129, + "std_time": 0.04800471696727656, + "std_energy": 10.962740650949112, + "total_energy": 37109.52999999374, + "total_time": 257.30993604660034, + "std_through": 7.813241217013678, + "mean_through": 59.78180284233567 + }, + "eval_time_s": 267.66468238830566 + }, + { + "exit_index": 19, + "codebleu": { + "codebleu": 0.33292683979730503, + "ngram_match_score": 0.2050238350348228, + "weighted_ngram_match_score": 0.23658767750902993, + "syntax_match_score": 0.3900958466453674, + "dataflow_match_score": 0.5 + }, + "chrf": { + "score": 29.928778613235725, + "char_order": 6, + "word_order": 0, + "beta": 2 + }, + "rouge": { + "rouge1": 0.258132344421675, + "rouge2": 0.12731499848867794, + "rougeL": 0.25625157353050265, + "rougeLsum": 0.2562569533657273 + }, + "bleu": { + "bleu": 0.20768210905216453, + "precisions": [ + 0.4447119375726382, + 0.22634676324128564, + 0.15568385427035636, + 0.11871338565270255 + ], + "brevity_penalty": 1.0, + "length_ratio": 1.1360935584268603, + "translation_length": 12046, + "reference_length": 10603 + }, + "rest": { + "exact_match": 0.011988011988011988, + "lev_dist": 35.75924075924076, + "mean_time": 0.3145247920528873, + "mean_energy": 45.84630869130244, + "std_time": 0.0602302781392137, + "std_energy": 12.234466055263793, + "total_energy": 45892.15499999374, + "total_time": 314.8393168449402, + "std_through": 6.5245759425711265, + "mean_through": 48.786965227281975 + }, + "eval_time_s": 326.15760707855225 + }, + { + "exit_index": 23, + "codebleu": { + "codebleu": 0.3463692551603624, + "ngram_match_score": 0.22670813177777038, + "weighted_ngram_match_score": 0.25655894883190106, + "syntax_match_score": 0.420926517571885, + "dataflow_match_score": 0.48128342245989303 + }, + "chrf": { + "score": 32.89891416039852, + "char_order": 6, + "word_order": 0, + "beta": 2 + }, + "rouge": { + "rouge1": 0.2901406447162942, + "rouge2": 0.14277519898369662, + "rougeL": 0.2873019930085162, + "rougeLsum": 0.287600276754196 + }, + "bleu": { + "bleu": 0.2306103978206576, + "precisions": [ + 0.47367086467754205, + 0.2528162511542013, + 0.17550106826737205, + 0.13457181694608064 + ], + "brevity_penalty": 1.0, + "length_ratio": 1.1158162784117702, + "translation_length": 11831, + "reference_length": 10603 + }, + "rest": { + "exact_match": 0.013986013986013986, + "lev_dist": 34.19180819180819, + "mean_time": 0.3732499042591015, + "mean_energy": 55.00681618381511, + "std_time": 0.07158166555203227, + "std_energy": 14.931885141495968, + "total_energy": 55061.82299999893, + "total_time": 373.6231541633606, + "std_through": 5.527807783240863, + "mean_through": 41.218827695156804 + }, + "eval_time_s": 384.3628373146057 + }, + { + "exit_index": 27, + "codebleu": { + "codebleu": 0.3627157562055855, + "ngram_match_score": 0.2464942151229068, + "weighted_ngram_match_score": 0.2809624096720993, + "syntax_match_score": 0.42875399361022365, + "dataflow_match_score": 0.4946524064171123 + }, + "chrf": { + "score": 34.76999760879607, + "char_order": 6, + "word_order": 0, + "beta": 2 + }, + "rouge": { + "rouge1": 0.3107995751636172, + "rouge2": 0.1661677227332164, + "rougeL": 0.30788978023979907, + "rougeLsum": 0.308413718917109 + }, + "bleu": { + "bleu": 0.24907409850058812, + "precisions": [ + 0.4838899144151703, + 0.27168636072181, + 0.19433239209358613, + 0.15064498037016263 + ], + "brevity_penalty": 1.0, + "length_ratio": 1.1240215033481091, + "translation_length": 11918, + "reference_length": 10603 + }, + "rest": { + "exact_match": 0.016983016983016984, + "lev_dist": 33.38061938061938, + "mean_time": 0.43135855390832617, + "mean_energy": 64.00646053945327, + "std_time": 0.08339636601453994, + "std_energy": 16.417450825348574, + "total_energy": 64070.46699999273, + "total_time": 431.7899124622345, + "std_through": 4.823876445521664, + "mean_through": 35.67643863236937 + }, + "eval_time_s": 442.76964235305786 + }, + { + "exit_index": 31, + "codebleu": { + "codebleu": 0.3991263642967176, + "ngram_match_score": 0.27604689925592335, + "weighted_ngram_match_score": 0.31123951161361074, + "syntax_match_score": 0.4450479233226837, + "dataflow_match_score": 0.5641711229946524 + }, + "chrf": { + "score": 36.88821883531422, + "char_order": 6, + "word_order": 0, + "beta": 2 + }, + "rouge": { + "rouge1": 0.33839891481634476, + "rouge2": 0.19533754485053206, + "rougeL": 0.3353110278328256, + "rougeLsum": 0.3356657620541966 + }, + "bleu": { + "bleu": 0.2783191796452495, + "precisions": [ + 0.49826843483402317, + 0.29885587746816755, + 0.22364542035173327, + 0.18017202354006337 + ], + "brevity_penalty": 1.0, + "length_ratio": 1.1165707818541921, + "translation_length": 11839, + "reference_length": 10603 + }, + "rest": { + "exact_match": 0.028971028971028972, + "lev_dist": 32.09390609390609, + "mean_time": 0.48922354739148183, + "mean_energy": 71.8862967032876, + "std_time": 0.09483094935342311, + "std_energy": 19.764103419251658, + "total_energy": 71958.18299999088, + "total_time": 489.7127709388733, + "std_through": 4.268431025505318, + "mean_through": 31.447385129456674 + }, + "eval_time_s": 502.876544713974 + } +] \ No newline at end of file diff --git a/results/replication/opt_java/eval_exit_11.json b/results/replication/opt_java/eval_exit_11.json new file mode 100644 index 0000000..fed5cd0 --- /dev/null +++ b/results/replication/opt_java/eval_exit_11.json @@ -0,0 +1,48 @@ +{ + "exit_index": 11, + "codebleu": { + "codebleu": 0.33120149846093827, + "ngram_match_score": 0.20701923639901434, + "weighted_ngram_match_score": 0.24382509610288555, + "syntax_match_score": 0.37396166134185305, + "dataflow_match_score": 0.5 + }, + "chrf": { + "score": 28.82049173604128, + "char_order": 6, + "word_order": 0, + "beta": 2 + }, + "rouge": { + "rouge1": 0.25585947109616974, + "rouge2": 0.13293999263030254, + "rougeL": 0.2542704145066956, + "rougeLsum": 0.2547394210624174 + }, + "bleu": { + "bleu": 0.20618886027628378, + "precisions": [ + 0.43295929232533376, + 0.22376873661670235, + 0.1565592240619183, + 0.11916141646752118 + ], + "brevity_penalty": 1.0, + "length_ratio": 1.1514665660662078, + "translation_length": 12209, + "reference_length": 10603 + }, + "rest": { + "exact_match": 0.003996003996003996, + "lev_dist": 36.396603396603396, + "mean_time": 0.19899982481926948, + "mean_energy": 27.99878421578404, + "std_time": 0.036461068208114095, + "std_energy": 8.586884540226105, + "total_energy": 28026.78299999982, + "total_time": 199.19882464408875, + "std_through": 9.934190896429508, + "mean_through": 77.16365839145136 + }, + "eval_time_s": 209.05940532684326 +} \ No newline at end of file diff --git a/results/replication/opt_java/eval_exit_13.json b/results/replication/opt_java/eval_exit_13.json new file mode 100644 index 0000000..f7d30cb --- /dev/null +++ b/results/replication/opt_java/eval_exit_13.json @@ -0,0 +1,48 @@ +{ + "exit_index": 13, + "codebleu": { + "codebleu": 0.33221771923670806, + "ngram_match_score": 0.2104466754551522, + "weighted_ngram_match_score": 0.24788551259178088, + "syntax_match_score": 0.3865814696485623, + "dataflow_match_score": 0.4839572192513369 + }, + "chrf": { + "score": 29.288377230375783, + "char_order": 6, + "word_order": 0, + "beta": 2 + }, + "rouge": { + "rouge1": 0.2650741303270669, + "rouge2": 0.13803171936288838, + "rougeL": 0.26144219694427273, + "rougeLsum": 0.2616926126452208 + }, + "bleu": { + "bleu": 0.21055753937088106, + "precisions": [ + 0.4401471790678659, + 0.23109804969275982, + 0.16041055718475072, + 0.12046365507528979 + ], + "brevity_penalty": 1.0, + "length_ratio": 1.1534471376025652, + "translation_length": 12230, + "reference_length": 10603 + }, + "rest": { + "exact_match": 0.003996003996003996, + "lev_dist": 35.75924075924076, + "mean_time": 0.22799557167571502, + "mean_energy": 32.54528171828874, + "std_time": 0.04219700284697554, + "std_energy": 9.127248044267345, + "total_energy": 32577.827000007033, + "total_time": 228.22356724739075, + "std_through": 8.73251354572754, + "mean_through": 67.37692714441317 + }, + "eval_time_s": 238.83869338035583 +} \ No newline at end of file diff --git a/results/replication/opt_java/eval_exit_15.json b/results/replication/opt_java/eval_exit_15.json new file mode 100644 index 0000000..18177aa --- /dev/null +++ b/results/replication/opt_java/eval_exit_15.json @@ -0,0 +1,48 @@ +{ + "exit_index": 15, + "codebleu": { + "codebleu": 0.32981240935779493, + "ngram_match_score": 0.21022796177535755, + "weighted_ngram_match_score": 0.24786604872308576, + "syntax_match_score": 0.3798722044728435, + "dataflow_match_score": 0.48128342245989303 + }, + "chrf": { + "score": 29.37609781524177, + "char_order": 6, + "word_order": 0, + "beta": 2 + }, + "rouge": { + "rouge1": 0.2663668773809412, + "rouge2": 0.13770367161975577, + "rougeL": 0.26332676304480884, + "rougeLsum": 0.2643884092531812 + }, + "bleu": { + "bleu": 0.2094901792722669, + "precisions": [ + 0.438232421875, + 0.22831576149552582, + 0.15950622083981336, + 0.12068037463666703 + ], + "brevity_penalty": 1.0, + "length_ratio": 1.1589172875601246, + "translation_length": 12288, + "reference_length": 10603 + }, + "rest": { + "exact_match": 0.005994005994005994, + "lev_dist": 35.54645354645355, + "mean_time": 0.2570528831634369, + "mean_energy": 37.07245754245129, + "std_time": 0.04800471696727656, + "std_energy": 10.962740650949112, + "total_energy": 37109.52999999374, + "total_time": 257.30993604660034, + "std_through": 7.813241217013678, + "mean_through": 59.78180284233567 + }, + "eval_time_s": 267.66468238830566 +} \ No newline at end of file diff --git a/results/replication/opt_java/eval_exit_19.json b/results/replication/opt_java/eval_exit_19.json new file mode 100644 index 0000000..769f33f --- /dev/null +++ b/results/replication/opt_java/eval_exit_19.json @@ -0,0 +1,48 @@ +{ + "exit_index": 19, + "codebleu": { + "codebleu": 0.33292683979730503, + "ngram_match_score": 0.2050238350348228, + "weighted_ngram_match_score": 0.23658767750902993, + "syntax_match_score": 0.3900958466453674, + "dataflow_match_score": 0.5 + }, + "chrf": { + "score": 29.928778613235725, + "char_order": 6, + "word_order": 0, + "beta": 2 + }, + "rouge": { + "rouge1": 0.258132344421675, + "rouge2": 0.12731499848867794, + "rougeL": 0.25625157353050265, + "rougeLsum": 0.2562569533657273 + }, + "bleu": { + "bleu": 0.20768210905216453, + "precisions": [ + 0.4447119375726382, + 0.22634676324128564, + 0.15568385427035636, + 0.11871338565270255 + ], + "brevity_penalty": 1.0, + "length_ratio": 1.1360935584268603, + "translation_length": 12046, + "reference_length": 10603 + }, + "rest": { + "exact_match": 0.011988011988011988, + "lev_dist": 35.75924075924076, + "mean_time": 0.3145247920528873, + "mean_energy": 45.84630869130244, + "std_time": 0.0602302781392137, + "std_energy": 12.234466055263793, + "total_energy": 45892.15499999374, + "total_time": 314.8393168449402, + "std_through": 6.5245759425711265, + "mean_through": 48.786965227281975 + }, + "eval_time_s": 326.15760707855225 +} \ No newline at end of file diff --git a/results/replication/opt_java/eval_exit_23.json b/results/replication/opt_java/eval_exit_23.json new file mode 100644 index 0000000..f0f6e92 --- /dev/null +++ b/results/replication/opt_java/eval_exit_23.json @@ -0,0 +1,48 @@ +{ + "exit_index": 23, + "codebleu": { + "codebleu": 0.3463692551603624, + "ngram_match_score": 0.22670813177777038, + "weighted_ngram_match_score": 0.25655894883190106, + "syntax_match_score": 0.420926517571885, + "dataflow_match_score": 0.48128342245989303 + }, + "chrf": { + "score": 32.89891416039852, + "char_order": 6, + "word_order": 0, + "beta": 2 + }, + "rouge": { + "rouge1": 0.2901406447162942, + "rouge2": 0.14277519898369662, + "rougeL": 0.2873019930085162, + "rougeLsum": 0.287600276754196 + }, + "bleu": { + "bleu": 0.2306103978206576, + "precisions": [ + 0.47367086467754205, + 0.2528162511542013, + 0.17550106826737205, + 0.13457181694608064 + ], + "brevity_penalty": 1.0, + "length_ratio": 1.1158162784117702, + "translation_length": 11831, + "reference_length": 10603 + }, + "rest": { + "exact_match": 0.013986013986013986, + "lev_dist": 34.19180819180819, + "mean_time": 0.3732499042591015, + "mean_energy": 55.00681618381511, + "std_time": 0.07158166555203227, + "std_energy": 14.931885141495968, + "total_energy": 55061.82299999893, + "total_time": 373.6231541633606, + "std_through": 5.527807783240863, + "mean_through": 41.218827695156804 + }, + "eval_time_s": 384.3628373146057 +} \ No newline at end of file diff --git a/results/replication/opt_java/eval_exit_27.json b/results/replication/opt_java/eval_exit_27.json new file mode 100644 index 0000000..3b429b6 --- /dev/null +++ b/results/replication/opt_java/eval_exit_27.json @@ -0,0 +1,48 @@ +{ + "exit_index": 27, + "codebleu": { + "codebleu": 0.3627157562055855, + "ngram_match_score": 0.2464942151229068, + "weighted_ngram_match_score": 0.2809624096720993, + "syntax_match_score": 0.42875399361022365, + "dataflow_match_score": 0.4946524064171123 + }, + "chrf": { + "score": 34.76999760879607, + "char_order": 6, + "word_order": 0, + "beta": 2 + }, + "rouge": { + "rouge1": 0.3107995751636172, + "rouge2": 0.1661677227332164, + "rougeL": 0.30788978023979907, + "rougeLsum": 0.308413718917109 + }, + "bleu": { + "bleu": 0.24907409850058812, + "precisions": [ + 0.4838899144151703, + 0.27168636072181, + 0.19433239209358613, + 0.15064498037016263 + ], + "brevity_penalty": 1.0, + "length_ratio": 1.1240215033481091, + "translation_length": 11918, + "reference_length": 10603 + }, + "rest": { + "exact_match": 0.016983016983016984, + "lev_dist": 33.38061938061938, + "mean_time": 0.43135855390832617, + "mean_energy": 64.00646053945327, + "std_time": 0.08339636601453994, + "std_energy": 16.417450825348574, + "total_energy": 64070.46699999273, + "total_time": 431.7899124622345, + "std_through": 4.823876445521664, + "mean_through": 35.67643863236937 + }, + "eval_time_s": 442.76964235305786 +} \ No newline at end of file diff --git a/results/replication/opt_java/eval_exit_3.json b/results/replication/opt_java/eval_exit_3.json new file mode 100644 index 0000000..0002375 --- /dev/null +++ b/results/replication/opt_java/eval_exit_3.json @@ -0,0 +1,48 @@ +{ + "exit_index": 3, + "codebleu": { + "codebleu": 0.3085419550981881, + "ngram_match_score": 0.19067456838371866, + "weighted_ngram_match_score": 0.22549509718509428, + "syntax_match_score": 0.35543130990415334, + "dataflow_match_score": 0.4625668449197861 + }, + "chrf": { + "score": 26.346815923235134, + "char_order": 6, + "word_order": 0, + "beta": 2 + }, + "rouge": { + "rouge1": 0.23502866734167188, + "rouge2": 0.12437429066666715, + "rougeL": 0.23316514868917615, + "rougeLsum": 0.23375735943570625 + }, + "bleu": { + "bleu": 0.18841011747985517, + "precisions": [ + 0.408708999838423, + 0.20435967302452315, + 0.1417558061096656, + 0.10643062813266503 + ], + "brevity_penalty": 1.0, + "length_ratio": 1.1674054512873715, + "translation_length": 12378, + "reference_length": 10603 + }, + "rest": { + "exact_match": 0.003996003996003996, + "lev_dist": 38.09090909090909, + "mean_time": 0.0835209745508093, + "mean_energy": 12.131913788013586, + "std_time": 0.014508562537882033, + "std_energy": 2.393643656542935, + "total_energy": 12144.045701801599, + "total_time": 83.60449552536011, + "std_through": 22.600805926068823, + "mean_through": 183.3797429552793 + }, + "eval_time_s": 97.11258339881897 +} \ No newline at end of file diff --git a/results/replication/opt_java/eval_exit_31.json b/results/replication/opt_java/eval_exit_31.json new file mode 100644 index 0000000..ceeec16 --- /dev/null +++ b/results/replication/opt_java/eval_exit_31.json @@ -0,0 +1,48 @@ +{ + "exit_index": 31, + "codebleu": { + "codebleu": 0.3991263642967176, + "ngram_match_score": 0.27604689925592335, + "weighted_ngram_match_score": 0.31123951161361074, + "syntax_match_score": 0.4450479233226837, + "dataflow_match_score": 0.5641711229946524 + }, + "chrf": { + "score": 36.88821883531422, + "char_order": 6, + "word_order": 0, + "beta": 2 + }, + "rouge": { + "rouge1": 0.33839891481634476, + "rouge2": 0.19533754485053206, + "rougeL": 0.3353110278328256, + "rougeLsum": 0.3356657620541966 + }, + "bleu": { + "bleu": 0.2783191796452495, + "precisions": [ + 0.49826843483402317, + 0.29885587746816755, + 0.22364542035173327, + 0.18017202354006337 + ], + "brevity_penalty": 1.0, + "length_ratio": 1.1165707818541921, + "translation_length": 11839, + "reference_length": 10603 + }, + "rest": { + "exact_match": 0.028971028971028972, + "lev_dist": 32.09390609390609, + "mean_time": 0.48922354739148183, + "mean_energy": 71.8862967032876, + "std_time": 0.09483094935342311, + "std_energy": 19.764103419251658, + "total_energy": 71958.18299999088, + "total_time": 489.7127709388733, + "std_through": 4.268431025505318, + "mean_through": 31.447385129456674 + }, + "eval_time_s": 502.876544713974 +} \ No newline at end of file diff --git a/results/replication/opt_java/eval_exit_5.json b/results/replication/opt_java/eval_exit_5.json new file mode 100644 index 0000000..595e54c --- /dev/null +++ b/results/replication/opt_java/eval_exit_5.json @@ -0,0 +1,48 @@ +{ + "exit_index": 5, + "codebleu": { + "codebleu": 0.3116000451857528, + "ngram_match_score": 0.19583596908350961, + "weighted_ngram_match_score": 0.2322465680176707, + "syntax_match_score": 0.35575079872204474, + "dataflow_match_score": 0.4625668449197861 + }, + "chrf": { + "score": 27.509499364602476, + "char_order": 6, + "word_order": 0, + "beta": 2 + }, + "rouge": { + "rouge1": 0.24760287562334615, + "rouge2": 0.1287703806703771, + "rougeL": 0.24618569927591427, + "rougeLsum": 0.24666538936015608 + }, + "bleu": { + "bleu": 0.19200024717807793, + "precisions": [ + 0.41269202927692433, + 0.20932470258922323, + 0.14560966257668712, + 0.10803647158608991 + ], + "brevity_penalty": 1.0, + "length_ratio": 1.1725926624540224, + "translation_length": 12433, + "reference_length": 10603 + }, + "rest": { + "exact_match": 0.001998001998001998, + "lev_dist": 37.7022977022977, + "mean_time": 0.11263286245690955, + "mean_energy": 14.654149850147528, + "std_time": 0.01926744220321838, + "std_energy": 5.193031602609825, + "total_energy": 14668.803999997675, + "total_time": 112.74549531936646, + "std_through": 16.709589051719462, + "mean_through": 135.96795867233908 + }, + "eval_time_s": 123.29629802703857 +} \ No newline at end of file diff --git a/results/replication/opt_java/eval_exit_7.json b/results/replication/opt_java/eval_exit_7.json new file mode 100644 index 0000000..da8fbd6 --- /dev/null +++ b/results/replication/opt_java/eval_exit_7.json @@ -0,0 +1,48 @@ +{ + "exit_index": 7, + "codebleu": { + "codebleu": 0.31729770479159014, + "ngram_match_score": 0.21048480986273896, + "weighted_ngram_match_score": 0.24257524099281183, + "syntax_match_score": 0.3669329073482428, + "dataflow_match_score": 0.44919786096256686 + }, + "chrf": { + "score": 27.994852690575495, + "char_order": 6, + "word_order": 0, + "beta": 2 + }, + "rouge": { + "rouge1": 0.2526891860010755, + "rouge2": 0.13690362274116008, + "rougeL": 0.25116825853929314, + "rougeLsum": 0.25160714377163584 + }, + "bleu": { + "bleu": 0.2074365721453364, + "precisions": [ + 0.42798115858193536, + 0.2245945945945946, + 0.15939015939015938, + 0.12085255987695012 + ], + "brevity_penalty": 1.0, + "length_ratio": 1.1412807695935112, + "translation_length": 12101, + "reference_length": 10603 + }, + "rest": { + "exact_match": 0.003996003996003996, + "lev_dist": 36.77622377622377, + "mean_time": 0.1412087634845928, + "mean_energy": 19.32034965033655, + "std_time": 0.024858495063354007, + "std_energy": 7.952233459576671, + "total_energy": 19339.669999986887, + "total_time": 141.3499722480774, + "std_through": 13.564945231147227, + "mean_through": 108.57195790893184 + }, + "eval_time_s": 151.60858178138733 +} \ No newline at end of file diff --git a/results/replication/opt_java/eval_exit_9.json b/results/replication/opt_java/eval_exit_9.json new file mode 100644 index 0000000..f286bfe --- /dev/null +++ b/results/replication/opt_java/eval_exit_9.json @@ -0,0 +1,48 @@ +{ + "exit_index": 9, + "codebleu": { + "codebleu": 0.3299713858015275, + "ngram_match_score": 0.2140210547595856, + "weighted_ngram_match_score": 0.2500299734032142, + "syntax_match_score": 0.3611821086261981, + "dataflow_match_score": 0.4946524064171123 + }, + "chrf": { + "score": 28.69955699307067, + "char_order": 6, + "word_order": 0, + "beta": 2 + }, + "rouge": { + "rouge1": 0.2584162173104967, + "rouge2": 0.14148841902303644, + "rougeL": 0.2568476311374282, + "rougeLsum": 0.2574396627317782 + }, + "bleu": { + "bleu": 0.21199501532394802, + "precisions": [ + 0.4338598223099704, + 0.22904527117884357, + 0.16336779911373708, + 0.12441288913162206 + ], + "brevity_penalty": 1.0, + "length_ratio": 1.1464679807601623, + "translation_length": 12156, + "reference_length": 10603 + }, + "rest": { + "exact_match": 0.005994005994005994, + "lev_dist": 36.64435564435564, + "mean_time": 0.1701601042256846, + "mean_energy": 24.00939460540866, + "std_time": 0.030554575006484837, + "std_energy": 8.554045285677534, + "total_energy": 24033.404000014067, + "total_time": 170.33026432991028, + "std_through": 11.413561760382295, + "mean_through": 90.1649261119857 + }, + "eval_time_s": 180.87501645088196 +} \ No newline at end of file diff --git a/results/replication/opt_java/phase1_log.txt b/results/replication/opt_java/phase1_log.txt new file mode 100644 index 0000000..7aec1d7 --- /dev/null +++ b/results/replication/opt_java/phase1_log.txt @@ -0,0 +1,150220 @@ +============================================================ +Phase 1: Fixed-Exit Evaluation — OPT-2.7B x JavaCorpus +Exit layers: [3, 5, 7, 9, 11, 13, 15, 19, 23, 27, 31] +============================================================ +Dataset: 8268 test samples + +--- Evaluating exit_index = 3 --- + Loading checkpoint shards: 0%| | 0/3 [00:00 + codebleu, chrf, rouge, bleu, rest = evaluator.evaluate( + ^^^^^^^^^^^^^^^^^^^ + File "/home/mluser/greencode/repo/src/eval/evaluator.py", line 84, in evaluate + output = self.model.generate(input_ids=input_ids, attention_mask=attention_mask, + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/mluser/greencode/venv/lib/python3.12/site-packages/torch/utils/_contextlib.py", line 116, in decorate_context + return func(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^ + File "/home/mluser/greencode/venv/lib/python3.12/site-packages/transformers/generation/utils.py", line 2047, in generate + result = self._sample( + ^^^^^^^^^^^^^ + File "/home/mluser/greencode/venv/lib/python3.12/site-packages/transformers/generation/utils.py", line 3007, in _sample + outputs = self(**model_inputs, return_dict=True) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/mluser/greencode/venv/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1553, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/mluser/greencode/venv/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1562, in _call_impl + return forward_call(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/mluser/greencode/repo/src/models/opt/modeling_opt_single_head_fixed_exit.py", line 576, in forward + raise ValueError(f"mode {self.mode} not supported") +ValueError: mode infer_ee not supported diff --git a/src/__pycache__/__init__.cpython-312.pyc b/src/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000..d912342 Binary files /dev/null and b/src/__pycache__/__init__.cpython-312.pyc differ diff --git a/src/eval/__pycache__/__init__.cpython-312.pyc b/src/eval/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000..f07d404 Binary files /dev/null and b/src/eval/__pycache__/__init__.cpython-312.pyc differ diff --git a/src/eval/__pycache__/evaluator.cpython-312.pyc b/src/eval/__pycache__/evaluator.cpython-312.pyc new file mode 100644 index 0000000..697de6d Binary files /dev/null and b/src/eval/__pycache__/evaluator.cpython-312.pyc differ diff --git a/src/eval/evaluator.py b/src/eval/evaluator.py index 8bcd9d4..04bb73a 100644 --- a/src/eval/evaluator.py +++ b/src/eval/evaluator.py @@ -9,7 +9,7 @@ from transformers import AutoTokenizer, AutoModelForCausalLM from zeus.monitor import ZeusMonitor -from src.models.opt.modeling_opt_single_head_fixed_exit import OPTFixedEESingleHead +from src.models.opt.modeling_opt_single_head_fixed_exit import OPTEESingleHead as OPTFixedEESingleHead class Evaluator: diff --git a/src/models/__pycache__/__init__.cpython-312.pyc b/src/models/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000..88bbf5a Binary files /dev/null and b/src/models/__pycache__/__init__.cpython-312.pyc differ diff --git a/src/models/opt/__pycache__/modeling_opt_ee_rl.cpython-312.pyc b/src/models/opt/__pycache__/modeling_opt_ee_rl.cpython-312.pyc new file mode 100644 index 0000000..d30fbd6 Binary files /dev/null and b/src/models/opt/__pycache__/modeling_opt_ee_rl.cpython-312.pyc differ diff --git a/src/models/opt/__pycache__/modeling_opt_single_head_fixed_exit.cpython-312.pyc b/src/models/opt/__pycache__/modeling_opt_single_head_fixed_exit.cpython-312.pyc new file mode 100644 index 0000000..e2ed312 Binary files /dev/null and b/src/models/opt/__pycache__/modeling_opt_single_head_fixed_exit.cpython-312.pyc differ diff --git a/src/models/rl/__pycache__/__init__.cpython-312.pyc b/src/models/rl/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000..14d67b0 Binary files /dev/null and b/src/models/rl/__pycache__/__init__.cpython-312.pyc differ diff --git a/src/models/rl/enviornments/__pycache__/__init__.cpython-312.pyc b/src/models/rl/enviornments/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000..16bf3e5 Binary files /dev/null and b/src/models/rl/enviornments/__pycache__/__init__.cpython-312.pyc differ diff --git a/src/models/rl/enviornments/__pycache__/sb3_torch_wrapper.cpython-312.pyc b/src/models/rl/enviornments/__pycache__/sb3_torch_wrapper.cpython-312.pyc new file mode 100644 index 0000000..ce8122f Binary files /dev/null and b/src/models/rl/enviornments/__pycache__/sb3_torch_wrapper.cpython-312.pyc differ diff --git a/src/requirements.txt b/src/requirements.txt index 660ef47..6785cae 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -1,5 +1,5 @@ accelerate==1.0.0 -codebleu==0.7.1 +codebleu==0.7.0 contourpy==1.3.0 datasets==2.20.0 evaluate==0.4.3 @@ -11,7 +11,7 @@ matplotlib==3.9.0 multiprocess==0.70.16 nltk==3.9.1 numpy==1.26.4 -nvidia-cuda-12-cu12==12.1.105 + pandas==2.2.2 rouge-score==0.1.2 sacrebleu==2.4.3 @@ -23,5 +23,5 @@ tensorboard==2.18.0 tokenizers==0.20.1 torch==2.4.1 transformers==4.45.2 -tree-sitter==0.23.1 +tree-sitter==0.22.3 zeus-ml==0.10.1 \ No newline at end of file