477 lines
6.6 KiB
JSON
477 lines
6.6 KiB
JSON
{
|
|
"MBPP loop s0 k=0 (base)": {
|
|
"overall": {
|
|
"acc": 0.518,
|
|
"n": 500,
|
|
"ci": [
|
|
0.474,
|
|
0.561
|
|
]
|
|
},
|
|
"hard": {
|
|
"acc": 0.05454545454545454,
|
|
"n": 55,
|
|
"ci": [
|
|
0.019,
|
|
0.149
|
|
]
|
|
}
|
|
},
|
|
"MBPP loop s0 k=2": {
|
|
"overall": {
|
|
"acc": 0.53,
|
|
"n": 500,
|
|
"ci": [
|
|
0.486,
|
|
0.573
|
|
]
|
|
},
|
|
"hard": {
|
|
"acc": 0.3090909090909091,
|
|
"n": 55,
|
|
"ci": [
|
|
0.203,
|
|
0.44
|
|
]
|
|
}
|
|
},
|
|
"MBPP loop s0 k=4": {
|
|
"overall": {
|
|
"acc": 0.536,
|
|
"n": 500,
|
|
"ci": [
|
|
0.492,
|
|
0.579
|
|
]
|
|
},
|
|
"hard": {
|
|
"acc": 0.43636363636363634,
|
|
"n": 55,
|
|
"ci": [
|
|
0.314,
|
|
0.567
|
|
]
|
|
}
|
|
},
|
|
"MBPP distill s1 k=1 (FF)": {
|
|
"overall": {
|
|
"acc": 0.544,
|
|
"n": 500,
|
|
"ci": [
|
|
0.5,
|
|
0.587
|
|
]
|
|
},
|
|
"hard": {
|
|
"acc": 0.41818181818181815,
|
|
"n": 55,
|
|
"ci": [
|
|
0.297,
|
|
0.55
|
|
]
|
|
}
|
|
},
|
|
"MBPP pause16 k=1": {
|
|
"overall": {
|
|
"acc": 0.552,
|
|
"n": 500,
|
|
"ci": [
|
|
0.508,
|
|
0.595
|
|
]
|
|
},
|
|
"hard": {
|
|
"acc": 0.36363636363636365,
|
|
"n": 55,
|
|
"ci": [
|
|
0.249,
|
|
0.496
|
|
]
|
|
}
|
|
},
|
|
"MBPP stack-train k=4": {
|
|
"overall": {
|
|
"acc": 0.524,
|
|
"n": 500,
|
|
"ci": [
|
|
0.48,
|
|
0.567
|
|
]
|
|
},
|
|
"hard": {
|
|
"acc": 0.34545454545454546,
|
|
"n": 55,
|
|
"ci": [
|
|
0.234,
|
|
0.477
|
|
]
|
|
}
|
|
},
|
|
"MBPP distill-in-loopmode k=2": {
|
|
"overall": {
|
|
"acc": 0.458,
|
|
"n": 500,
|
|
"ci": [
|
|
0.415,
|
|
0.502
|
|
]
|
|
},
|
|
"hard": {
|
|
"acc": 0.2,
|
|
"n": 55,
|
|
"ci": [
|
|
0.116,
|
|
0.324
|
|
]
|
|
}
|
|
},
|
|
"Rust transfer k=0": {
|
|
"overall": {
|
|
"acc": 0.5909090909090909,
|
|
"n": 154,
|
|
"ci": [
|
|
0.512,
|
|
0.665
|
|
]
|
|
},
|
|
"hard": {
|
|
"acc": 0.08,
|
|
"n": 25,
|
|
"ci": [
|
|
0.022,
|
|
0.25
|
|
]
|
|
}
|
|
},
|
|
"Rust transfer k=4": {
|
|
"overall": {
|
|
"acc": 0.551948051948052,
|
|
"n": 154,
|
|
"ci": [
|
|
0.473,
|
|
0.628
|
|
]
|
|
},
|
|
"hard": {
|
|
"acc": 0.24,
|
|
"n": 25,
|
|
"ci": [
|
|
0.115,
|
|
0.434
|
|
]
|
|
}
|
|
},
|
|
"HumanEval loop k=4": {
|
|
"overall": {
|
|
"acc": 0.6646341463414634,
|
|
"n": 164,
|
|
"ci": [
|
|
0.589,
|
|
0.732
|
|
]
|
|
},
|
|
"hard": {
|
|
"acc": 0.3157894736842105,
|
|
"n": 38,
|
|
"ci": [
|
|
0.191,
|
|
0.475
|
|
]
|
|
}
|
|
},
|
|
"HumanEval distill k=1": {
|
|
"overall": {
|
|
"acc": 0.6463414634146342,
|
|
"n": 164,
|
|
"ci": [
|
|
0.571,
|
|
0.715
|
|
]
|
|
},
|
|
"hard": {
|
|
"acc": 0.23684210526315788,
|
|
"n": 38,
|
|
"ci": [
|
|
0.13,
|
|
0.392
|
|
]
|
|
}
|
|
},
|
|
"MBPP best-of-3 (compute-matched)": {
|
|
"overall": {
|
|
"acc": 0.572,
|
|
"n": 500,
|
|
"ci": [
|
|
0.528,
|
|
0.615
|
|
]
|
|
},
|
|
"hard": {
|
|
"acc": 0.32727272727272727,
|
|
"n": 55,
|
|
"ci": [
|
|
0.218,
|
|
0.459
|
|
]
|
|
}
|
|
},
|
|
"MBPP budget-CoT-50": {
|
|
"overall": {
|
|
"acc": 0.538,
|
|
"n": 500,
|
|
"ci": [
|
|
0.494,
|
|
0.581
|
|
]
|
|
},
|
|
"hard": {
|
|
"acc": 0.4,
|
|
"n": 55,
|
|
"ci": [
|
|
0.281,
|
|
0.532
|
|
]
|
|
}
|
|
},
|
|
"distill_seed_spread": {
|
|
"hard": [
|
|
0.4909090909090909,
|
|
0.41818181818181815,
|
|
0.5454545454545454,
|
|
0.43636363636363634,
|
|
0.45454545454545453,
|
|
0.4727272727272727,
|
|
0.43636363636363634,
|
|
0.4
|
|
],
|
|
"overall": [
|
|
0.55,
|
|
0.544,
|
|
0.562,
|
|
0.546,
|
|
0.566,
|
|
0.562,
|
|
0.564,
|
|
0.55
|
|
]
|
|
},
|
|
"loop_seed_spread_hard": [
|
|
0.43636363636363634,
|
|
0.41818181818181815,
|
|
0.3090909090909091,
|
|
0.32727272727272727,
|
|
0.38181818181818183
|
|
],
|
|
"mcnemar": {
|
|
"loop k=4 vs k=0, overall": {
|
|
"n": 500,
|
|
"a_only": 30,
|
|
"b_only": 39,
|
|
"p": 0.33555761823401514
|
|
},
|
|
"loop k=4 vs k=0, hard": {
|
|
"n": 55,
|
|
"a_only": 1,
|
|
"b_only": 22,
|
|
"p": 5.7220458984375e-06
|
|
},
|
|
"distill k=1 vs loop k=4, overall": {
|
|
"n": 500,
|
|
"a_only": 33,
|
|
"b_only": 37,
|
|
"p": 0.7202027723528613
|
|
},
|
|
"distill k=1 vs loop k=4, hard": {
|
|
"n": 55,
|
|
"a_only": 10,
|
|
"b_only": 9,
|
|
"p": 1.0
|
|
},
|
|
"stack-train k=4 vs distill k=1, hard": {
|
|
"n": 55,
|
|
"a_only": 10,
|
|
"b_only": 6,
|
|
"p": 0.454498291015625
|
|
},
|
|
"HumanEval loop k=4 vs k=0, overall": {
|
|
"n": 164,
|
|
"a_only": 5,
|
|
"b_only": 18,
|
|
"p": 0.010622024536132812
|
|
},
|
|
"HumanEval distill k=1 vs k=0, overall": {
|
|
"n": 164,
|
|
"a_only": 7,
|
|
"b_only": 17,
|
|
"p": 0.06391465663909912
|
|
},
|
|
"HumanEval trained vs UNTRAINED merge (k=2), overall": {
|
|
"n": 164,
|
|
"a_only": 7,
|
|
"b_only": 9,
|
|
"p": 0.803619384765625
|
|
},
|
|
"Rust loop k=4 vs k=0, overall": {
|
|
"n": 154,
|
|
"a_only": 11,
|
|
"b_only": 5,
|
|
"p": 0.210113525390625
|
|
},
|
|
"Rust loop k=4 vs k=0, hard": {
|
|
"n": 25,
|
|
"a_only": 0,
|
|
"b_only": 4,
|
|
"p": 0.125
|
|
},
|
|
"bo3-oracle vs loop k=4, overall": {
|
|
"n": 500,
|
|
"a_only": 27,
|
|
"b_only": 48,
|
|
"p": 0.020298406990107445
|
|
},
|
|
"bo3-oracle vs loop k=4, hard": {
|
|
"n": 55,
|
|
"a_only": 14,
|
|
"b_only": 9,
|
|
"p": 0.4048728942871094
|
|
},
|
|
"bo3-oracle vs distill k=1, overall": {
|
|
"n": 500,
|
|
"a_only": 27,
|
|
"b_only": 44,
|
|
"p": 0.056814677932839015
|
|
},
|
|
"bo3-oracle vs distill k=1, hard": {
|
|
"n": 55,
|
|
"a_only": 14,
|
|
"b_only": 10,
|
|
"p": 0.5412561893463135
|
|
},
|
|
"bo3-deployable vs loop k=4, overall": {
|
|
"n": 500,
|
|
"a_only": 31,
|
|
"b_only": 38,
|
|
"p": 0.4703685318581444
|
|
},
|
|
"bo3-deployable vs loop k=4, hard": {
|
|
"n": 55,
|
|
"a_only": 16,
|
|
"b_only": 7,
|
|
"p": 0.0931396484375
|
|
},
|
|
"bo3-deployable vs distill k=1, overall": {
|
|
"n": 500,
|
|
"a_only": 31,
|
|
"b_only": 34,
|
|
"p": 0.8043170001933986
|
|
},
|
|
"bo3-deployable vs distill k=1, hard": {
|
|
"n": 55,
|
|
"a_only": 15,
|
|
"b_only": 7,
|
|
"p": 0.13380050659179688
|
|
}
|
|
},
|
|
"best-of-3 ORACLE (any-pass)": {
|
|
"overall": {
|
|
"acc": 0.578,
|
|
"n": 500,
|
|
"ci": [
|
|
0.534,
|
|
0.621
|
|
]
|
|
},
|
|
"hard": {
|
|
"acc": 0.34545454545454546,
|
|
"n": 55,
|
|
"ci": [
|
|
0.234,
|
|
0.477
|
|
]
|
|
}
|
|
},
|
|
"best-of-3 oracle (selector run)": {
|
|
"overall": {
|
|
"acc": 0.578,
|
|
"n": 500,
|
|
"ci": [
|
|
0.534,
|
|
0.621
|
|
]
|
|
},
|
|
"hard": {
|
|
"acc": 0.34545454545454546,
|
|
"n": 55,
|
|
"ci": [
|
|
0.234,
|
|
0.477
|
|
]
|
|
}
|
|
},
|
|
"best-of-3 DEPLOYABLE (logprob-selected)": {
|
|
"overall": {
|
|
"acc": 0.55,
|
|
"n": 500,
|
|
"ci": [
|
|
0.506,
|
|
0.593
|
|
]
|
|
},
|
|
"hard": {
|
|
"acc": 0.2727272727272727,
|
|
"n": 55,
|
|
"ci": [
|
|
0.173,
|
|
0.402
|
|
]
|
|
}
|
|
},
|
|
"pooled_hard_loop": {
|
|
"base": 5,
|
|
"arm": 42,
|
|
"n": 118,
|
|
"mcnemar": {
|
|
"n": 118,
|
|
"a_only": 1,
|
|
"b_only": 38,
|
|
"p": 1.4551915228366852e-10
|
|
}
|
|
},
|
|
"pooled_hard_distill": {
|
|
"base": 3,
|
|
"arm": 32,
|
|
"n": 93,
|
|
"mcnemar": {
|
|
"n": 93,
|
|
"a_only": 1,
|
|
"b_only": 30,
|
|
"p": 2.9802322387695312e-08
|
|
}
|
|
},
|
|
"consensus_hard": {
|
|
"MBPP loop s0 k=4": {
|
|
"acc": 0.4230769230769231,
|
|
"n": 52,
|
|
"ci": [
|
|
0.299,
|
|
0.558
|
|
]
|
|
},
|
|
"MBPP distill s1 k=1 (FF)": {
|
|
"acc": 0.40384615384615385,
|
|
"n": 52,
|
|
"ci": [
|
|
0.282,
|
|
0.539
|
|
]
|
|
},
|
|
"MBPP stack-train k=4": {
|
|
"acc": 0.34615384615384615,
|
|
"n": 52,
|
|
"ci": [
|
|
0.232,
|
|
0.482
|
|
]
|
|
}
|
|
}
|
|
} |