{
 "MBPP loop s0 k=0 (base)": {
  "overall": {
   "acc": 0.518,
   "n": 500,
   "ci": [
    0.474,
    0.561
   ]
  },
  "hard": {
   "acc": 0.05454545454545454,
   "n": 55,
   "ci": [
    0.019,
    0.149
   ]
  }
 },
 "MBPP loop s0 k=2": {
  "overall": {
   "acc": 0.53,
   "n": 500,
   "ci": [
    0.486,
    0.573
   ]
  },
  "hard": {
   "acc": 0.3090909090909091,
   "n": 55,
   "ci": [
    0.203,
    0.44
   ]
  }
 },
 "MBPP loop s0 k=4": {
  "overall": {
   "acc": 0.536,
   "n": 500,
   "ci": [
    0.492,
    0.579
   ]
  },
  "hard": {
   "acc": 0.43636363636363634,
   "n": 55,
   "ci": [
    0.314,
    0.567
   ]
  }
 },
 "MBPP distill s1 k=1 (FF)": {
  "overall": {
   "acc": 0.544,
   "n": 500,
   "ci": [
    0.5,
    0.587
   ]
  },
  "hard": {
   "acc": 0.41818181818181815,
   "n": 55,
   "ci": [
    0.297,
    0.55
   ]
  }
 },
 "MBPP pause16 k=1": {
  "overall": {
   "acc": 0.552,
   "n": 500,
   "ci": [
    0.508,
    0.595
   ]
  },
  "hard": {
   "acc": 0.36363636363636365,
   "n": 55,
   "ci": [
    0.249,
    0.496
   ]
  }
 },
 "MBPP stack-train k=4": {
  "overall": {
   "acc": 0.524,
   "n": 500,
   "ci": [
    0.48,
    0.567
   ]
  },
  "hard": {
   "acc": 0.34545454545454546,
   "n": 55,
   "ci": [
    0.234,
    0.477
   ]
  }
 },
 "MBPP distill-in-loopmode k=2": {
  "overall": {
   "acc": 0.458,
   "n": 500,
   "ci": [
    0.415,
    0.502
   ]
  },
  "hard": {
   "acc": 0.2,
   "n": 55,
   "ci": [
    0.116,
    0.324
   ]
  }
 },
 "Rust transfer k=0": {
  "overall": {
   "acc": 0.5909090909090909,
   "n": 154,
   "ci": [
    0.512,
    0.665
   ]
  },
  "hard": {
   "acc": 0.08,
   "n": 25,
   "ci": [
    0.022,
    0.25
   ]
  }
 },
 "Rust transfer k=4": {
  "overall": {
   "acc": 0.551948051948052,
   "n": 154,
   "ci": [
    0.473,
    0.628
   ]
  },
  "hard": {
   "acc": 0.24,
   "n": 25,
   "ci": [
    0.115,
    0.434
   ]
  }
 },
 "HumanEval loop k=4": {
  "overall": {
   "acc": 0.6646341463414634,
   "n": 164,
   "ci": [
    0.589,
    0.732
   ]
  },
  "hard": {
   "acc": 0.3157894736842105,
   "n": 38,
   "ci": [
    0.191,
    0.475
   ]
  }
 },
 "HumanEval distill k=1": {
  "overall": {
   "acc": 0.6463414634146342,
   "n": 164,
   "ci": [
    0.571,
    0.715
   ]
  },
  "hard": {
   "acc": 0.23684210526315788,
   "n": 38,
   "ci": [
    0.13,
    0.392
   ]
  }
 },
 "MBPP best-of-3 (compute-matched)": {
  "overall": {
   "acc": 0.572,
   "n": 500,
   "ci": [
    0.528,
    0.615
   ]
  },
  "hard": {
   "acc": 0.32727272727272727,
   "n": 55,
   "ci": [
    0.218,
    0.459
   ]
  }
 },
 "MBPP budget-CoT-50": {
  "overall": {
   "acc": 0.538,
   "n": 500,
   "ci": [
    0.494,
    0.581
   ]
  },
  "hard": {
   "acc": 0.4,
   "n": 55,
   "ci": [
    0.281,
    0.532
   ]
  }
 },
 "distill_seed_spread": {
  "hard": [
   0.4909090909090909,
   0.41818181818181815,
   0.5454545454545454,
   0.43636363636363634,
   0.45454545454545453,
   0.4727272727272727,
   0.43636363636363634,
   0.4
  ],
  "overall": [
   0.55,
   0.544,
   0.562,
   0.546,
   0.566,
   0.562,
   0.564,
   0.55
  ]
 },
 "loop_seed_spread_hard": [
  0.43636363636363634,
  0.41818181818181815,
  0.3090909090909091,
  0.32727272727272727,
  0.38181818181818183
 ],
 "mcnemar": {
  "loop k=4 vs k=0, overall": {
   "n": 500,
   "a_only": 30,
   "b_only": 39,
   "p": 0.33555761823401514
  },
  "loop k=4 vs k=0, hard": {
   "n": 55,
   "a_only": 1,
   "b_only": 22,
   "p": 5.7220458984375e-06
  },
  "distill k=1 vs loop k=4, overall": {
   "n": 500,
   "a_only": 33,
   "b_only": 37,
   "p": 0.7202027723528613
  },
  "distill k=1 vs loop k=4, hard": {
   "n": 55,
   "a_only": 10,
   "b_only": 9,
   "p": 1.0
  },
  "stack-train k=4 vs distill k=1, hard": {
   "n": 55,
   "a_only": 10,
   "b_only": 6,
   "p": 0.454498291015625
  },
  "HumanEval loop k=4 vs k=0, overall": {
   "n": 164,
   "a_only": 5,
   "b_only": 18,
   "p": 0.010622024536132812
  },
  "HumanEval distill k=1 vs k=0, overall": {
   "n": 164,
   "a_only": 7,
   "b_only": 17,
   "p": 0.06391465663909912
  },
  "HumanEval trained vs UNTRAINED merge (k=2), overall": {
   "n": 164,
   "a_only": 7,
   "b_only": 9,
   "p": 0.803619384765625
  },
  "Rust loop k=4 vs k=0, overall": {
   "n": 154,
   "a_only": 11,
   "b_only": 5,
   "p": 0.210113525390625
  },
  "Rust loop k=4 vs k=0, hard": {
   "n": 25,
   "a_only": 0,
   "b_only": 4,
   "p": 0.125
  },
  "bo3-oracle vs loop k=4, overall": {
   "n": 500,
   "a_only": 27,
   "b_only": 48,
   "p": 0.020298406990107445
  },
  "bo3-oracle vs loop k=4, hard": {
   "n": 55,
   "a_only": 14,
   "b_only": 9,
   "p": 0.4048728942871094
  },
  "bo3-oracle vs distill k=1, overall": {
   "n": 500,
   "a_only": 27,
   "b_only": 44,
   "p": 0.056814677932839015
  },
  "bo3-oracle vs distill k=1, hard": {
   "n": 55,
   "a_only": 14,
   "b_only": 10,
   "p": 0.5412561893463135
  },
  "bo3-deployable vs loop k=4, overall": {
   "n": 500,
   "a_only": 31,
   "b_only": 38,
   "p": 0.4703685318581444
  },
  "bo3-deployable vs loop k=4, hard": {
   "n": 55,
   "a_only": 16,
   "b_only": 7,
   "p": 0.0931396484375
  },
  "bo3-deployable vs distill k=1, overall": {
   "n": 500,
   "a_only": 31,
   "b_only": 34,
   "p": 0.8043170001933986
  },
  "bo3-deployable vs distill k=1, hard": {
   "n": 55,
   "a_only": 15,
   "b_only": 7,
   "p": 0.13380050659179688
  },
  "budget-cot vs loop k=4, overall": {
   "n": 500,
   "a_only": 36,
   "b_only": 37,
   "p": 1.0
  },
  "budget-cot vs loop k=4, hard": {
   "n": 55,
   "a_only": 14,
   "b_only": 11,
   "p": 0.6900379657745361
  },
  "budget-cot vs distill k=1, overall": {
   "n": 500,
   "a_only": 29,
   "b_only": 26,
   "p": 0.7877061896700435
  },
  "budget-cot vs distill k=1, hard": {
   "n": 55,
   "a_only": 8,
   "b_only": 6,
   "p": 0.79052734375
  }
 },
 "best-of-3 ORACLE (any-pass)": {
  "overall": {
   "acc": 0.578,
   "n": 500,
   "ci": [
    0.534,
    0.621
   ]
  },
  "hard": {
   "acc": 0.34545454545454546,
   "n": 55,
   "ci": [
    0.234,
    0.477
   ]
  }
 },
 "best-of-3 oracle (selector run)": {
  "overall": {
   "acc": 0.578,
   "n": 500,
   "ci": [
    0.534,
    0.621
   ]
  },
  "hard": {
   "acc": 0.34545454545454546,
   "n": 55,
   "ci": [
    0.234,
    0.477
   ]
  }
 },
 "best-of-3 DEPLOYABLE (logprob-selected)": {
  "overall": {
   "acc": 0.55,
   "n": 500,
   "ci": [
    0.506,
    0.593
   ]
  },
  "hard": {
   "acc": 0.2727272727272727,
   "n": 55,
   "ci": [
    0.173,
    0.402
   ]
  }
 },
 "pooled_hard_loop": {
  "base": 5,
  "arm": 42,
  "n": 118,
  "mcnemar": {
   "n": 118,
   "a_only": 1,
   "b_only": 38,
   "p": 1.4551915228366852e-10
  }
 },
 "pooled_hard_distill": {
  "base": 3,
  "arm": 32,
  "n": 93,
  "mcnemar": {
   "n": 93,
   "a_only": 1,
   "b_only": 30,
   "p": 2.9802322387695312e-08
  }
 },
 "consensus_hard": {
  "MBPP loop s0 k=4": {
   "acc": 0.4230769230769231,
   "n": 52,
   "ci": [
    0.299,
    0.558
   ]
  },
  "MBPP distill s1 k=1 (FF)": {
   "acc": 0.40384615384615385,
   "n": 52,
   "ci": [
    0.282,
    0.539
   ]
  },
  "MBPP stack-train k=4": {
   "acc": 0.34615384615384615,
   "n": 52,
   "ci": [
    0.232,
    0.482
   ]
  }
 }
}